Merge branch 'maint-0.4.2' into bug40076_042

891 files changed, 64626 insertions, 18980 deletions

diff --git a/src/app/config/config.c b/src/app/config/config.c
index 1d61b76310..f27fcd3108 100644
--- a/src/app/config/config.c
+++ b/src/app/config/config.c
@@ -61,9 +61,10 @@
 #define CONFIG_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "app/config/statefile.h"
 #include "app/main/main.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
@@ -85,6 +86,8 @@
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/guardfraction.h"
 #include "feature/dircache/consdiffmgr.h"
@@ -108,13 +111,14 @@
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
+#include "lib/confmgt/structvar.h"
 #include "lib/crypt_ops/crypto_init.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/encoding/confline.h"
-#include "lib/log/git_revision.h"
 #include "lib/net/resolve.h"
 #include "lib/sandbox/sandbox.h"
+#include "lib/version/torversion.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
@@ -144,7 +148,7 @@
 #include "lib/process/pidfile.h"
 #include "lib/process/restrict.h"
 #include "lib/process/setuid.h"
-#include "lib/process/subprocess.h"
+#include "lib/process/process.h"
 #include "lib/net/gethostname.h"
 #include "lib/thread/numcpus.h"
 
@@ -153,6 +157,7 @@
 #include "lib/evloop/procmon.h"
 
 #include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
 #include "feature/dirauth/recommend_pkg.h"
 #include "feature/dirauth/authmode.h"
 
@@ -186,7 +191,7 @@ static const char unix_q_socket_prefix[] = "unix:\"";
 
 /** A list of abbreviations and aliases to map command-line options, obsolete
  * option names, or alternative option names, to their current values. */
-static config_abbrev_t option_abbrevs_[] = {
+static const config_abbrev_t option_abbrevs_[] = {
   PLURAL(AuthDirBadDirCC),
   PLURAL(AuthDirBadExitCC),
   PLURAL(AuthDirInvalidCC),
@@ -249,22 +254,33 @@ static config_abbrev_t option_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_options_t);
 
-/** An entry for config_vars: "The option <b>name</b> has type
+/** An entry for config_vars: "The option <b>varname</b> has type
  * CONFIG_TYPE_<b>conftype</b>, and corresponds to
  * or_options_t.<b>member</b>"
  */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member),     \
-      initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, 0, initvalue)
+
+/* As VAR, but uses a type definition in addition to a type enum. */
+#define VAR_D(varname,conftype,member,initvalue)                        \
+  CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, 0, initvalue)
+
+#define VAR_NODUMP(varname,conftype,member,initvalue)             \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP, initvalue)
+#define VAR_INVIS(varname,conftype,member,initvalue)              \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP | CFLG_NOSET | CFLG_NOLIST, initvalue)
+
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
-/** An entry for config_vars: "The option <b>name</b> is obsolete." */
-#ifdef TOR_UNIT_TESTS
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} }
-#else
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL }
-#endif
+
+/** As V, but uses a type definition instead of a type enum */
+#define V_D(member,type,initvalue)              \
+  VAR_D(#member, type, member, initvalue)
+
+/** An entry for config_vars: "The option <b>varname</b> is obsolete." */
+#define OBSOLETE(varname) CONFIG_VAR_OBSOLETE(varname)
 
 /**
  * Macro to declare *Port options.  Each one comes in three entries.
@@ -276,7 +292,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
 #define VPORT(member)                                           \
   VAR(#member "Lines", LINELIST_V, member ## _lines, NULL),     \
   VAR(#member, LINELIST_S, member ## _lines, NULL),             \
-  VAR("__" #member, LINELIST_S, member ## _lines, NULL)
+  VAR_NODUMP("__" #member, LINELIST_S, member ## _lines, NULL)
 
 /** UINT64_MAX as a decimal string */
 #define UINT64_MAX_STRING "18446744073709551615"
@@ -285,7 +301,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
  * abbreviations, order is significant, since the first matching option will
  * be chosen first.
  */
-static config_var_t option_vars_[] = {
+static const config_var_t option_vars_[] = {
   V(AccountingMax,               MEMUNIT,  "0 bytes"),
   VAR("AccountingRule",          STRING,   AccountingRule_option,  "max"),
   V(AccountingStart,             STRING,   NULL),
@@ -313,7 +329,7 @@ static config_var_t option_vars_[] = {
   OBSOLETE("AuthDirRejectUnlisted"),
   OBSOLETE("AuthDirListBadDirs"),
   V(AuthDirListBadExits,         BOOL,     "0"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "2"),
+  V(AuthDirMaxServersPerAddr,    POSINT,     "2"),
   OBSOLETE("AuthDirMaxServersPerAuthAddr"),
   V(AuthDirHasIPv6Connectivity,  BOOL,     "0"),
   VAR("AuthoritativeDirectory",  BOOL, AuthoritativeDir,    "0"),
@@ -342,12 +358,13 @@ static config_var_t option_vars_[] = {
   V(ClientOnly,                  BOOL,     "0"),
   V(ClientPreferIPv6ORPort,      AUTOBOOL, "auto"),
   V(ClientPreferIPv6DirPort,     AUTOBOOL, "auto"),
+  V(ClientAutoIPv6ORPort,        BOOL,     "0"),
   V(ClientRejectInternalAddresses, BOOL,   "1"),
   V(ClientTransportPlugin,       LINELIST, NULL),
   V(ClientUseIPv6,               BOOL,     "0"),
   V(ClientUseIPv4,               BOOL,     "1"),
   V(ConsensusParams,             STRING,   NULL),
-  V(ConnLimit,                   UINT,     "1000"),
+  V(ConnLimit,                   POSINT,     "1000"),
   V(ConnDirectionStatistics,     BOOL,     "0"),
   V(ConstrainedSockets,          BOOL,     "0"),
   V(ConstrainedSockSize,         MEMUNIT,  "8192"),
@@ -391,16 +408,20 @@ static config_var_t option_vars_[] = {
   OBSOLETE("DynamicDHGroups"),
   VPORT(DNSPort),
   OBSOLETE("DNSListenAddress"),
+  V(DormantClientTimeout,         INTERVAL, "24 hours"),
+  V(DormantTimeoutDisabledByIdleStreams, BOOL,     "1"),
+  V(DormantOnFirstStartup,       BOOL,      "0"),
+  V(DormantCanceledByStartup,    BOOL,      "0"),
   /* DoS circuit creation options. */
   V(DoSCircuitCreationEnabled,   AUTOBOOL, "auto"),
-  V(DoSCircuitCreationMinConnections,      UINT, "0"),
-  V(DoSCircuitCreationRate,      UINT,     "0"),
-  V(DoSCircuitCreationBurst,     UINT,     "0"),
+  V(DoSCircuitCreationMinConnections,      POSINT, "0"),
+  V(DoSCircuitCreationRate,      POSINT,     "0"),
+  V(DoSCircuitCreationBurst,     POSINT,     "0"),
   V(DoSCircuitCreationDefenseType,         INT,  "0"),
   V(DoSCircuitCreationDefenseTimePeriod,   INTERVAL, "0"),
   /* DoS connection options. */
   V(DoSConnectionEnabled,        AUTOBOOL, "auto"),
-  V(DoSConnectionMaxConcurrentCount,       UINT, "0"),
+  V(DoSConnectionMaxConcurrentCount,       POSINT, "0"),
   V(DoSConnectionDefenseType,    INT,      "0"),
   /* DoS single hop client options. */
   V(DoSRefuseSingleHopClientRendezvous,    AUTOBOOL, "auto"),
@@ -409,13 +430,17 @@ static config_var_t option_vars_[] = {
   V(TestingEnableCellStatsEvent, BOOL,     "0"),
   OBSOLETE("TestingEnableTbEmptyEvent"),
   V(EnforceDistinctSubnets,      BOOL,     "1"),
-  V(EntryNodes,                  ROUTERSET,   NULL),
+  V_D(EntryNodes,                ROUTERSET,   NULL),
   V(EntryStatistics,             BOOL,     "0"),
   V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"),
-  V(ExcludeNodes,                ROUTERSET, NULL),
-  V(ExcludeExitNodes,            ROUTERSET, NULL),
+  V_D(ExcludeNodes,              ROUTERSET, NULL),
+  V_D(ExcludeExitNodes,          ROUTERSET, NULL),
   OBSOLETE("ExcludeSingleHopRelays"),
-  V(ExitNodes,                   ROUTERSET, NULL),
+  V_D(ExitNodes,                 ROUTERSET, NULL),
+  /* Researchers need a way to tell their clients to use specific
+   * middles that they also control, to allow safe live-network
+   * experimentation with new padding machines. */
+  V_D(MiddleNodes,               ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
   V(ExitPolicyRejectLocalInterfaces, BOOL, "0"),
@@ -471,6 +496,11 @@ static config_var_t option_vars_[] = {
   VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceExportCircuitID", LINELIST_S,  RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSDefense", LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSRatePerSec",
+      LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSBurstPerSec",
+      LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"),
   V(HidServAuth,                 LINELIST, NULL),
   V(ClientOnionAuthDir,          FILENAME, NULL),
@@ -494,8 +524,8 @@ static config_var_t option_vars_[] = {
   V(Socks5ProxyPassword,         STRING,   NULL),
   VAR("KeyDirectory",            FILENAME, KeyDirectory_option, NULL),
   V(KeyDirectoryGroupReadable,   BOOL,     "0"),
-  VAR("HSLayer2Nodes",           ROUTERSET,  HSLayer2Nodes,  NULL),
-  VAR("HSLayer3Nodes",           ROUTERSET,  HSLayer3Nodes,  NULL),
+  VAR_D("HSLayer2Nodes",         ROUTERSET,  HSLayer2Nodes,  NULL),
+  VAR_D("HSLayer3Nodes",         ROUTERSET,  HSLayer3Nodes,  NULL),
   V(KeepalivePeriod,             INTERVAL, "5 minutes"),
   V(KeepBindCapabilities,            AUTOBOOL, "auto"),
   VAR("Log",                     LINELIST, Logs,             NULL),
@@ -509,7 +539,7 @@ static config_var_t option_vars_[] = {
   VAR("MapAddress",              LINELIST, AddressMap,           NULL),
   V(MaxAdvertisedBandwidth,      MEMUNIT,  "1 GB"),
   V(MaxCircuitDirtiness,         INTERVAL, "10 minutes"),
-  V(MaxClientCircuitsPending,    UINT,     "32"),
+  V(MaxClientCircuitsPending,    POSINT,     "32"),
   V(MaxConsensusAgeForDiffs,     INTERVAL, "0 seconds"),
   VAR("MaxMemInQueues",          MEMUNIT,   MaxMemInQueues_raw, "0"),
   OBSOLETE("MaxOnionsPending"),
@@ -526,10 +556,10 @@ static config_var_t option_vars_[] = {
   OBSOLETE("WarnUnsafeSocks"),
   VAR("NodeFamily",              LINELIST, NodeFamilies,         NULL),
   V(NoExec,                      BOOL,     "0"),
-  V(NumCPUs,                     UINT,     "0"),
-  V(NumDirectoryGuards,          UINT,     "0"),
-  V(NumEntryGuards,              UINT,     "0"),
-  V(NumPrimaryGuards,            UINT,     "0"),
+  V(NumCPUs,                     POSINT,     "0"),
+  V(NumDirectoryGuards,          POSINT,     "0"),
+  V(NumEntryGuards,              POSINT,     "0"),
+  V(NumPrimaryGuards,            POSINT,     "0"),
   V(OfflineMasterKey,            BOOL,     "0"),
   OBSOLETE("ORListenAddress"),
   VPORT(ORPort),
@@ -580,10 +610,12 @@ static config_var_t option_vars_[] = {
   V(RecommendedVersions,         LINELIST, NULL),
   V(RecommendedClientVersions,   LINELIST, NULL),
   V(RecommendedServerVersions,   LINELIST, NULL),
-  V(RecommendedPackages,         LINELIST, NULL),
+  OBSOLETE("RecommendedPackages"),
   V(ReducedConnectionPadding,    BOOL,     "0"),
   V(ConnectionPadding,           AUTOBOOL, "auto"),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
+  V(CircuitPadding,              BOOL,     "1"),
+  V(ReducedCircuitPadding,       BOOL,     "0"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
   V(RelayBandwidthRate,          MEMUNIT,  "0"),
@@ -639,6 +671,7 @@ static config_var_t option_vars_[] = {
   OBSOLETE("UseNTorHandshake"),
   V(User,                        STRING,   NULL),
   OBSOLETE("UserspaceIOCPBuffers"),
+  V(AuthDirRejectRequestsUnderLoad, BOOL,  "1"),
   V(AuthDirSharedRandomness,     BOOL,     "1"),
   V(AuthDirTestEd25519LinkKeys,  BOOL,     "1"),
   OBSOLETE("V1AuthoritativeDirectory"),
@@ -651,7 +684,7 @@ static config_var_t option_vars_[] = {
   V(V3AuthVotingInterval,        INTERVAL, "1 hour"),
   V(V3AuthVoteDelay,             INTERVAL, "5 minutes"),
   V(V3AuthDistDelay,             INTERVAL, "5 minutes"),
-  V(V3AuthNIntervalsValid,       UINT,     "3"),
+  V(V3AuthNIntervalsValid,       POSINT,     "3"),
   V(V3AuthUseLegacyKey,          BOOL,     "0"),
   V(V3BandwidthsFile,            FILENAME, NULL),
   V(GuardfractionFile,           FILENAME, NULL),
@@ -662,15 +695,17 @@ static config_var_t option_vars_[] = {
   V(WarnPlaintextPorts,          CSV,      "23,109,110,143"),
   OBSOLETE("UseFilteringSSLBufferevents"),
   OBSOLETE("__UseFilteringSSLBufferevents"),
-  VAR("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
-  VAR("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
-  VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
-  VAR("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
-  VAR("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
-  VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword,
+  VAR_NODUMP("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
+  VAR_NODUMP("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
+  VAR_NODUMP("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
+  VAR_NODUMP("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
+  VAR_NODUMP("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
+  VAR_NODUMP("__HashedControlSessionPassword", LINELIST,
+             HashedControlSessionPassword,
       NULL),
-  VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
-  VAR("__OwningControllerFD", UINT64, OwningControllerFD, UINT64_MAX_STRING),
+  VAR_NODUMP("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
+  VAR_NODUMP("__OwningControllerFD", UINT64, OwningControllerFD,
+             UINT64_MAX_STRING),
   V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"),
   V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
@@ -700,7 +735,7 @@ static config_var_t option_vars_[] = {
    * blocked), but we also don't want to fail if only some mirrors are
    * blackholed. Clients will try 3 directories simultaneously.
    * (Relays never use simultaneous connections.) */
-  V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"),
+  V(ClientBootstrapConsensusMaxInProgressTries, POSINT, "3"),
   /* When a client has any running bridges, check each bridge occasionally,
     * whether or not that bridge is actually up. */
   V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"),
@@ -717,56 +752,40 @@ static config_var_t option_vars_[] = {
   OBSOLETE("TestingDescriptorMaxDownloadTries"),
   OBSOLETE("TestingMicrodescMaxDownloadTries"),
   OBSOLETE("TestingCertMaxDownloadTries"),
-  V(TestingDirAuthVoteExit, ROUTERSET, NULL),
+  V_D(TestingDirAuthVoteExit, ROUTERSET, NULL),
   V(TestingDirAuthVoteExitIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteGuard, ROUTERSET, NULL),
+  V_D(TestingDirAuthVoteGuard, ROUTERSET, NULL),
   V(TestingDirAuthVoteGuardIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteHSDir, ROUTERSET, NULL),
+  V_D(TestingDirAuthVoteHSDir, ROUTERSET, NULL),
   V(TestingDirAuthVoteHSDirIsStrict,  BOOL,     "0"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"),
+  VAR_INVIS("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_,
+            "0"),
 
   END_OF_CONFIG_VARS
 };
 
+/** List of default directory authorities */
+static const char *default_authorities[] = {
+#include "auth_dirs.inc"
+  NULL
+};
+
+/** List of fallback directory authorities. The list is generated by opt-in of
+ * relays that meet certain stability criteria.
+ */
+static const char *default_fallbacks[] = {
+#include "fallback_dirs.inc"
+  NULL
+};
+
 /** Override default values with these if the user sets the TestingTorNetwork
  * option. */
-static const config_var_t testing_tor_network_defaults[] = {
-  V(DirAllowPrivateAddresses,    BOOL,     "1"),
-  V(EnforceDistinctSubnets,      BOOL,     "0"),
-  V(AssumeReachable,             BOOL,     "1"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "0"),
-  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
-    "0"),
-  V(ClientDNSRejectInternalAddresses, BOOL,"0"),
-  V(ClientRejectInternalAddresses, BOOL,   "0"),
-  V(CountPrivateBandwidth,       BOOL,     "1"),
-  V(ExitPolicyRejectPrivate,     BOOL,     "0"),
-  V(ExtendAllowPrivateAddresses, BOOL,     "1"),
-  V(V3AuthVotingInterval,        INTERVAL, "5 minutes"),
-  V(V3AuthVoteDelay,             INTERVAL, "20 seconds"),
-  V(V3AuthDistDelay,             INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"),
-  V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"),
-  V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"),
-  V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"),
-  V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"),
-  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"),
-  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"),
-  V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"),
-  V(TestingEnableConnBwEvent,    BOOL,     "1"),
-  V(TestingEnableCellStatsEvent, BOOL,     "1"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"),
-  V(RendPostPeriod,              INTERVAL, "2 minutes"),
-
-  END_OF_CONFIG_VARS
+static const struct {
+  const char *k;
+  const char *v;
+} testing_tor_network_defaults[] = {
+#include "testnet.inc"
+  { NULL, NULL }
 };
 
 #undef VAR
@@ -830,24 +849,28 @@ static void config_maybe_load_geoip_files_(const or_options_t *options,
 static int options_validate_cb(void *old_options, void *options,
                                void *default_options,
                                int from_setconf, char **msg);
-static void options_free_cb(void *options);
 static void cleanup_protocol_warning_severity_level(void);
 static void set_protocol_warning_severity_level(int warning_severity);
+static void options_clear_cb(const config_mgr_t *mgr, void *opts);
 
 /** Magic value for or_options_t. */
 #define OR_OPTIONS_MAGIC 9090909
 
 /** Configuration format for or_options_t. */
-STATIC config_format_t options_format = {
+static const config_format_t options_format = {
   sizeof(or_options_t),
-  OR_OPTIONS_MAGIC,
-  offsetof(or_options_t, magic_),
+  {
+   "or_options_t",
+   OR_OPTIONS_MAGIC,
+   offsetof(or_options_t, magic_),
+  },
   option_abbrevs_,
   option_deprecation_notes_,
   option_vars_,
   options_validate_cb,
-  options_free_cb,
-  NULL
+  options_clear_cb,
+  NULL,
+  offsetof(or_options_t, subconfigs_),
 };
 
 /*
@@ -879,6 +902,20 @@ static int in_option_validation = 0;
 /* True iff we've initialized libevent */
 static int libevent_initialized = 0;
 
+/* A global configuration manager to handle all configuration objects. */
+static config_mgr_t *options_mgr = NULL;
+
+/** Return the global configuration manager object for torrc options. */
+STATIC const config_mgr_t *
+get_options_mgr(void)
+{
+  if (PREDICT_UNLIKELY(options_mgr == NULL)) {
+    options_mgr = config_mgr_new(&options_format);
+    config_mgr_freeze(options_mgr);
+  }
+  return options_mgr;
+}
+
 /** Return the contents of our frontpage string, or NULL if not configured. */
 MOCK_IMPL(const char*,
 get_dirportfrontpage, (void))
@@ -902,6 +939,32 @@ get_options,(void))
   return get_options_mutable();
 }
 
+/**
+ * True iff we have noticed that this is a testing tor network, and we
+ * should use the corresponding defaults.
+ **/
+static bool testing_network_configured = false;
+
+/** Return a set of lines for any default options that we want to override
+ * from those set in our config_var_t values. */
+static config_line_t *
+get_options_defaults(void)
+{
+  int i;
+  config_line_t *result = NULL, **next = &result;
+
+  if (testing_network_configured) {
+    for (i = 0; testing_tor_network_defaults[i].k; ++i) {
+      config_line_append(next,
+                         testing_tor_network_defaults[i].k,
+                         testing_tor_network_defaults[i].v);
+      next = &(*next)->next;
+    }
+  }
+
+  return result;
+}
+
 /** Change the current global options to contain <b>new_val</b> instead of
  * their current value; take action based on the new value; free the old value
  * as necessary.  Returns 0 on success, -1 on failure.
@@ -909,9 +972,6 @@ get_options,(void))
 int
 set_options(or_options_t *new_val, char **msg)
 {
-  int i;
-  smartlist_t *elements;
-  config_line_t *line;
   or_options_t *old_options = global_options;
   global_options = new_val;
   /* Note that we pass the *old* options below, for comparison. It
@@ -933,35 +993,16 @@ set_options(or_options_t *new_val, char **msg)
   /* Issues a CONF_CHANGED event to notify controller of the change. If Tor is
    * just starting up then the old_options will be undefined. */
   if (old_options && old_options != global_options) {
-    elements = smartlist_new();
-    for (i=0; options_format.vars[i].name; ++i) {
-      const config_var_t *var = &options_format.vars[i];
-      const char *var_name = var->name;
-      if (var->type == CONFIG_TYPE_LINELIST_S ||
-          var->type == CONFIG_TYPE_OBSOLETE) {
-        continue;
-      }
-      if (!config_is_same(&options_format, new_val, old_options, var_name)) {
-        line = config_get_assigned_option(&options_format, new_val,
-                                          var_name, 1);
-
-        if (line) {
-          config_line_t *next;
-          for (; line; line = next) {
-            next = line->next;
-            smartlist_add(elements, line->key);
-            smartlist_add(elements, line->value);
-            tor_free(line);
-          }
-        } else {
-          smartlist_add_strdup(elements, options_format.vars[i].name);
-          smartlist_add(elements, NULL);
-        }
-      }
+    smartlist_t *elements = smartlist_new();
+    config_line_t *changes =
+      config_get_changes(get_options_mgr(), old_options, new_val);
+    for (config_line_t *line = changes; line; line = line->next) {
+      smartlist_add(elements, line->key);
+      smartlist_add(elements, line->value);
     }
     control_event_conf_changed(elements);
-    SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
     smartlist_free(elements);
+    config_free_lines(changes);
   }
 
   if (old_options != global_options) {
@@ -975,49 +1016,13 @@ set_options(or_options_t *new_val, char **msg)
   return 0;
 }
 
-/** The version of this Tor process, as parsed. */
-static char *the_tor_version = NULL;
-/** A shorter version of this Tor process's version, for export in our router
- *  descriptor.  (Does not include the git version, if any.) */
-static char *the_short_tor_version = NULL;
-
-/** Return the current Tor version. */
-const char *
-get_version(void)
-{
-  if (the_tor_version == NULL) {
-    if (strlen(tor_git_revision)) {
-      tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(),
-                   tor_git_revision);
-    } else {
-      the_tor_version = tor_strdup(get_short_version());
-    }
-  }
-  return the_tor_version;
-}
-
-/** Return the current Tor version, without any git tag. */
-const char *
-get_short_version(void)
-{
-
-  if (the_short_tor_version == NULL) {
-#ifdef TOR_BUILD_TAG
-    tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG);
-#else
-    the_short_tor_version = tor_strdup(VERSION);
-#endif
-  }
-  return the_short_tor_version;
-}
-
 /** Release additional memory allocated in options
  */
-STATIC void
-or_options_free_(or_options_t *options)
+static void
+options_clear_cb(const config_mgr_t *mgr, void *opts)
 {
-  if (!options)
-    return;
+  (void)mgr;
+  or_options_t *options = opts;
 
   routerset_free(options->ExcludeExitNodesUnion_);
   if (options->NodeFamilySets) {
@@ -1040,7 +1045,14 @@ or_options_free_(or_options_t *options)
   tor_free(options->command_arg);
   tor_free(options->master_key_fname);
   config_free_lines(options->MyFamily);
-  config_free(&options_format, options);
+}
+
+/** Release all memory allocated in options
+ */
+STATIC void
+or_options_free_(or_options_t *options)
+{
+  config_free(get_options_mgr(), options);
 }
 
 /** Release all memory and resources held by global configuration structures.
@@ -1070,13 +1082,12 @@ config_free_all(void)
   tor_free(torrc_defaults_fname);
   tor_free(global_dirfrontpagecontents);
 
-  tor_free(the_short_tor_version);
-  tor_free(the_tor_version);
-
   cleanup_protocol_warning_severity_level();
 
   have_parsed_cmdline = 0;
   libevent_initialized = 0;
+
+  config_mgr_free(options_mgr);
 }
 
 /** Make <b>address</b> -- a piece of information related to our operation as
@@ -1187,24 +1198,13 @@ init_protocol_warning_severity_level(void)
 static void
 cleanup_protocol_warning_severity_level(void)
 {
-   atomic_counter_destroy(&protocol_warning_severity_level);
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
+  atomic_counter_destroy(&protocol_warning_severity_level);
 }
 
-/** List of default directory authorities */
-
-static const char *default_authorities[] = {
-#include "auth_dirs.inc"
-  NULL
-};
-
-/** List of fallback directory authorities. The list is generated by opt-in of
- * relays that meet certain stability criteria.
- */
-static const char *default_fallbacks[] = {
-#include "fallback_dirs.inc"
-  NULL
-};
-
 /** Add the default directory authorities directly into the trusted dir list,
  * but only add them insofar as they share bits with <b>type</b>.
  * Each authority's bits are restricted to the bits shared with <b>type</b>.
@@ -1443,10 +1443,10 @@ options_act_reversible(const or_options_t *old_options, char **msg)
    * processes. */
   if (running_tor && options->RunAsDaemon) {
     if (! start_daemon_has_been_called())
-      crypto_prefork();
+      subsystems_prefork();
     /* No need to roll back, since you can't change the value. */
     if (start_daemon())
-      crypto_postfork();
+      subsystems_postfork();
   }
 
 #ifdef HAVE_SYSTEMD
@@ -1460,7 +1460,7 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                       "on this OS/with this build.");
     goto rollback;
   }
-#else /* !(!defined(HAVE_SYS_UN_H)) */
+#else /* defined(HAVE_SYS_UN_H) */
   if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
     *msg = tor_strdup("Setting ControlSocketGroupWritable without setting"
                       "a ControlSocket makes no sense.");
@@ -1735,6 +1735,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out)
   int routerset_usage =
     routerset_needs_geoip(options->EntryNodes) ||
     routerset_needs_geoip(options->ExitNodes) ||
+    routerset_needs_geoip(options->MiddleNodes) ||
     routerset_needs_geoip(options->ExcludeExitNodes) ||
     routerset_needs_geoip(options->ExcludeNodes) ||
     routerset_needs_geoip(options->HSLayer2Nodes) ||
@@ -2042,9 +2043,6 @@ options_act(const or_options_t *old_options)
     finish_daemon(options->DataDirectory);
   }
 
-  /* See whether we need to enable/disable our once-a-second timer. */
-  reschedule_per_second_timer();
-
   /* We want to reinit keys as needed before we do much of anything else:
      keys are important, and other things can depend on them. */
   if (transition_affects_workers ||
@@ -2177,6 +2175,7 @@ options_act(const or_options_t *old_options)
                          options->HSLayer2Nodes) ||
         !routerset_equal(old_options->HSLayer3Nodes,
                          options->HSLayer3Nodes) ||
+        !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) ||
         options->StrictNodes != old_options->StrictNodes) {
       log_info(LD_CIRC,
                "Changed to using entry guards or bridges, or changed "
@@ -2560,7 +2559,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
 
     param = tor_malloc_zero(sizeof(config_line_t));
     param->key = is_cmdline ? tor_strdup(argv[i]) :
-                   tor_strdup(config_expand_abbrev(&options_format, s, 1, 1));
+                 tor_strdup(config_expand_abbrev(get_options_mgr(), s, 1, 1));
     param->value = arg;
     param->command = command;
     param->next = NULL;
@@ -2586,8 +2585,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
 int
 option_is_recognized(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return (var != NULL);
+  return config_find_option_name(get_options_mgr(), key) != NULL;
 }
 
 /** Return the canonical name of a configuration option, or NULL
@@ -2595,8 +2593,7 @@ option_is_recognized(const char *key)
 const char *
 option_get_canonical_name(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return var ? var->name : NULL;
+  return config_find_option_name(get_options_mgr(), key);
 }
 
 /** Return a canonical list of the options assigned for key.
@@ -2604,7 +2601,7 @@ option_get_canonical_name(const char *key)
 config_line_t *
 option_get_assignment(const or_options_t *options, const char *key)
 {
-  return config_get_assigned_option(&options_format, options, key, 1);
+  return config_get_assigned_option(get_options_mgr(), options, key, 1);
 }
 
 /** Try assigning <b>list</b> to the global options. You do this by duping
@@ -2620,9 +2617,9 @@ setopt_err_t
 options_trial_assign(config_line_t *list, unsigned flags, char **msg)
 {
   int r;
-  or_options_t *trial_options = config_dup(&options_format, get_options());
+  or_options_t *trial_options = config_dup(get_options_mgr(), get_options());
 
-  if ((r=config_assign(&options_format, trial_options,
+  if ((r=config_assign(get_options_mgr(), trial_options,
                        list, flags, msg)) < 0) {
     or_options_free(trial_options);
     return r;
@@ -2677,24 +2674,30 @@ print_usage(void)
 static void
 list_torrc_options(void)
 {
-  int i;
-  for (i = 0; option_vars_[i].name; ++i) {
-    const config_var_t *var = &option_vars_[i];
-    if (var->type == CONFIG_TYPE_OBSOLETE ||
-        var->type == CONFIG_TYPE_LINELIST_V)
+  smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+  SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+    /* Possibly this should check listable, rather than (or in addition to)
+     * settable. See ticket 31654.
+     */
+    if (! config_var_is_settable(var)) {
+      /* This variable cannot be set, or cannot be set by this name. */
       continue;
-    printf("%s\n", var->name);
-  }
+    }
+    printf("%s\n", var->member.name);
+  } SMARTLIST_FOREACH_END(var);
+  smartlist_free(vars);
 }
 
 /** Print all deprecated but non-obsolete torrc options. */
 static void
 list_deprecated_options(void)
 {
-  const config_deprecation_t *d;
-  for (d = option_deprecation_notes_; d->name; ++d) {
-    printf("%s\n", d->name);
-  }
+  smartlist_t *deps = config_mgr_list_deprecated_vars(get_options_mgr());
+  /* Possibly this should check whether the variables are listable,
+   * but currently it does not.  See ticket 31654. */
+  SMARTLIST_FOREACH(deps, const char *, name,
+                    printf("%s\n", name));
+  smartlist_free(deps);
 }
 
 /** Print all compile-time modules and their enabled/disabled status. */
@@ -3004,7 +3007,7 @@ is_local_addr, (const tor_addr_t *addr))
 or_options_t *
 options_new(void)
 {
-  return config_new(&options_format);
+  return config_new(get_options_mgr());
 }
 
 /** Set <b>options</b> to hold reasonable defaults for most options.
@@ -3012,7 +3015,17 @@ options_new(void)
 void
 options_init(or_options_t *options)
 {
-  config_init(&options_format, options);
+  config_init(get_options_mgr(), options);
+  config_line_t *dflts = get_options_defaults();
+  char *msg=NULL;
+  if (config_assign(get_options_mgr(), options, dflts,
+                    CAL_WARN_DEPRECATIONS, &msg)<0) {
+    log_err(LD_BUG, "Unable to set default options: %s", msg);
+    tor_free(msg);
+    tor_assert_unreached();
+  }
+  config_free_lines(dflts);
+  tor_free(msg);
 }
 
 /** Return a string containing a possible configuration file that would give
@@ -3042,7 +3055,7 @@ options_dump(const or_options_t *options, int how_to_dump)
       return NULL;
   }
 
-  return config_dump(&options_format, use_defaults, options, minimal, 0);
+  return config_dump(get_options_mgr(), use_defaults, options, minimal, 0);
 }
 
 /** Return 0 if every element of sl is a string holding a decimal
@@ -3174,13 +3187,6 @@ options_validate_cb(void *old_options, void *options, void *default_options,
   return rv;
 }
 
-/** Callback to free an or_options_t */
-static void
-options_free_cb(void *options)
-{
-  or_options_free_(options);
-}
-
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
 #if defined(__GNUC__) && __GNUC__ <= 3
@@ -3446,6 +3452,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo)))
     REJECT("ContactInfo config option must be UTF-8.");
 
+  check_network_configuration(server_mode(options));
+
   /* Special case on first boot if no Log options are given. */
   if (!options->Logs && !options->RunAsDaemon && !from_setconf) {
     if (quiet_level == 0)
@@ -3516,7 +3524,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("Cannot use TransProxyType without any valid TransPort.");
     }
   }
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   if (options->TransPort_set)
     REJECT("TransPort is disabled in this build.");
 #endif /* defined(USE_TRANSPARENT) */
@@ -3550,13 +3558,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "features to be broken in unpredictable ways.");
   }
 
-  for (cl = options->RecommendedPackages; cl; cl = cl->next) {
-    if (! validate_recommended_package_line(cl->value)) {
-      log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored",
-               escaped(cl->value));
-    }
-  }
-
   if (options->AuthoritativeDir) {
     if (!options->ContactInfo && !options->TestingTorNetwork)
       REJECT("Authoritative directory servers must set ContactInfo");
@@ -3579,7 +3580,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
     tor_free(t);
     t = format_recommended_version_list(options->RecommendedServerVersions, 1);
     tor_free(t);
-#endif
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
     if (options->UseEntryGuards) {
       log_info(LD_CONFIG, "Authoritative directory servers can't set "
@@ -3595,14 +3596,17 @@ options_validate(or_options_t *old_options, or_options_t *options,
           options->V3AuthoritativeDir))
       REJECT("AuthoritativeDir is set, but none of "
              "(Bridge/V3)AuthoritativeDir is set.");
+#ifdef HAVE_MODULE_DIRAUTH
     /* If we have a v3bandwidthsfile and it's broken, complain on startup */
     if (options->V3BandwidthsFile && !old_options) {
-      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
+      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                       NULL);
     }
     /* same for guardfraction file */
     if (options->GuardfractionFile && !old_options) {
       dirserv_read_guardfraction_file(options->GuardfractionFile, NULL);
     }
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
   }
 
   if (options->AuthoritativeDir && !options->DirPort_set)
@@ -3780,6 +3784,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
     REJECT("Relays cannot set ReducedConnectionPadding. ");
   }
 
+  if (server_mode(options) && options->CircuitPadding == 0) {
+    REJECT("Relays cannot set CircuitPadding to 0. ");
+  }
+
+  if (server_mode(options) && options->ReducedCircuitPadding == 1) {
+    REJECT("Relays cannot set ReducedCircuitPadding. ");
+  }
+
   if (options->BridgeDistribution) {
     if (!options->BridgeRelay) {
       REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
@@ -3902,6 +3914,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "default.");
   }
 
+  if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) {
+    REJECT("DormantClientTimeout is too low. It must be at least 10 minutes.");
+  }
+
   if (options->PathBiasNoticeRate > 1.0) {
     tor_asprintf(msg,
               "PathBiasNoticeRate is too high. "
@@ -3953,7 +3969,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   if (options->HeartbeatPeriod &&
-      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) {
+      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD &&
+      !options->TestingTorNetwork) {
     log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; "
              "raising to %d seconds.", MIN_HEARTBEAT_PERIOD);
     options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD;
@@ -4225,6 +4242,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "You should also make sure you aren't listing this bridge's "
              "fingerprint in any other MyFamily.");
   }
+  if (options->MyFamily_lines && !options->ContactInfo) {
+    log_warn(LD_CONFIG, "MyFamily is set but ContactInfo is not configured. "
+             "ContactInfo should always be set when MyFamily option is too.");
+  }
   if (normalize_nickname_list(&options->MyFamily,
                               options->MyFamily_lines, "MyFamily", msg))
     return -1;
@@ -4427,7 +4448,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
   STMT_BEGIN                                                            \
     if (!options->TestingTorNetwork &&                                  \
         !options->UsingTestNetworkDefaults_ &&                          \
-        !config_is_same(&options_format,options,                        \
+        !config_is_same(get_options_mgr(),options,                        \
                         default_options,#arg)) {                        \
       REJECT(#arg " may only be changed in testing Tor "                \
              "networks!");                                              \
@@ -4613,7 +4634,7 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 #else
 /* On a 32-bit platform, we can't have 8GB of ram. */
 #define RAM_IS_VERY_LARGE(x) (0)
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
       if (RAM_IS_VERY_LARGE(ram)) {
         /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues
@@ -5090,7 +5111,7 @@ find_torrc_filename(config_line_t *cmd_arg,
       } else {
         fname = dflt ? tor_strdup(dflt) : NULL;
       }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
       fname = dflt ? tor_strdup(dflt) : NULL;
 #endif /* !defined(_WIN32) */
     }
@@ -5392,6 +5413,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
                          int command, const char *command_arg,
                          char **msg)
 {
+  bool retry = false;
   or_options_t *oldoptions, *newoptions, *newdefaultoptions=NULL;
   config_line_t *cl;
   int retval;
@@ -5402,8 +5424,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   oldoptions = global_options; /* get_options unfortunately asserts if
                                   this is the first time we run*/
 
-  newoptions = tor_malloc_zero(sizeof(or_options_t));
-  newoptions->magic_ = OR_OPTIONS_MAGIC;
+  newoptions = options_new();
   options_init(newoptions);
   newoptions->command = command;
   newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
@@ -5422,7 +5443,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       err = SETOPT_ERR_PARSE;
       goto err;
     }
-    retval = config_assign(&options_format, newoptions, cl,
+    retval = config_assign(get_options_mgr(), newoptions, cl,
                            CAL_WARN_DEPRECATIONS, msg);
     config_free_lines(cl);
     if (retval < 0) {
@@ -5430,15 +5451,15 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       goto err;
     }
     if (i==0)
-      newdefaultoptions = config_dup(&options_format, newoptions);
+      newdefaultoptions = config_dup(get_options_mgr(), newoptions);
   }
 
   if (newdefaultoptions == NULL) {
-    newdefaultoptions = config_dup(&options_format, global_default_options);
+    newdefaultoptions = config_dup(get_options_mgr(), global_default_options);
   }
 
   /* Go through command-line variables too */
-  retval = config_assign(&options_format, newoptions,
+  retval = config_assign(get_options_mgr(), newoptions,
                          global_cmdline_options, CAL_WARN_DEPRECATIONS, msg);
   if (retval < 0) {
     err = SETOPT_ERR_PARSE;
@@ -5449,73 +5470,12 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   newoptions->FilesOpenedByIncludes = opened_files;
 
   /* If this is a testing network configuration, change defaults
-   * for a list of dependent config options, re-initialize newoptions
-   * with the new defaults, and assign all options to it second time. */
-  if (newoptions->TestingTorNetwork) {
-    /* XXXX this is a bit of a kludge.  perhaps there's a better way to do
-     * this?  We could, for example, make the parsing algorithm do two passes
-     * over the configuration.  If it finds any "suite" options like
-     * TestingTorNetwork, it could change the defaults before its second pass.
-     * Not urgent so long as this seems to work, but at any sign of trouble,
-     * let's clean it up.  -NM */
-
-    /* Change defaults. */
-    for (int i = 0; testing_tor_network_defaults[i].name; ++i) {
-      const config_var_t *new_var = &testing_tor_network_defaults[i];
-      config_var_t *old_var =
-          config_find_option_mutable(&options_format, new_var->name);
-      tor_assert(new_var);
-      tor_assert(old_var);
-      old_var->initvalue = new_var->initvalue;
-
-      if ((config_find_deprecation(&options_format, new_var->name))) {
-        log_warn(LD_GENERAL, "Testing options override the deprecated "
-                 "option %s. Is that intentional?",
-                 new_var->name);
-      }
-    }
-
-    /* Clear newoptions and re-initialize them with new defaults. */
-    or_options_free(newoptions);
-    or_options_free(newdefaultoptions);
-    newdefaultoptions = NULL;
-    newoptions = tor_malloc_zero(sizeof(or_options_t));
-    newoptions->magic_ = OR_OPTIONS_MAGIC;
-    options_init(newoptions);
-    newoptions->command = command;
-    newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
-
-    /* Assign all options a second time. */
-    opened_files = smartlist_new();
-    for (int i = 0; i < 2; ++i) {
-      const char *body = i==0 ? cf_defaults : cf;
-      if (!body)
-        continue;
-
-      /* get config lines, assign them */
-      retval = config_get_lines_include(body, &cl, 1,
-                                        body == cf ? &cf_has_include : NULL,
-                                        opened_files);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      retval = config_assign(&options_format, newoptions, cl, 0, msg);
-      config_free_lines(cl);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      if (i==0)
-        newdefaultoptions = config_dup(&options_format, newoptions);
-    }
-    /* Assign command-line variables a second time too */
-    retval = config_assign(&options_format, newoptions,
-                           global_cmdline_options, 0, msg);
-    if (retval < 0) {
-      err = SETOPT_ERR_PARSE;
-      goto err;
-    }
+   * for a list of dependent config options, and try this function again. */
+  if (newoptions->TestingTorNetwork && ! testing_network_configured) {
+    // retry with the testing defaults.
+    testing_network_configured = true;
+    retry = true;
+    goto err;
   }
 
   newoptions->IncludeUsed = cf_has_include;
@@ -5560,6 +5520,9 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg);
     tor_free(old_msg);
   }
+  if (retry)
+    return options_init_from_string(cf_defaults, cf, command, command_arg,
+                                    msg);
   return err;
 }
 
@@ -5785,7 +5748,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
 #else
         log_warn(LD_CONFIG, "Android logging is not supported"
                             " on this system. Sorry.");
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
         goto cleanup;
       }
     }
@@ -7838,7 +7801,7 @@ get_data_directory(const char *val)
   } else {
     return tor_strdup(get_windows_conf_root());
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *d = val;
   if (!d)
     d = "~/.tor";
@@ -8182,72 +8145,48 @@ getinfo_helper_config(control_connection_t *conn,
   (void) errmsg;
   if (!strcmp(question, "config/names")) {
     smartlist_t *sl = smartlist_new();
-    int i;
-    for (i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
-      const char *type;
-      /* don't tell controller about triple-underscore options */
-      if (!strncmp(option_vars_[i].name, "___", 3))
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+      /* don't tell controller about invisible options */
+      if (! config_var_is_listable(var))
         continue;
-      switch (var->type) {
-        case CONFIG_TYPE_STRING: type = "String"; break;
-        case CONFIG_TYPE_FILENAME: type = "Filename"; break;
-        case CONFIG_TYPE_UINT: type = "Integer"; break;
-        case CONFIG_TYPE_UINT64: type = "Integer"; break;
-        case CONFIG_TYPE_INT: type = "SignedInteger"; break;
-        case CONFIG_TYPE_PORT: type = "Port"; break;
-        case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break;
-        case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break;
-        case CONFIG_TYPE_DOUBLE: type = "Float"; break;
-        case CONFIG_TYPE_BOOL: type = "Boolean"; break;
-        case CONFIG_TYPE_AUTOBOOL: type = "Boolean+Auto"; break;
-        case CONFIG_TYPE_ISOTIME: type = "Time"; break;
-        case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break;
-        case CONFIG_TYPE_CSV: type = "CommaList"; break;
-        /* This type accepts more inputs than TimeInterval, but it ignores
-         * everything after the first entry, so we may as well pretend
-         * it's a TimeInterval. */
-        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_LINELIST: type = "LineList"; break;
-        case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break;
-        case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break;
-        default:
-        case CONFIG_TYPE_OBSOLETE:
-          type = NULL; break;
-      }
+      const char *type = struct_var_get_typename(&var->member);
       if (!type)
         continue;
-      smartlist_add_asprintf(sl, "%s %s\n",var->name,type);
-    }
+      smartlist_add_asprintf(sl, "%s %s\n",var->member.name,type);
+    } SMARTLIST_FOREACH_END(var);
     *answer = smartlist_join_strings(sl, "", 0, NULL);
     SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
     smartlist_free(sl);
+    smartlist_free(vars);
   } else if (!strcmp(question, "config/defaults")) {
     smartlist_t *sl = smartlist_new();
     int dirauth_lines_seen = 0, fallback_lines_seen = 0;
-    for (int i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
+    /* Possibly this should check whether the variables are listable,
+     * but currently it does not.  See ticket 31654. */
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
       if (var->initvalue != NULL) {
-        if (strcmp(option_vars_[i].name, "DirAuthority") == 0) {
+        if (strcmp(var->member.name, "DirAuthority") == 0) {
           /*
            * Count dirauth lines we have a default for; we'll use the
            * count later to decide whether to add the defaults manually
            */
           ++dirauth_lines_seen;
         }
-        if (strcmp(option_vars_[i].name, "FallbackDir") == 0) {
+        if (strcmp(var->member.name, "FallbackDir") == 0) {
           /*
-           * Similarly count fallback lines, so that we can decided later
+           * Similarly count fallback lines, so that we can decide later
            * to add the defaults manually.
            */
           ++fallback_lines_seen;
         }
         char *val = esc_for_log(var->initvalue);
-        smartlist_add_asprintf(sl, "%s %s\n",var->name,val);
+        smartlist_add_asprintf(sl, "%s %s\n",var->member.name,val);
         tor_free(val);
       }
-    }
+    } SMARTLIST_FOREACH_END(var);
+    smartlist_free(vars);
 
     if (dirauth_lines_seen == 0) {
       /*
@@ -8408,7 +8347,7 @@ config_load_geoip_file_(sa_family_t family,
   }
   r = geoip_load_file(family, fname, severity);
   tor_free(free_fname);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)default_fname;
   r = geoip_load_file(family, fname, severity);
 #endif /* defined(_WIN32) */
@@ -8496,7 +8435,7 @@ init_cookie_authentication(const char *fname, const char *header,
       log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname));
     }
   }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void) group_readable;
 #endif /* !defined(_WIN32) */
 
diff --git a/src/app/config/config.h b/src/app/config/config.h
index 301faf7067..44f09e5ee9 100644
--- a/src/app/config/config.h
+++ b/src/app/config/config.h
@@ -41,8 +41,6 @@ const char *escaped_safe_str_client(const char *address);
 const char *escaped_safe_str(const char *address);
 void init_protocol_warning_severity_level(void);
 int get_protocol_warning_severity_level(void);
-const char *get_version(void);
-const char *get_short_version(void);
 
 /** An error from options_trial_assign() or options_init_from_string(). */
 typedef enum setopt_err_t {
@@ -249,9 +247,8 @@ int options_any_client_port_set(const or_options_t *options);
 #define CL_PORT_DFLT_GROUP_WRITABLE (1u<<7)
 
 STATIC int options_act(const or_options_t *old_options);
-#ifdef TOR_UNIT_TESTS
-extern struct config_format_t options_format;
-#endif
+struct config_mgr_t;
+STATIC const struct config_mgr_t *get_options_mgr(void);
 
 STATIC port_cfg_t *port_cfg_new(size_t namelen);
 #define port_cfg_free(port) \
diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c
deleted file mode 100644
index efa0c19fa6..0000000000
--- a/src/app/config/confparse.c
+++ /dev/null
@@ -1,1207 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.c
- *
- * \brief Back-end for parsing and generating key-value files, used to
- *   implement the torrc file format and the state file.
- *
- * This module is used by config.c to parse and encode torrc
- * configuration files, and by statefile.c to parse and encode the
- * $DATADIR/state file.
- *
- * To use this module, its callers provide an instance of
- * config_format_t to describe the mappings from a set of configuration
- * options to a number of fields in a C structure.  With this mapping,
- * the functions here can convert back and forth between the C structure
- * specified, and a linked list of key-value pairs.
- */
-
-#include "core/or/or.h"
-#include "app/config/confparse.h"
-#include "feature/nodelist/routerset.h"
-
-#include "lib/container/bitarray.h"
-#include "lib/encoding/confline.h"
-
-static uint64_t config_parse_memunit(const char *s, int *ok);
-static int config_parse_msec_interval(const char *s, int *ok);
-static int config_parse_interval(const char *s, int *ok);
-static void config_reset(const config_format_t *fmt, void *options,
-                         const config_var_t *var, int use_defaults);
-
-/** Allocate an empty configuration object of a given format type. */
-void *
-config_new(const config_format_t *fmt)
-{
-  void *opts = tor_malloc_zero(fmt->size);
-  *(uint32_t*)STRUCT_VAR_P(opts, fmt->magic_offset) = fmt->magic;
-  CONFIG_CHECK(fmt, opts);
-  return opts;
-}
-
-/*
- * Functions to parse config options
- */
-
-/** If <b>option</b> is an official abbreviation for a longer option,
- * return the longer option.  Otherwise return <b>option</b>.
- * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
- * apply abbreviations that work for the config file and the command line.
- * If <b>warn_obsolete</b> is set, warn about deprecated names. */
-const char *
-config_expand_abbrev(const config_format_t *fmt, const char *option,
-                     int command_line, int warn_obsolete)
-{
-  int i;
-  if (! fmt->abbrevs)
-    return option;
-  for (i=0; fmt->abbrevs[i].abbreviated; ++i) {
-    /* Abbreviations are case insensitive. */
-    if (!strcasecmp(option,fmt->abbrevs[i].abbreviated) &&
-        (command_line || !fmt->abbrevs[i].commandline_only)) {
-      if (warn_obsolete && fmt->abbrevs[i].warn) {
-        log_warn(LD_CONFIG,
-                 "The configuration option '%s' is deprecated; "
-                 "use '%s' instead.",
-                 fmt->abbrevs[i].abbreviated,
-                 fmt->abbrevs[i].full);
-      }
-      /* Keep going through the list in case we want to rewrite it more.
-       * (We could imagine recursing here, but I don't want to get the
-       * user into an infinite loop if we craft our list wrong.) */
-      option = fmt->abbrevs[i].full;
-    }
-  }
-  return option;
-}
-
-/** If <b>key</b> is a deprecated configuration option, return the message
- * explaining why it is deprecated (which may be an empty string). Return NULL
- * if it is not deprecated. The <b>key</b> field must be fully expanded. */
-const char *
-config_find_deprecation(const config_format_t *fmt, const char *key)
-{
-  if (BUG(fmt == NULL) || BUG(key == NULL))
-    return NULL;
-  if (fmt->deprecations == NULL)
-    return NULL;
-
-  const config_deprecation_t *d;
-  for (d = fmt->deprecations; d->name; ++d) {
-    if (!strcasecmp(d->name, key)) {
-      return d->why_deprecated ? d->why_deprecated : "";
-    }
-  }
-  return NULL;
-}
-
-/** As config_find_option, but return a non-const pointer. */
-config_var_t *
-config_find_option_mutable(config_format_t *fmt, const char *key)
-{
-  int i;
-  size_t keylen = strlen(key);
-  if (!keylen)
-    return NULL; /* if they say "--" on the command line, it's not an option */
-  /* First, check for an exact (case-insensitive) match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strcasecmp(key, fmt->vars[i].name)) {
-      return &fmt->vars[i];
-    }
-  }
-  /* If none, check for an abbreviated match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strncasecmp(key, fmt->vars[i].name, keylen)) {
-      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
-               "Please use '%s' instead",
-               key, fmt->vars[i].name);
-      return &fmt->vars[i];
-    }
-  }
-  /* Okay, unrecognized option */
-  return NULL;
-}
-
-/** If <b>key</b> is a configuration option, return the corresponding const
- * config_var_t.  Otherwise, if <b>key</b> is a non-standard abbreviation,
- * warn, and return the corresponding const config_var_t.  Otherwise return
- * NULL.
- */
-const config_var_t *
-config_find_option(const config_format_t *fmt, const char *key)
-{
-  return config_find_option_mutable((config_format_t*)fmt, key);
-}
-
-/** Return the number of option entries in <b>fmt</b>. */
-static int
-config_count_options(const config_format_t *fmt)
-{
-  int i;
-  for (i=0; fmt->vars[i].name; ++i)
-    ;
-  return i;
-}
-
-/*
- * Functions to assign config options.
- */
-
-/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
- * with <b>c</b>-\>value and return 0, or return -1 if bad value.
- *
- * Called from config_assign_line() and option_reset().
- */
-static int
-config_assign_value(const config_format_t *fmt, void *options,
-                    config_line_t *c, char **msg)
-{
-  int i, ok;
-  const config_var_t *var;
-  void *lvalue;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  tor_assert(var);
-
-  lvalue = STRUCT_VAR_P(options, var->var_offset);
-
-  switch (var->type) {
-
-  case CONFIG_TYPE_PORT:
-    if (!strcasecmp(c->value, "auto")) {
-      *(int *)lvalue = CFG_AUTO_PORT;
-      break;
-    }
-    FALLTHROUGH;
-  case CONFIG_TYPE_INT:
-  case CONFIG_TYPE_UINT:
-    i = (int)tor_parse_long(c->value, 10,
-                            var->type==CONFIG_TYPE_INT ? INT_MIN : 0,
-                            var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX,
-                            &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Int keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_UINT64: {
-    uint64_t u64 = tor_parse_uint64(c->value, 10,
-                                    0, UINT64_MAX, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "uint64 keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_CSV_INTERVAL: {
-    /* We used to have entire smartlists here.  But now that all of our
-     * download schedules use exponential backoff, only the first part
-     * matters. */
-    const char *comma = strchr(c->value, ',');
-    const char *val = c->value;
-    char *tmp = NULL;
-    if (comma) {
-      tmp = tor_strndup(c->value, comma - c->value);
-      val = tmp;
-    }
-
-    i = config_parse_interval(val, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      tor_free(tmp);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    tor_free(tmp);
-    break;
-  }
-
-  case CONFIG_TYPE_INTERVAL: {
-    i = config_parse_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MSEC_INTERVAL: {
-    i = config_parse_msec_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Msec interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MEMUNIT: {
-    uint64_t u64 = config_parse_memunit(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Value '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_BOOL:
-    i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Boolean '%s %s' expects 0 or 1.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_AUTOBOOL:
-    if (!strcasecmp(c->value, "auto"))
-      *(int *)lvalue = -1;
-    else if (!strcmp(c->value, "0"))
-      *(int *)lvalue = 0;
-    else if (!strcmp(c->value, "1"))
-      *(int *)lvalue = 1;
-    else {
-      tor_asprintf(msg, "Boolean '%s %s' expects 0, 1, or 'auto'.",
-                   c->key, c->value);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_STRING:
-  case CONFIG_TYPE_FILENAME:
-    tor_free(*(char **)lvalue);
-    *(char **)lvalue = tor_strdup(c->value);
-    break;
-
-  case CONFIG_TYPE_DOUBLE:
-    *(double *)lvalue = atof(c->value);
-    break;
-
-  case CONFIG_TYPE_ISOTIME:
-    if (parse_iso_time(c->value, (time_t *)lvalue)) {
-      tor_asprintf(msg,
-          "Invalid time '%s' for keyword '%s'", c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_ROUTERSET:
-    if (*(routerset_t**)lvalue) {
-      routerset_free(*(routerset_t**)lvalue);
-    }
-    *(routerset_t**)lvalue = routerset_new();
-    if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) {
-      tor_asprintf(msg, "Invalid exit list '%s' for option '%s'",
-                   c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_CSV:
-    if (*(smartlist_t**)lvalue) {
-      SMARTLIST_FOREACH(*(smartlist_t**)lvalue, char *, cp, tor_free(cp));
-      smartlist_clear(*(smartlist_t**)lvalue);
-    } else {
-      *(smartlist_t**)lvalue = smartlist_new();
-    }
-
-    smartlist_split_string(*(smartlist_t**)lvalue, c->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    break;
-
-  case CONFIG_TYPE_LINELIST:
-  case CONFIG_TYPE_LINELIST_S:
-    {
-      config_line_t *lastval = *(config_line_t**)lvalue;
-      if (lastval && lastval->fragile) {
-        if (c->command != CONFIG_LINE_APPEND) {
-          config_free_lines(lastval);
-          *(config_line_t**)lvalue = NULL;
-        } else {
-          lastval->fragile = 0;
-        }
-      }
-
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-    }
-    break;
-  case CONFIG_TYPE_OBSOLETE:
-    log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key);
-    break;
-  case CONFIG_TYPE_LINELIST_V:
-    tor_asprintf(msg,
-        "You may not provide a value for virtual option '%s'", c->key);
-    return -1;
-  default:
-    tor_assert(0);
-    break;
-  }
-  return 0;
-}
-
-/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
- * to it will replace old ones. */
-static void
-config_mark_lists_fragile(const config_format_t *fmt, void *options)
-{
-  int i;
-  tor_assert(fmt);
-  tor_assert(options);
-
-  for (i = 0; fmt->vars[i].name; ++i) {
-    const config_var_t *var = &fmt->vars[i];
-    config_line_t *list;
-    if (var->type != CONFIG_TYPE_LINELIST &&
-        var->type != CONFIG_TYPE_LINELIST_V)
-      continue;
-
-    list = *(config_line_t **)STRUCT_VAR_P(options, var->var_offset);
-    if (list)
-      list->fragile = 1;
-  }
-}
-
-void
-warn_deprecated_option(const char *what, const char *why)
-{
-  const char *space = (why && strlen(why)) ? " " : "";
-  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
-           "be removed in a future version of Tor.%s%s (If you think this is "
-           "a mistake, please let us know!)",
-           what, space, why);
-}
-
-/** If <b>c</b> is a syntactically valid configuration line, update
- * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
- * key, -2 for bad value.
- *
- * If <b>clear_first</b> is set, clear the value first. Then if
- * <b>use_defaults</b> is set, set the value to the default.
- *
- * Called from config_assign().
- */
-static int
-config_assign_line(const config_format_t *fmt, void *options,
-                   config_line_t *c, unsigned flags,
-                   bitarray_t *options_seen, char **msg)
-{
-  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
-  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
-  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  if (!var) {
-    if (fmt->extra) {
-      void *lvalue = STRUCT_VAR_P(options, fmt->extra->var_offset);
-      log_info(LD_CONFIG,
-               "Found unrecognized option '%s'; saving it.", c->key);
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-      return 0;
-    } else {
-      tor_asprintf(msg,
-                "Unknown option '%s'.  Failing.", c->key);
-      return -1;
-    }
-  }
-
-  /* Put keyword into canonical case. */
-  if (strcmp(var->name, c->key)) {
-    tor_free(c->key);
-    c->key = tor_strdup(var->name);
-  }
-
-  const char *deprecation_msg;
-  if (warn_deprecations &&
-      (deprecation_msg = config_find_deprecation(fmt, var->name))) {
-    warn_deprecated_option(var->name, deprecation_msg);
-  }
-
-  if (!strlen(c->value)) {
-    /* reset or clear it, then return */
-    if (!clear_first) {
-      if ((var->type == CONFIG_TYPE_LINELIST ||
-           var->type == CONFIG_TYPE_LINELIST_S) &&
-          c->command != CONFIG_LINE_CLEAR) {
-        /* We got an empty linelist from the torrc or command line.
-           As a special case, call this an error. Warn and ignore. */
-        log_warn(LD_CONFIG,
-                 "Linelist option '%s' has no value. Skipping.", c->key);
-      } else { /* not already cleared */
-        config_reset(fmt, options, var, use_defaults);
-      }
-    }
-    return 0;
-  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
-    config_reset(fmt, options, var, use_defaults);
-  }
-
-  if (options_seen && (var->type != CONFIG_TYPE_LINELIST &&
-                       var->type != CONFIG_TYPE_LINELIST_S)) {
-    /* We're tracking which options we've seen, and this option is not
-     * supposed to occur more than once. */
-    int var_index = (int)(var - fmt->vars);
-    if (bitarray_is_set(options_seen, var_index)) {
-      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
-               "value will be ignored.", var->name);
-    }
-    bitarray_set(options_seen, var_index);
-  }
-
-  if (config_assign_value(fmt, options, c, msg) < 0)
-    return -2;
-  return 0;
-}
-
-/** Restore the option named <b>key</b> in options to its default value.
- * Called from config_assign(). */
-static void
-config_reset_line(const config_format_t *fmt, void *options,
-                  const char *key, int use_defaults)
-{
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var)
-    return; /* give error on next pass. */
-
-  config_reset(fmt, options, var, use_defaults);
-}
-
-/** Return true iff value needs to be quoted and escaped to be used in
- * a configuration file. */
-static int
-config_value_needs_escape(const char *value)
-{
-  if (*value == '\"')
-    return 1;
-  while (*value) {
-    switch (*value)
-    {
-    case '\r':
-    case '\n':
-    case '#':
-      /* Note: quotes and backspaces need special handling when we are using
-       * quotes, not otherwise, so they don't trigger escaping on their
-       * own. */
-      return 1;
-    default:
-      if (!TOR_ISPRINT(*value))
-        return 1;
-    }
-    ++value;
-  }
-  return 0;
-}
-
-/** Return newly allocated line or lines corresponding to <b>key</b> in the
- * configuration <b>options</b>.  If <b>escape_val</b> is true and a
- * value needs to be quoted before it's put in a config file, quote and
- * escape that value. Return NULL if no such key exists. */
-config_line_t *
-config_get_assigned_option(const config_format_t *fmt, const void *options,
-                           const char *key, int escape_val)
-{
-  const config_var_t *var;
-  const void *value;
-  config_line_t *result;
-  tor_assert(options && key);
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var) {
-    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
-    return NULL;
-  }
-  value = STRUCT_VAR_P(options, var->var_offset);
-
-  result = tor_malloc_zero(sizeof(config_line_t));
-  result->key = tor_strdup(var->name);
-  switch (var->type)
-    {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      if (*(char**)value) {
-        result->value = tor_strdup(*(char**)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-        return NULL;
-      }
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      if (*(time_t*)value) {
-        result->value = tor_malloc(ISO_TIME_LEN+1);
-        format_iso_time(result->value, *(time_t*)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-      }
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_PORT:
-      if (*(int*)value == CFG_AUTO_PORT) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-      /* This means every or_options_t uint or bool element
-       * needs to be an int. Not, say, a uint16_t or char. */
-      tor_asprintf(&result->value, "%d", *(int*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_UINT64: FALLTHROUGH;
-    case CONFIG_TYPE_MEMUNIT:
-      tor_asprintf(&result->value, "%"PRIu64,
-                   (*(uint64_t*)value));
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      tor_asprintf(&result->value, "%f", *(double*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-
-    case CONFIG_TYPE_AUTOBOOL:
-      if (*(int*)value == -1) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_BOOL:
-      result->value = tor_strdup(*(int*)value ? "1" : "0");
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      result->value = routerset_to_string(*(routerset_t**)value);
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)value)
-        result->value =
-          smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
-      else
-        result->value = tor_strdup("");
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      log_fn(LOG_INFO, LD_CONFIG,
-             "You asked me for the value of an obsolete config option '%s'.",
-             key);
-      tor_free(result->key);
-      tor_free(result);
-      return NULL;
-    case CONFIG_TYPE_LINELIST_S:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup_and_filter(*(const config_line_t **)value,
-                                           key);
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_V:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup(*(const config_line_t**)value);
-      break;
-    default:
-      tor_free(result->key);
-      tor_free(result);
-      log_warn(LD_BUG,"Unknown type %d for known key '%s'",
-               var->type, key);
-      return NULL;
-    }
-
-  if (escape_val) {
-    config_line_t *line;
-    for (line = result; line; line = line->next) {
-      if (line->value && config_value_needs_escape(line->value)) {
-        char *newval = esc_for_log(line->value);
-        tor_free(line->value);
-        line->value = newval;
-      }
-    }
-  }
-
-  return result;
-}
-/** Iterate through the linked list of requested options <b>list</b>.
- * For each item, convert as appropriate and assign to <b>options</b>.
- * If an item is unrecognized, set *msg and return -1 immediately,
- * else return 0 for success.
- *
- * If <b>clear_first</b>, interpret config options as replacing (not
- * extending) their previous values. If <b>clear_first</b> is set,
- * then <b>use_defaults</b> to decide if you set to defaults after
- * clearing, or make the value 0 or NULL.
- *
- * Here are the use cases:
- * 1. A non-empty AllowInvalid line in your torrc. Appends to current
- *    if linelist, replaces current if csv.
- * 2. An empty AllowInvalid line in your torrc. Should clear it.
- * 3. "RESETCONF AllowInvalid" sets it to default.
- * 4. "SETCONF AllowInvalid" makes it NULL.
- * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
- *
- * Use_defaults   Clear_first
- *    0                0       "append"
- *    1                0       undefined, don't use
- *    0                1       "set to null first"
- *    1                1       "set to defaults first"
- * Return 0 on success, -1 on bad key, -2 on bad value.
- *
- * As an additional special case, if a LINELIST config option has
- * no value and clear_first is 0, then warn and ignore it.
- */
-
-/*
-There are three call cases for config_assign() currently.
-
-Case one: Torrc entry
-options_init_from_torrc() calls config_assign(0, 0)
-  calls config_assign_line(0, 0).
-    if value is empty, calls config_reset(0) and returns.
-    calls config_assign_value(), appends.
-
-Case two: setconf
-options_trial_assign() calls config_assign(0, 1)
-  calls config_reset_line(0)
-    calls config_reset(0)
-      calls option_clear().
-  calls config_assign_line(0, 1).
-    if value is empty, returns.
-    calls config_assign_value(), appends.
-
-Case three: resetconf
-options_trial_assign() calls config_assign(1, 1)
-  calls config_reset_line(1)
-    calls config_reset(1)
-      calls option_clear().
-      calls config_assign_value(default)
-  calls config_assign_line(1, 1).
-    returns.
-*/
-int
-config_assign(const config_format_t *fmt, void *options, config_line_t *list,
-              unsigned config_assign_flags, char **msg)
-{
-  config_line_t *p;
-  bitarray_t *options_seen;
-  const int n_options = config_count_options(fmt);
-  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
-  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
-
-  CONFIG_CHECK(fmt, options);
-
-  /* pass 1: normalize keys */
-  for (p = list; p; p = p->next) {
-    const char *full = config_expand_abbrev(fmt, p->key, 0, 1);
-    if (strcmp(full,p->key)) {
-      tor_free(p->key);
-      p->key = tor_strdup(full);
-    }
-  }
-
-  /* pass 2: if we're reading from a resetting source, clear all
-   * mentioned config options, and maybe set to their defaults. */
-  if (clear_first) {
-    for (p = list; p; p = p->next)
-      config_reset_line(fmt, options, p->key, use_defaults);
-  }
-
-  options_seen = bitarray_init_zero(n_options);
-  /* pass 3: assign. */
-  while (list) {
-    int r;
-    if ((r=config_assign_line(fmt, options, list, config_assign_flags,
-                              options_seen, msg))) {
-      bitarray_free(options_seen);
-      return r;
-    }
-    list = list->next;
-  }
-  bitarray_free(options_seen);
-
-  /** Now we're done assigning a group of options to the configuration.
-   * Subsequent group assignments should _replace_ linelists, not extend
-   * them. */
-  config_mark_lists_fragile(fmt, options);
-
-  return 0;
-}
-
-/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
- * Called from config_reset() and config_free(). */
-static void
-config_clear(const config_format_t *fmt, void *options,
-             const config_var_t *var)
-{
-  void *lvalue = STRUCT_VAR_P(options, var->var_offset);
-  (void)fmt; /* unused */
-  switch (var->type) {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      tor_free(*(char**)lvalue);
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      *(double*)lvalue = 0.0;
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      *(time_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-    case CONFIG_TYPE_PORT:
-    case CONFIG_TYPE_BOOL:
-      *(int*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_AUTOBOOL:
-      *(int*)lvalue = -1;
-      break;
-    case CONFIG_TYPE_UINT64:
-    case CONFIG_TYPE_MEMUNIT:
-      *(uint64_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      if (*(routerset_t**)lvalue) {
-        routerset_free(*(routerset_t**)lvalue);
-        *(routerset_t**)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)lvalue) {
-        SMARTLIST_FOREACH(*(smartlist_t **)lvalue, char *, cp, tor_free(cp));
-        smartlist_free(*(smartlist_t **)lvalue);
-        *(smartlist_t **)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_S:
-      config_free_lines(*(config_line_t **)lvalue);
-      *(config_line_t **)lvalue = NULL;
-      break;
-    case CONFIG_TYPE_LINELIST_V:
-      /* handled by linelist_s. */
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      break;
-  }
-}
-
-/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
- * <b>use_defaults</b>, set it to its default value.
- * Called by config_init() and option_reset_line() and option_assign_line(). */
-static void
-config_reset(const config_format_t *fmt, void *options,
-             const config_var_t *var, int use_defaults)
-{
-  config_line_t *c;
-  char *msg = NULL;
-  CONFIG_CHECK(fmt, options);
-  config_clear(fmt, options, var); /* clear it first */
-  if (!use_defaults)
-    return; /* all done */
-  if (var->initvalue) {
-    c = tor_malloc_zero(sizeof(config_line_t));
-    c->key = tor_strdup(var->name);
-    c->value = tor_strdup(var->initvalue);
-    if (config_assign_value(fmt, options, c, &msg) < 0) {
-      log_warn(LD_BUG, "Failed to assign default: %s", msg);
-      tor_free(msg); /* if this happens it's a bug */
-    }
-    config_free_lines(c);
-  }
-}
-
-/** Release storage held by <b>options</b>. */
-void
-config_free_(const config_format_t *fmt, void *options)
-{
-  int i;
-
-  if (!options)
-    return;
-
-  tor_assert(fmt);
-
-  for (i=0; fmt->vars[i].name; ++i)
-    config_clear(fmt, options, &(fmt->vars[i]));
-  if (fmt->extra) {
-    config_line_t **linep = STRUCT_VAR_P(options, fmt->extra->var_offset);
-    config_free_lines(*linep);
-    *linep = NULL;
-  }
-  tor_free(options);
-}
-
-/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
- * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
- */
-int
-config_is_same(const config_format_t *fmt,
-               const void *o1, const void *o2,
-               const char *name)
-{
-  config_line_t *c1, *c2;
-  int r = 1;
-  CONFIG_CHECK(fmt, o1);
-  CONFIG_CHECK(fmt, o2);
-
-  c1 = config_get_assigned_option(fmt, o1, name, 0);
-  c2 = config_get_assigned_option(fmt, o2, name, 0);
-  r = config_lines_eq(c1, c2);
-  config_free_lines(c1);
-  config_free_lines(c2);
-  return r;
-}
-
-/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
-void *
-config_dup(const config_format_t *fmt, const void *old)
-{
-  void *newopts;
-  int i;
-  config_line_t *line;
-
-  newopts = config_new(fmt);
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE)
-      continue;
-    line = config_get_assigned_option(fmt, old, fmt->vars[i].name, 0);
-    if (line) {
-      char *msg = NULL;
-      if (config_assign(fmt, newopts, line, 0, &msg) < 0) {
-        log_err(LD_BUG, "config_get_assigned_option() generated "
-                "something we couldn't config_assign(): %s", msg);
-        tor_free(msg);
-        tor_assert(0);
-      }
-    }
-    config_free_lines(line);
-  }
-  return newopts;
-}
-/** Set all vars in the configuration object <b>options</b> to their default
- * values. */
-void
-config_init(const config_format_t *fmt, void *options)
-{
-  int i;
-  const config_var_t *var;
-  CONFIG_CHECK(fmt, options);
-
-  for (i=0; fmt->vars[i].name; ++i) {
-    var = &fmt->vars[i];
-    if (!var->initvalue)
-      continue; /* defaults to NULL or 0 */
-    config_reset(fmt, options, var, 1);
-  }
-}
-
-/** Allocate and return a new string holding the written-out values of the vars
- * in 'options'.  If 'minimal', do not write out any default-valued vars.
- * Else, if comment_defaults, write default values as comments.
- */
-char *
-config_dump(const config_format_t *fmt, const void *default_options,
-            const void *options, int minimal,
-            int comment_defaults)
-{
-  smartlist_t *elements;
-  const void *defaults = default_options;
-  void *defaults_tmp = NULL;
-  config_line_t *line, *assigned;
-  char *result;
-  int i;
-  char *msg = NULL;
-
-  if (defaults == NULL) {
-    defaults = defaults_tmp = config_new(fmt);
-    config_init(fmt, defaults_tmp);
-  }
-
-  /* XXX use a 1 here so we don't add a new log line while dumping */
-  if (default_options == NULL) {
-    if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) {
-      log_err(LD_BUG, "Failed to validate default config: %s", msg);
-      tor_free(msg);
-      tor_assert(0);
-    }
-  }
-
-  elements = smartlist_new();
-  for (i=0; fmt->vars[i].name; ++i) {
-    int comment_option = 0;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE ||
-        fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    /* Don't save 'hidden' control variables. */
-    if (!strcmpstart(fmt->vars[i].name, "__"))
-      continue;
-    if (minimal && config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      continue;
-    else if (comment_defaults &&
-             config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      comment_option = 1;
-
-    line = assigned =
-      config_get_assigned_option(fmt, options, fmt->vars[i].name, 1);
-
-    for (; line; line = line->next) {
-      if (!strcmpstart(line->key, "__")) {
-        /* This check detects "hidden" variables inside LINELIST_V structures.
-         */
-        continue;
-      }
-      smartlist_add_asprintf(elements, "%s%s %s\n",
-                   comment_option ? "# " : "",
-                   line->key, line->value);
-    }
-    config_free_lines(assigned);
-  }
-
-  if (fmt->extra) {
-    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset);
-    for (; line; line = line->next) {
-      smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value);
-    }
-  }
-
-  result = smartlist_join_strings(elements, "", 0, NULL);
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-  if (defaults_tmp) {
-    fmt->free_fn(defaults_tmp);
-  }
-  return result;
-}
-
-/** Mapping from a unit name to a multiplier for converting that unit into a
- * base unit.  Used by config_parse_unit. */
-struct unit_table_t {
-  const char *unit; /**< The name of the unit */
-  uint64_t multiplier; /**< How many of the base unit appear in this unit */
-};
-
-/** Table to map the names of memory units to the number of bytes they
- * contain. */
-static struct unit_table_t memory_units[] = {
-  { "",          1 },
-  { "b",         1<< 0 },
-  { "byte",      1<< 0 },
-  { "bytes",     1<< 0 },
-  { "kb",        1<<10 },
-  { "kbyte",     1<<10 },
-  { "kbytes",    1<<10 },
-  { "kilobyte",  1<<10 },
-  { "kilobytes", 1<<10 },
-  { "kilobits",  1<<7  },
-  { "kilobit",   1<<7  },
-  { "kbits",     1<<7  },
-  { "kbit",      1<<7  },
-  { "m",         1<<20 },
-  { "mb",        1<<20 },
-  { "mbyte",     1<<20 },
-  { "mbytes",    1<<20 },
-  { "megabyte",  1<<20 },
-  { "megabytes", 1<<20 },
-  { "megabits",  1<<17 },
-  { "megabit",   1<<17 },
-  { "mbits",     1<<17 },
-  { "mbit",      1<<17 },
-  { "gb",        1<<30 },
-  { "gbyte",     1<<30 },
-  { "gbytes",    1<<30 },
-  { "gigabyte",  1<<30 },
-  { "gigabytes", 1<<30 },
-  { "gigabits",  1<<27 },
-  { "gigabit",   1<<27 },
-  { "gbits",     1<<27 },
-  { "gbit",      1<<27 },
-  { "tb",        UINT64_C(1)<<40 },
-  { "tbyte",     UINT64_C(1)<<40 },
-  { "tbytes",    UINT64_C(1)<<40 },
-  { "terabyte",  UINT64_C(1)<<40 },
-  { "terabytes", UINT64_C(1)<<40 },
-  { "terabits",  UINT64_C(1)<<37 },
-  { "terabit",   UINT64_C(1)<<37 },
-  { "tbits",     UINT64_C(1)<<37 },
-  { "tbit",      UINT64_C(1)<<37 },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of seconds they
- * contain. */
-static struct unit_table_t time_units[] = {
-  { "",         1 },
-  { "second",   1 },
-  { "seconds",  1 },
-  { "minute",   60 },
-  { "minutes",  60 },
-  { "hour",     60*60 },
-  { "hours",    60*60 },
-  { "day",      24*60*60 },
-  { "days",     24*60*60 },
-  { "week",     7*24*60*60 },
-  { "weeks",    7*24*60*60 },
-  { "month",    2629728, }, /* about 30.437 days */
-  { "months",   2629728, },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of milliseconds
- * they contain. */
-static struct unit_table_t time_msec_units[] = {
-  { "",         1 },
-  { "msec",     1 },
-  { "millisecond", 1 },
-  { "milliseconds", 1 },
-  { "second",   1000 },
-  { "seconds",  1000 },
-  { "minute",   60*1000 },
-  { "minutes",  60*1000 },
-  { "hour",     60*60*1000 },
-  { "hours",    60*60*1000 },
-  { "day",      24*60*60*1000 },
-  { "days",     24*60*60*1000 },
-  { "week",     7*24*60*60*1000 },
-  { "weeks",    7*24*60*60*1000 },
-  { NULL, 0 },
-};
-
-/** Parse a string <b>val</b> containing a number, zero or more
- * spaces, and an optional unit string.  If the unit appears in the
- * table <b>u</b>, then multiply the number by the unit multiplier.
- * On success, set *<b>ok</b> to 1 and return this product.
- * Otherwise, set *<b>ok</b> to 0.
- */
-static uint64_t
-config_parse_units(const char *val, struct unit_table_t *u, int *ok)
-{
-  uint64_t v = 0;
-  double d = 0;
-  int use_float = 0;
-  char *cp;
-
-  tor_assert(ok);
-
-  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
-  if (!*ok || (cp && *cp == '.')) {
-    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
-    if (!*ok)
-      goto done;
-    use_float = 1;
-  }
-
-  if (!cp) {
-    *ok = 1;
-    v = use_float ? ((uint64_t)d) :  v;
-    goto done;
-  }
-
-  cp = (char*) eat_whitespace(cp);
-
-  for ( ;u->unit;++u) {
-    if (!strcasecmp(u->unit, cp)) {
-      if (use_float)
-        v = (uint64_t)(u->multiplier * d);
-      else
-        v *= u->multiplier;
-      *ok = 1;
-      goto done;
-    }
-  }
-  log_warn(LD_CONFIG, "Unknown unit '%s'.", cp);
-  *ok = 0;
- done:
-
-  if (*ok)
-    return v;
-  else
-    return 0;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
- * and return the number of bytes specified.  Otherwise, set
- * *<b>ok</b> to false and return 0. */
-static uint64_t
-config_parse_memunit(const char *s, int *ok)
-{
-  uint64_t u = config_parse_units(s, memory_units, ok);
-  return u;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * time in milliseconds.  On success, set *<b>ok</b> to true and return
- * the number of milliseconds in the provided interval.  Otherwise, set
- * *<b>ok</b> to 0 and return -1. */
-static int
-config_parse_msec_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_msec_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of time.
- * On success, set *<b>ok</b> to true and return the number of seconds in
- * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
- */
-static int
-config_parse_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h
deleted file mode 100644
index 57f1ec1762..0000000000
--- a/src/app/config/confparse.h
+++ /dev/null
@@ -1,233 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.h
- *
- * \brief Header for confparse.c.
- */
-
-#ifndef TOR_CONFPARSE_H
-#define TOR_CONFPARSE_H
-
-/** Enumeration of types which option values can take */
-typedef enum config_type_t {
-  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
-  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
-  CONFIG_TYPE_UINT,         /**< A non-negative integer less than MAX_INT */
-  CONFIG_TYPE_INT,          /**< Any integer. */
-  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
-  CONFIG_TYPE_PORT,         /**< A port from 1...65535, 0 for "not set", or
-                             * "auto".  */
-  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
-  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
-                              * units */
-  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
-  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
-  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
-  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
-                             * 1 for true, and -1 for auto  */
-  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
-  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
-                              * optional whitespace. */
-  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
-                              * optional whitespace, representing intervals in
-                              * seconds, with optional units.  We allow
-                              * multiple values here for legacy reasons, but
-                              * ignore every value after the first. */
-  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
-  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
-                             * mixed with other keywords. */
-  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
-                             * context-sensitive config lines when fetching.
-                             */
-  CONFIG_TYPE_ROUTERSET,    /**< A list of router names, addrs, and fps,
-                             * parsed into a routerset_t. */
-  CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
-} config_type_t;
-
-#ifdef TOR_UNIT_TESTS
-/**
- * Union used when building in test mode typechecking the members of a type
- * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
- * it is used. */
-typedef union {
-  char **STRING;
-  char **FILENAME;
-  int *UINT; /* yes, really: Even though the confparse type is called
-              * "UINT", it still uses the C int type -- it just enforces that
-              * the values are in range [0,INT_MAX].
-              */
-  uint64_t *UINT64;
-  int *INT;
-  int *PORT;
-  int *INTERVAL;
-  int *MSEC_INTERVAL;
-  uint64_t *MEMUNIT;
-  double *DOUBLE;
-  int *BOOL;
-  int *AUTOBOOL;
-  time_t *ISOTIME;
-  smartlist_t **CSV;
-  int *CSV_INTERVAL;
-  struct config_line_t **LINELIST;
-  struct config_line_t **LINELIST_S;
-  struct config_line_t **LINELIST_V;
-  routerset_t **ROUTERSET;
-} confparse_dummy_values_t;
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** An abbreviation for a configuration option allowed on the command line. */
-typedef struct config_abbrev_t {
-  const char *abbreviated;
-  const char *full;
-  int commandline_only;
-  int warn;
-} config_abbrev_t;
-
-typedef struct config_deprecation_t {
-  const char *name;
-  const char *why_deprecated;
-} config_deprecation_t;
-
-/* Handy macro for declaring "In the config file or on the command line,
- * you can abbreviate <b>tok</b>s as <b>tok</b>". */
-#define PLURAL(tok) { #tok, #tok "s", 0, 0 }
-
-/** A variable allowed in the configuration file or on the command line. */
-typedef struct config_var_t {
-  const char *name; /**< The full keyword (case insensitive). */
-  config_type_t type; /**< How to interpret the type and turn it into a
-                       * value. */
-  off_t var_offset; /**< Offset of the corresponding member of or_options_t. */
-  const char *initvalue; /**< String (or null) describing initial value. */
-
-#ifdef TOR_UNIT_TESTS
-  /** Used for compiler-magic to typecheck the corresponding field in the
-   * corresponding struct. Only used in unit test mode, at compile-time. */
-  confparse_dummy_values_t var_ptr_dummy;
-#endif
-} config_var_t;
-
-/* Macros to define extra members inside config_var_t fields, and at the
- * end of a list of them.
- */
-#ifdef TOR_UNIT_TESTS
-/* This is a somewhat magic type-checking macro for users of confparse.c.
- * It initializes a union member "confparse_dummy_values_t.conftype" with
- * the address of a static member "tp_dummy.member".   This
- * will give a compiler warning unless the member field is of the correct
- * type.
- *
- * (This warning is mandatory, because a type mismatch here violates the type
- * compatibility constraint for simple assignment, and requires a diagnostic,
- * according to the C spec.)
- *
- * For example, suppose you say:
- *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
- * Then this macro will evaluate to:
- *     { .STRING = &or_options_t_dummy.Address }
- * And since confparse_dummy_values_t.STRING has type "char **", that
- * expression will create a warning unless or_options_t.Address also
- * has type "char *".
- */
-#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
-  { . conftype = &tp ## _dummy . member }
-#define CONF_TEST_MEMBERS(tp, conftype, member) \
-  , CONF_CHECK_VAR_TYPE(tp, conftype, member)
-#define END_OF_CONFIG_VARS                                      \
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } }
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  static tp tp ## _dummy
-#else /* !(defined(TOR_UNIT_TESTS)) */
-#define CONF_TEST_MEMBERS(tp, conftype, member)
-#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
-/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  struct tor_semicolon_eater
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Type of a callback to validate whether a given configuration is
- * well-formed and consistent. See options_trial_assign() for documentation
- * of arguments. */
-typedef int (*validate_fn_t)(void*,void*,void*,int,char**);
-
-/** Callback to free a configuration object. */
-typedef void (*free_cfg_fn_t)(void*);
-
-/** Information on the keys, value types, key-to-struct-member mappings,
- * variable descriptions, validation functions, and abbreviations for a
- * configuration or storage format. */
-typedef struct config_format_t {
-  size_t size; /**< Size of the struct that everything gets parsed into. */
-  uint32_t magic; /**< Required 'magic value' to make sure we have a struct
-                   * of the right type. */
-  off_t magic_offset; /**< Offset of the magic value within the struct. */
-  config_abbrev_t *abbrevs; /**< List of abbreviations that we expand when
-                             * parsing this format. */
-  const config_deprecation_t *deprecations; /** List of deprecated options */
-  config_var_t *vars; /**< List of variables we recognize, their default
-                       * values, and where we stick them in the structure. */
-  validate_fn_t validate_fn; /**< Function to validate config. */
-  free_cfg_fn_t free_fn; /**< Function to free the configuration. */
-  /** If present, extra is a LINELIST variable for unrecognized
-   * lines.  Otherwise, unrecognized lines are an error. */
-  config_var_t *extra;
-} config_format_t;
-
-/** Macro: assert that <b>cfg</b> has the right magic field for format
- * <b>fmt</b>. */
-#define CONFIG_CHECK(fmt, cfg) STMT_BEGIN                               \
-    tor_assert(fmt && cfg);                                             \
-    tor_assert((fmt)->magic ==                                          \
-               *(uint32_t*)STRUCT_VAR_P(cfg,fmt->magic_offset));        \
-  STMT_END
-
-#define CAL_USE_DEFAULTS      (1u<<0)
-#define CAL_CLEAR_FIRST       (1u<<1)
-#define CAL_WARN_DEPRECATIONS (1u<<2)
-
-void *config_new(const config_format_t *fmt);
-void config_free_(const config_format_t *fmt, void *options);
-#define config_free(fmt, options) do {                \
-    config_free_((fmt), (options));                   \
-    (options) = NULL;                                 \
-  } while (0)
-
-struct config_line_t *config_get_assigned_option(const config_format_t *fmt,
-                                          const void *options, const char *key,
-                                          int escape_val);
-int config_is_same(const config_format_t *fmt,
-                   const void *o1, const void *o2,
-                   const char *name);
-void config_init(const config_format_t *fmt, void *options);
-void *config_dup(const config_format_t *fmt, const void *old);
-char *config_dump(const config_format_t *fmt, const void *default_options,
-                  const void *options, int minimal,
-                  int comment_defaults);
-int config_assign(const config_format_t *fmt, void *options,
-                  struct config_line_t *list,
-                  unsigned flags, char **msg);
-config_var_t *config_find_option_mutable(config_format_t *fmt,
-                                         const char *key);
-const char *config_find_deprecation(const config_format_t *fmt,
-                                     const char *key);
-const config_var_t *config_find_option(const config_format_t *fmt,
-                                       const char *key);
-const char *config_expand_abbrev(const config_format_t *fmt,
-                                 const char *option,
-                                 int command_line, int warn_obsolete);
-void warn_deprecated_option(const char *what, const char *why);
-
-/* Helper macros to compare an option across two configuration objects */
-#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
-#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
-#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
-#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
-
-#endif /* !defined(TOR_CONFPARSE_H) */
diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h
index 74d2fefa16..e6be797017 100644
--- a/src/app/config/or_options_st.h
+++ b/src/app/config/or_options_st.h
@@ -18,6 +18,7 @@
 
 struct smartlist_t;
 struct config_line_t;
+struct config_suite_t;
 
 /** Enumeration of outbound address configuration types:
  * Exit-only, OR-only, or both */
@@ -72,6 +73,9 @@ struct or_options_t {
   routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as exits. */
+  routerset_t *MiddleNodes; /**< Structure containing nicknames, digests,
+                             * country codes and IP address patterns of ORs to
+                             * consider as middles. */
   routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as entry points. */
@@ -118,7 +122,6 @@ struct or_options_t {
   struct config_line_t *RecommendedVersions;
   struct config_line_t *RecommendedClientVersions;
   struct config_line_t *RecommendedServerVersions;
-  struct config_line_t *RecommendedPackages;
   /** Whether dirservers allow router descriptors with private IPs. */
   int DirAllowPrivateAddresses;
   /** Whether routers accept EXTEND cells to routers with private IPs. */
@@ -245,6 +248,17 @@ struct or_options_t {
    * pad to the server regardless of server support. */
   int ConnectionPadding;
 
+  /** Boolean: if true, then circuit padding will be negotiated by client
+   * and server, subject to consenus limits (default). If 0, it will be fully
+   * disabled. */
+  int CircuitPadding;
+
+  /** Boolean: if true, then this client will only use circuit padding
+   * algorithms that are known to use a low amount of overhead. If false,
+   * we will use all available circuit padding algorithms.
+   */
+  int ReducedCircuitPadding;
+
   /** To what authority types do we publish our descriptor? Choices are
    * "v1", "v2", "v3", "bridge", or "". */
   struct smartlist_t *PublishServerDescriptor;
@@ -666,6 +680,9 @@ struct or_options_t {
    * accessing this value directly.  */
   int ClientPreferIPv6DirPort;
 
+  /** If true, prefer an IPv4 or IPv6 OR port at random. */
+  int ClientAutoIPv6ORPort;
+
   /** The length of time that we think a consensus should be fresh. */
   int V3AuthVotingInterval;
   /** The length of time we think it will take to distribute votes. */
@@ -991,6 +1008,13 @@ struct or_options_t {
    */
   uint64_t MaxUnparseableDescSizeToLog;
 
+  /** Bool (default: 1): Under bandwidth pressure, if set to 1, the authority
+   * will always answer directory requests from relays but will start sending
+   * 503 error code for the other connections. If set to 0, all connections
+   * are considered the same and the authority will try to answer them all
+   * regardless of bandwidth pressure or not. */
+  int AuthDirRejectRequestsUnderLoad;
+
   /** Bool (default: 1): Switch for the shared random protocol. Only
    * relevant to a directory authority. If off, the authority won't
    * participate in the protocol. If on (default), a flag is added to the
@@ -1072,6 +1096,33 @@ struct or_options_t {
 
   /** Autobool: Do we refuse single hop client rendezvous? */
   int DoSRefuseSingleHopClientRendezvous;
+
+  /** Interval: how long without activity does it take for a client
+   * to become dormant?
+   **/
+  int DormantClientTimeout;
+
+  /** Boolean: true if having an idle stream is sufficient to prevent a client
+   * from becoming dormant.
+   **/
+  int DormantTimeoutDisabledByIdleStreams;
+
+  /** Boolean: true if Tor should be dormant the first time it starts with
+   * a datadirectory; false otherwise. */
+  int DormantOnFirstStartup;
+  /**
+   * Boolean: true if Tor should treat every startup event as cancelling
+   * a possible previous dormant state.
+   **/
+  int DormantCanceledByStartup;
+
+  /**
+   * Configuration objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant configuration object.
+   */
+  struct config_suite_t *subconfigs_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_OPTIONS_ST_H) */
diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h
index 5f8214d146..225003bb7e 100644
--- a/src/app/config/or_state_st.h
+++ b/src/app/config/or_state_st.h
@@ -15,6 +15,7 @@
 
 #include "lib/cc/torint.h"
 struct smartlist_t;
+struct config_suite_t;
 
 /** Persistent state for an onion router, as saved to disk. */
 struct or_state_t {
@@ -87,6 +88,21 @@ struct or_state_t {
 
   /** When did we last rotate our onion key?  "0" for 'no idea'. */
   time_t LastRotatedOnionKey;
+
+  /** Number of minutes since the last user-initiated request (as defined by
+   * the dormant net-status system.) Set to zero if we are dormant. */
+  int MinutesSinceUserActivity;
+  /** True if we were dormant when we last wrote the file; false if we
+   * weren't.  "auto" on initial startup. */
+  int Dormant;
+
+  /**
+   * State objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant state object if you must.
+   */
+  struct config_suite_t *substates_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_STATE_ST_H) */
diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c
index 89039a05b5..552bd2c443 100644
--- a/src/app/config/statefile.c
+++ b/src/app/config/statefile.c
@@ -32,10 +32,11 @@
 #include "core/or/or.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/stats/rephist.h"
@@ -45,6 +46,7 @@
 #include "app/config/statefile.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
@@ -68,16 +70,13 @@ static config_abbrev_t state_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_state_t);
 
-/*XXXX these next two are duplicates or near-duplicates from config.c */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member),       \
-      initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
 
 /** Array of "state" variables saved to the ~/.tor/state file. */
-static config_var_t state_vars_[] = {
+static const config_var_t state_vars_[] = {
   /* Remember to document these in state-contents.txt ! */
 
   V(AccountingBytesReadInInterval,    MEMUNIT,  NULL),
@@ -103,19 +102,19 @@ static config_var_t state_vars_[] = {
   V(HidServRevCounter,            LINELIST, NULL),
 
   V(BWHistoryReadEnds,                ISOTIME,  NULL),
-  V(BWHistoryReadInterval,            UINT,     "900"),
+  V(BWHistoryReadInterval,            POSINT,     "900"),
   V(BWHistoryReadValues,              CSV,      ""),
   V(BWHistoryReadMaxima,              CSV,      ""),
   V(BWHistoryWriteEnds,               ISOTIME,  NULL),
-  V(BWHistoryWriteInterval,           UINT,     "900"),
+  V(BWHistoryWriteInterval,           POSINT,     "900"),
   V(BWHistoryWriteValues,             CSV,      ""),
   V(BWHistoryWriteMaxima,             CSV,      ""),
   V(BWHistoryDirReadEnds,             ISOTIME,  NULL),
-  V(BWHistoryDirReadInterval,         UINT,     "900"),
+  V(BWHistoryDirReadInterval,         POSINT,     "900"),
   V(BWHistoryDirReadValues,           CSV,      ""),
   V(BWHistoryDirReadMaxima,           CSV,      ""),
   V(BWHistoryDirWriteEnds,            ISOTIME,  NULL),
-  V(BWHistoryDirWriteInterval,        UINT,     "900"),
+  V(BWHistoryDirWriteInterval,        POSINT,     "900"),
   V(BWHistoryDirWriteValues,          CSV,      ""),
   V(BWHistoryDirWriteMaxima,          CSV,      ""),
 
@@ -126,11 +125,14 @@ static config_var_t state_vars_[] = {
   V(LastRotatedOnionKey,              ISOTIME,  NULL),
   V(LastWritten,                      ISOTIME,  NULL),
 
-  V(TotalBuildTimes,                  UINT,     NULL),
-  V(CircuitBuildAbandonedCount,       UINT,     "0"),
+  V(TotalBuildTimes,                  POSINT,     NULL),
+  V(CircuitBuildAbandonedCount,       POSINT,     "0"),
   VAR("CircuitBuildTimeBin",          LINELIST_S, BuildtimeHistogram, NULL),
   VAR("BuildtimeHistogram",           LINELIST_V, BuildtimeHistogram, NULL),
 
+  V(MinutesSinceUserActivity,         POSINT,     NULL),
+  V(Dormant,                          AUTOBOOL, "auto"),
+
   END_OF_CONFIG_VARS
 };
 
@@ -143,31 +145,48 @@ static int or_state_validate_cb(void *old_options, void *options,
                                 void *default_options,
                                 int from_setconf, char **msg);
 
-static void or_state_free_cb(void *state);
-
 /** Magic value for or_state_t. */
 #define OR_STATE_MAGIC 0x57A73f57
 
 /** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines)
+static struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(or_state_t, ExtraLines),
 };
 
 /** Configuration format for or_state_t. */
 static const config_format_t state_format = {
   sizeof(or_state_t),
-  OR_STATE_MAGIC,
-  offsetof(or_state_t, magic_),
+  {
+   "or_state_t",
+   OR_STATE_MAGIC,
+   offsetof(or_state_t, magic_),
+  },
   state_abbrevs_,
   NULL,
   state_vars_,
   or_state_validate_cb,
-  or_state_free_cb,
+  NULL,
   &state_extra_var,
+  offsetof(or_state_t, substates_),
 };
 
+/* A global configuration manager for state-file objects */
+static config_mgr_t *state_mgr = NULL;
+
+/** Return the configuration manager for state-file objects. */
+static const config_mgr_t *
+get_state_mgr(void)
+{
+  if (PREDICT_UNLIKELY(state_mgr == NULL)) {
+    state_mgr = config_mgr_new(&state_format);
+    config_mgr_freeze(state_mgr);
+  }
+  return state_mgr;
+}
+
 /** Persistent serialized state. */
 static or_state_t *global_state = NULL;
 
@@ -262,12 +281,6 @@ or_state_validate_cb(void *old_state, void *state, void *default_state,
   return or_state_validate(state, msg);
 }
 
-static void
-or_state_free_cb(void *state)
-{
-  or_state_free_(state);
-}
-
 /** Return 0 if every setting in <b>state</b> is reasonable, and a
  * permissible transition from <b>old_state</b>.  Else warn and return -1.
  * Should have no side effects, except for normalizing the contents of
@@ -292,7 +305,7 @@ or_state_set(or_state_t *new_state)
   char *err = NULL;
   int ret = 0;
   tor_assert(new_state);
-  config_free(&state_format, global_state);
+  config_free(get_state_mgr(), global_state);
   global_state = new_state;
   if (entry_guards_parse_state(global_state, 1, &err)<0) {
     log_warn(LD_GENERAL,"%s",err);
@@ -308,6 +321,8 @@ or_state_set(or_state_t *new_state)
       get_circuit_build_times_mutable(),global_state) < 0) {
     ret = -1;
   }
+  netstatus_load_from_state(global_state, time(NULL));
+
   return ret;
 }
 
@@ -353,9 +368,8 @@ or_state_save_broken(char *fname)
 STATIC or_state_t *
 or_state_new(void)
 {
-  or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t));
-  new_state->magic_ = OR_STATE_MAGIC;
-  config_init(&state_format, new_state);
+  or_state_t *new_state = config_new(get_state_mgr());
+  config_init(get_state_mgr(), new_state);
 
   return new_state;
 }
@@ -396,7 +410,7 @@ or_state_load(void)
     int assign_retval;
     if (config_get_lines(contents, &lines, 0)<0)
       goto done;
-    assign_retval = config_assign(&state_format, new_state,
+    assign_retval = config_assign(get_state_mgr(), new_state,
                                   lines, 0, &errmsg);
     config_free_lines(lines);
     if (assign_retval<0)
@@ -423,7 +437,7 @@ or_state_load(void)
     or_state_save_broken(fname);
 
     tor_free(contents);
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
     new_state = or_state_new();
   } else if (contents) {
@@ -456,7 +470,7 @@ or_state_load(void)
   tor_free(fname);
   tor_free(contents);
   if (new_state)
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
   return r;
 }
@@ -499,6 +513,8 @@ or_state_save(time_t now)
   entry_guards_update_state(global_state);
   rep_hist_update_state(global_state);
   circuit_build_times_update_state(get_circuit_build_times(), global_state);
+  netstatus_flush_to_state(global_state, now);
+
   if (accounting_is_enabled(get_options()))
     accounting_run_housekeeping(now);
 
@@ -507,7 +523,7 @@ or_state_save(time_t now)
   tor_free(global_state->TorVersion);
   tor_asprintf(&global_state->TorVersion, "Tor %s", get_version());
 
-  state = config_dump(&state_format, NULL, global_state, 1, 0);
+  state = config_dump(get_state_mgr(), NULL, global_state, 1, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&contents,
                "# Tor state file last generated on %s local time\n"
@@ -717,7 +733,7 @@ or_state_free_(or_state_t *state)
   if (!state)
     return;
 
-  config_free(&state_format, state);
+  config_free(get_state_mgr(), state);
 }
 
 void
@@ -725,4 +741,5 @@ or_state_free_all(void)
 {
   or_state_free(global_state);
   global_state = NULL;
+  config_mgr_free(state_mgr);
 }
diff --git a/src/app/config/statefile.h b/src/app/config/statefile.h
index 1950078450..515c90a52f 100644
--- a/src/app/config/statefile.h
+++ b/src/app/config/statefile.h
@@ -31,6 +31,6 @@ STATIC struct config_line_t *get_transport_in_state_by_name(
 STATIC void or_state_free_(or_state_t *state);
 #define or_state_free(st) FREE_AND_NULL(or_state_t, or_state_free_, (st))
 STATIC or_state_t *or_state_new(void);
-#endif
+#endif /* defined(STATEFILE_PRIVATE) */
 
 #endif /* !defined(TOR_STATEFILE_H) */
diff --git a/src/app/config/testnet.inc b/src/app/config/testnet.inc
new file mode 100644
index 0000000000..0ed3c38627
--- /dev/null
+++ b/src/app/config/testnet.inc
@@ -0,0 +1,33 @@
+{ "DirAllowPrivateAddresses", "1" },
+{ "EnforceDistinctSubnets", "0" },
+{ "AssumeReachable", "1" },
+{ "AuthDirMaxServersPerAddr", "0" },
+{ "ClientBootstrapConsensusAuthorityDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusFallbackDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay", "0" },
+{ "ClientDNSRejectInternalAddresses", "0" },
+{ "ClientRejectInternalAddresses", "0" },
+{ "CountPrivateBandwidth", "1" },
+{ "ExitPolicyRejectPrivate", "0" },
+{ "ExtendAllowPrivateAddresses", "1" },
+{ "V3AuthVotingInterval", "5 minutes" },
+{ "V3AuthVoteDelay", "20 seconds" },
+{ "V3AuthDistDelay", "20 seconds" },
+{ "TestingV3AuthInitialVotingInterval", "150 seconds" },
+{ "TestingV3AuthInitialVoteDelay", "20 seconds" },
+{ "TestingV3AuthInitialDistDelay", "20 seconds" },
+{ "TestingAuthDirTimeToLearnReachability", "0 minutes" },
+{ "TestingEstimatedDescriptorPropagationTime", "0 minutes" },
+{ "MinUptimeHidServDirectoryV2", "0 minutes" },
+{ "TestingServerDownloadInitialDelay", "0" },
+{ "TestingClientDownloadInitialDelay", "0" },
+{ "TestingServerConsensusDownloadInitialDelay", "0" },
+{ "TestingClientConsensusDownloadInitialDelay", "0" },
+{ "TestingBridgeDownloadInitialDelay", "10" },
+{ "TestingBridgeBootstrapDownloadInitialDelay", "0" },
+{ "TestingClientMaxIntervalWithoutRequest", "5 seconds" },
+{ "TestingDirConnectionMaxStall", "30 seconds" },
+{ "TestingEnableConnBwEvent", "1" },
+{ "TestingEnableCellStatsEvent", "1" },
+{ "RendPostPeriod", "2 minutes" },
+{ "___UsingTestNetworkDefaults", "1" },
diff --git a/src/app/main/main.c b/src/app/main/main.c
index 67f2181cd5..1901479eba 100644
--- a/src/app/main/main.c
+++ b/src/app/main/main.c
@@ -15,63 +15,52 @@
 #include "app/config/statefile.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
-#include "core/or/channeltls.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitlist.h"
-#include "core/or/circuitmux_ewma.h"
 #include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/dos.h"
-#include "core/or/policies.h"
-#include "core/or/protover.h"
 #include "core/or/relay.h"
-#include "core/or/scheduler.h"
 #include "core/or/status.h"
 #include "feature/api/tor_api.h"
 #include "feature/api/tor_api_internal.h"
 #include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/entrynodes.h"
-#include "feature/client/transports.h"
 #include "feature/control/control.h"
-#include "feature/dirauth/bwauth.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
 #include "feature/dirparse/routerparse.h"
 #include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
-#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/relay/onion_queue.h"
 #include "feature/relay/routerkeys.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
-#include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/err/backtrace.h"
-#include "lib/geoip/geoip.h"
+#include "lib/net/resolve.h"
 
 #include "lib/process/waitpid.h"
+#include "lib/pubsub/pubsub_build.h"
 
 #include "lib/meminfo/meminfo.h"
 #include "lib/osinfo/uname.h"
@@ -83,10 +72,10 @@
 #include "lib/encoding/confline.h"
 #include "lib/evloop/timers.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
 
@@ -107,8 +96,6 @@
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
-void evdns_shutdown(int);
-
 #ifdef HAVE_RUST
 // helper function defined in Rust to output a log message indicating if tor is
 // running with Rust enabled. See src/rust/tor_util
@@ -301,6 +288,19 @@ process_signal(int sig)
       log_heartbeat(time(NULL));
       control_event_signal(sig);
       break;
+    case SIGACTIVE:
+      /* "SIGACTIVE" counts as ersatz user activity. */
+      note_user_activity(approx_time());
+      control_event_signal(sig);
+      break;
+    case SIGDORMANT:
+      /* "SIGDORMANT" means to ignore past user activity */
+      log_notice(LD_GENERAL, "Going dormant because of controller request.");
+      reset_user_activity(0);
+      set_network_participation(false);
+      schedule_rescan_periodic_events();
+      control_event_signal(sig);
+      break;
   }
 }
 
@@ -426,18 +426,6 @@ dumpstats(int severity)
   rend_service_dump_stats(severity);
 }
 
-/** Called by exit() as we shut down the process.
- */
-static void
-exit_function(void)
-{
-  /* NOTE: If we ever daemonize, this gets called immediately.  That's
-   * okay for now, because we only use this on Windows.  */
-#ifdef _WIN32
-  WSACleanup();
-#endif
-}
-
 #ifdef _WIN32
 #define UNIX_ONLY 0
 #else
@@ -482,6 +470,8 @@ static struct {
   { SIGNEWNYM, 0, NULL },
   { SIGCLEARDNSCACHE, 0, NULL },
   { SIGHEARTBEAT, 0, NULL },
+  { SIGACTIVE, 0, NULL },
+  { SIGDORMANT, 0, NULL },
   { -1, -1, NULL }
 };
 
@@ -546,18 +536,13 @@ tor_init(int argc, char *argv[])
   tor_snprintf(progname, sizeof(progname), "Tor %s", get_version());
   log_set_application_name(progname);
 
-  /* Set up the crypto nice and early */
-  if (crypto_early_init() < 0) {
-    log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!");
-    return -1;
-  }
-
   /* Initialize the history structures. */
   rep_hist_init();
   /* Initialize the service cache. */
   rend_cache_init();
   addressmap_init(); /* Init the client dns cache. Do it always, since it's
                       * cheap. */
+
   /* Initialize the HS subsystem. */
   hs_init();
 
@@ -632,12 +617,6 @@ tor_init(int argc, char *argv[])
   rust_log_welcome_string();
 #endif /* defined(HAVE_RUST) */
 
-  if (network_init()<0) {
-    log_err(LD_BUG,"Error initializing network; exiting.");
-    return -1;
-  }
-  atexit(exit_function);
-
   int init_rv = options_init_from_torrc(argc,argv);
   if (init_rv < 0) {
     log_err(LD_CONFIG,"Reading config failed--see warnings above.");
@@ -651,9 +630,17 @@ tor_init(int argc, char *argv[])
   /* The options are now initialised */
   const or_options_t *options = get_options();
 
-  /* Initialize channelpadding parameters to defaults until we get
-   * a consensus */
+  /* Initialize channelpadding and circpad parameters to defaults
+   * until we get a consensus */
   channelpadding_new_consensus_params(NULL);
+  circpad_new_consensus_params(NULL);
+
+  /* Initialize circuit padding to defaults+torrc until we get a consensus */
+  circpad_machines_init();
+
+  /* Initialize hidden service DoS subsystem. We need to do this once the
+   * configuration object has been set because it can be accessed. */
+  hs_dos_init();
 
   /* Initialize predicted ports list after loading options */
   predicted_ports_init();
@@ -670,10 +657,6 @@ tor_init(int argc, char *argv[])
     log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
     return -1;
   }
-  stream_choice_seed_weak_rng();
-  if (tor_init_libevent_rng() < 0) {
-    log_warn(LD_NET, "Problem initializing libevent RNG.");
-  }
 
   /* Scan/clean unparseable descriptors; after reading config */
   routerparse_init();
@@ -743,86 +726,6 @@ release_lockfile(void)
   }
 }
 
-/** Free all memory that we might have allocated somewhere.
- * If <b>postfork</b>, we are a worker process and we want to free
- * only the parts of memory that we won't touch. If !<b>postfork</b>,
- * Tor is shutting down and we should free everything.
- *
- * Helps us find the real leaks with sanitizers and the like. Also valgrind
- * should then report 0 reachable in its leak report (in an ideal world --
- * in practice libevent, SSL, libc etc never quite free everything). */
-void
-tor_free_all(int postfork)
-{
-  if (!postfork) {
-    evdns_shutdown(1);
-  }
-  geoip_free_all();
-  geoip_stats_free_all();
-  dirvote_free_all();
-  routerlist_free_all();
-  networkstatus_free_all();
-  addressmap_free_all();
-  dirserv_free_fingerprint_list();
-  dirserv_free_all();
-  dirserv_clear_measured_bw_cache();
-  rend_cache_free_all();
-  rend_service_authorization_free_all();
-  rep_hist_free_all();
-  dns_free_all();
-  clear_pending_onions();
-  circuit_free_all();
-  entry_guards_free_all();
-  pt_free_all();
-  channel_tls_free_all();
-  channel_free_all();
-  connection_free_all();
-  connection_edge_free_all();
-  scheduler_free_all();
-  nodelist_free_all();
-  microdesc_free_all();
-  routerparse_free_all();
-  ext_orport_free_all();
-  control_free_all();
-  tor_free_getaddrinfo_cache();
-  protover_free_all();
-  bridges_free_all();
-  consdiffmgr_free_all();
-  hs_free_all();
-  dos_free_all();
-  circuitmux_ewma_free_all();
-  accounting_free_all();
-
-  if (!postfork) {
-    config_free_all();
-    or_state_free_all();
-    router_free_all();
-    routerkeys_free_all();
-    policies_free_all();
-  }
-  if (!postfork) {
-    tor_tls_free_all();
-#ifndef _WIN32
-    tor_getpwnam(NULL);
-#endif
-  }
-  /* stuff in main.c */
-
-  tor_mainloop_free_all();
-
-  if (!postfork) {
-    release_lockfile();
-  }
-  tor_libevent_free_all();
-  /* Stuff in util.c and address.c*/
-  if (!postfork) {
-    escaped(NULL);
-    esc_router_info(NULL);
-    clean_up_backtrace_handler();
-    logs_free_all(); /* free log strings. do this last so logs keep working. */
-  }
-}
-
 /**
  * Remove the specified file, and log a warning if the operation fails for
  * any reason other than the file not existing. Ignores NULL filenames.
@@ -836,51 +739,6 @@ tor_remove_file(const char *filename)
   }
 }
 
-/** Do whatever cleanup is necessary before shutting Tor down. */
-void
-tor_cleanup(void)
-{
-  const or_options_t *options = get_options();
-  if (options->command == CMD_RUN_TOR) {
-    time_t now = time(NULL);
-    /* Remove our pid file. We don't care if there was an error when we
-     * unlink, nothing we could do about it anyways. */
-    tor_remove_file(options->PidFile);
-    /* Remove control port file */
-    tor_remove_file(options->ControlPortWriteToFile);
-    /* Remove cookie authentication file */
-    {
-      char *cookie_fname = get_controller_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    /* Remove Extended ORPort cookie authentication file */
-    {
-      char *cookie_fname = get_ext_or_auth_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    if (accounting_is_enabled(options))
-      accounting_record_bandwidth_usage(now, get_or_state());
-    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
-    or_state_save(now);
-    if (authdir_mode(options)) {
-      sr_save_and_cleanup();
-    }
-    if (authdir_mode_tests_reachability(options))
-      rep_hist_record_mtbf_data(now, 0);
-    keypin_close_journal();
-  }
-
-  timers_shutdown();
-
-  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
-                      later, if it makes shutdown unacceptably slow.  But for
-                      now, leave it here: it's helped us catch bugs in the
-                      past. */
-  crypto_global_cleanup();
-}
-
 /** Read/create keys as needed, and echo our fingerprint to stdout. */
 static int
 do_list_fingerprint(void)
@@ -1274,7 +1132,6 @@ int
 run_tor_main_loop(void)
 {
   handle_signals();
-  monotime_init();
   timers_initialize();
   initialize_mainloop_events();
 
@@ -1379,6 +1236,32 @@ run_tor_main_loop(void)
   return do_main_loop();
 }
 
+/** Install the publish/subscribe relationships for all the subsystems. */
+void
+pubsub_install(void)
+{
+    pubsub_builder_t *builder = pubsub_builder_new();
+    int r = subsystems_add_pubsub(builder);
+    tor_assert(r == 0);
+    r = tor_mainloop_connect_pubsub(builder); // consumes builder
+    tor_assert(r == 0);
+}
+
+/** Connect the mainloop to its publish/subscribe message delivery events if
+ * appropriate, and configure the global channels appropriately. */
+void
+pubsub_connect(void)
+{
+  if (get_options()->command == CMD_RUN_TOR) {
+    tor_mainloop_connect_pubsub_events();
+    /* XXXX For each pubsub channel, its delivery strategy should be set at
+     * this XXXX point, using tor_mainloop_set_delivery_strategy().
+     */
+    tor_mainloop_set_delivery_strategy("orconn", DELIV_IMMEDIATE);
+    tor_mainloop_set_delivery_strategy("ocirc", DELIV_IMMEDIATE);
+  }
+}
+
 /* Main entry point for the Tor process.  Called from tor_main(), and by
  * anybody embedding Tor. */
 int
@@ -1386,54 +1269,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
 {
   int result = 0;
 
-#ifdef _WIN32
-#ifndef HeapEnableTerminationOnCorruption
-#define HeapEnableTerminationOnCorruption 1
-#endif
-  /* On heap corruption, just give up; don't try to play along. */
-  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
-
-  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
-   * (On 64-bit Windows it always fails, and some compilers don't like the
-   * PSETDEP cast.)
-   * 32-bit Windows defines _WIN32.
-   * 64-bit Windows defines _WIN32 and _WIN64. */
-#ifndef _WIN64
-  /* Call SetProcessDEPPolicy to permanently enable DEP.
-     The function will not resolve on earlier versions of Windows,
-     and failure is not dangerous. */
-  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
-  if (hMod) {
-    typedef BOOL (WINAPI *PSETDEP)(DWORD);
-    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
-                           "SetProcessDEPPolicy");
-    if (setdeppolicy) {
-      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
-      setdeppolicy(3);
-    }
-  }
-#endif /* !defined(_WIN64) */
-#endif /* defined(_WIN32) */
-
-  {
-    int bt_err = configure_backtrace_handler(get_version());
-    if (bt_err < 0) {
-      log_warn(LD_BUG, "Unable to install backtrace handler: %s",
-               strerror(-bt_err));
-    }
-  }
-
 #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED
   event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_);
 #endif
 
-  init_protocol_warning_severity_level();
+  subsystems_init();
 
-  update_approx_time(time(NULL));
-  tor_threads_init();
-  tor_compress_init();
-  init_logging(0);
-  monotime_init();
+  init_protocol_warning_severity_level();
 
   int argc = tor_cfg->argc + tor_cfg->argc_owned;
   char **argv = tor_calloc(argc, sizeof(char*));
@@ -1451,6 +1293,9 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
      }
   }
 #endif /* defined(NT_SERVICE) */
+
+  pubsub_install();
+
   {
     int init_rv = tor_init(argc, argv);
     if (init_rv) {
@@ -1460,6 +1305,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     }
   }
 
+  pubsub_connect();
+
   if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
     sandbox_cfg_t* cfg = sandbox_init_filter();
 
@@ -1469,6 +1316,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
       tor_free_all(0);
       return -1;
     }
+    tor_make_getaddrinfo_cache_active();
 
     // registering libevent rng
 #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE
diff --git a/src/app/main/main.h b/src/app/main/main.h
index bbbbf984fb..76574a9071 100644
--- a/src/app/main/main.h
+++ b/src/app/main/main.h
@@ -21,11 +21,11 @@ void release_lockfile(void);
 
 void tor_remove_file(const char *filename);
 
-void tor_cleanup(void);
-void tor_free_all(int postfork);
-
 int tor_init(int argc, char **argv);
 
 int run_tor_main_loop(void);
 
+void pubsub_install(void);
+void pubsub_connect(void);
+
 #endif /* !defined(TOR_MAIN_H) */
diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c
index 05d203b0be..de82eeb1de 100644
--- a/src/app/main/ntmain.c
+++ b/src/app/main/ntmain.c
@@ -24,6 +24,7 @@
 #include "app/config/config.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
 #include "core/mainloop/mainloop.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/fs/winlib.h"
@@ -282,7 +283,9 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_install();
   r = tor_init(backup_argc, backup_argv);
+
   if (r) {
     /* Failed to start the Tor service */
     r = NT_SERVICE_ERROR_TORINIT_FAILED;
@@ -293,6 +296,8 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_connect();
+
   /* Set the service's status to SERVICE_RUNNING and start the main
    * event loop */
   service_status.dwCurrentState = SERVICE_RUNNING;
@@ -321,9 +326,12 @@ nt_service_main(void)
     errmsg = format_win32_error(result);
     printf("Service error %d : %s\n", (int) result, errmsg);
     tor_free(errmsg);
+
+    pubsub_install();
     if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
       if (tor_init(backup_argc, backup_argv))
         return;
+      pubsub_connect();
       switch (get_options()->command) {
       case CMD_RUN_TOR:
         run_tor_main_loop();
@@ -607,6 +615,7 @@ nt_service_install(int argc, char **argv)
                             &sidUse) == 0) {
     /* XXXX For some reason, the above test segfaults. Fix that. */
     printf("User \"%s\" doesn't seem to exist.\n", user_acct);
+    tor_free(command);
     return -1;
   } else {
     printf("Will try to install service as user \"%s\".\n", user_acct);
diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c
new file mode 100644
index 0000000000..93d6351d1b
--- /dev/null
+++ b/src/app/main/shutdown.c
@@ -0,0 +1,167 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file shutdown.c
+ * @brief Code to free global resources used by Tor.
+ *
+ * In the future, this should all be handled by the subsystem manager. */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop_pubsub.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/connection_edge.h"
+#include "core/or/dos.h"
+#include "core/or/scheduler.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendclient.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/geoip/geoip.h"
+
+void evdns_shutdown(int);
+
+/** Do whatever cleanup is necessary before shutting Tor down. */
+void
+tor_cleanup(void)
+{
+  const or_options_t *options = get_options();
+  if (options->command == CMD_RUN_TOR) {
+    time_t now = time(NULL);
+    /* Remove our pid file. We don't care if there was an error when we
+     * unlink, nothing we could do about it anyways. */
+    tor_remove_file(options->PidFile);
+    /* Remove control port file */
+    tor_remove_file(options->ControlPortWriteToFile);
+    /* Remove cookie authentication file */
+    {
+      char *cookie_fname = get_controller_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    /* Remove Extended ORPort cookie authentication file */
+    {
+      char *cookie_fname = get_ext_or_auth_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    if (accounting_is_enabled(options))
+      accounting_record_bandwidth_usage(now, get_or_state());
+    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
+    or_state_save(now);
+    if (authdir_mode(options)) {
+      sr_save_and_cleanup();
+    }
+    if (authdir_mode_tests_reachability(options))
+      rep_hist_record_mtbf_data(now, 0);
+  }
+
+  timers_shutdown();
+
+  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
+                      later, if it makes shutdown unacceptably slow.  But for
+                      now, leave it here: it's helped us catch bugs in the
+                      past. */
+}
+
+/** Free all memory that we might have allocated somewhere.
+ * If <b>postfork</b>, we are a worker process and we want to free
+ * only the parts of memory that we won't touch. If !<b>postfork</b>,
+ * Tor is shutting down and we should free everything.
+ *
+ * Helps us find the real leaks with sanitizers and the like. Also valgrind
+ * should then report 0 reachable in its leak report (in an ideal world --
+ * in practice libevent, SSL, libc etc never quite free everything). */
+void
+tor_free_all(int postfork)
+{
+  if (!postfork) {
+    evdns_shutdown(1);
+  }
+  geoip_free_all();
+  geoip_stats_free_all();
+  routerlist_free_all();
+  networkstatus_free_all();
+  addressmap_free_all();
+  dirserv_free_all();
+  rend_cache_free_all();
+  rend_service_authorization_free_all();
+  rep_hist_free_all();
+  circuit_free_all();
+  circpad_machines_free();
+  entry_guards_free_all();
+  pt_free_all();
+  channel_tls_free_all();
+  channel_free_all();
+  connection_free_all();
+  connection_edge_free_all();
+  scheduler_free_all();
+  nodelist_free_all();
+  microdesc_free_all();
+  routerparse_free_all();
+  control_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
+  hs_free_all();
+  dos_free_all();
+  circuitmux_ewma_free_all();
+  accounting_free_all();
+  circpad_free_all();
+
+  if (!postfork) {
+    config_free_all();
+    or_state_free_all();
+  }
+  if (!postfork) {
+#ifndef _WIN32
+    tor_getpwnam(NULL);
+#endif
+  }
+  /* stuff in main.c */
+
+  tor_mainloop_disconnect_pubsub();
+
+  if (!postfork) {
+    release_lockfile();
+  }
+
+  subsystems_shutdown();
+
+  /* Stuff in util.c and address.c*/
+  if (!postfork) {
+    esc_router_info(NULL);
+  }
+}
diff --git a/src/app/main/shutdown.h b/src/app/main/shutdown.h
new file mode 100644
index 0000000000..1bca96a0aa
--- /dev/null
+++ b/src/app/main/shutdown.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shutdown.h
+ * \brief Header file for shutdown.c.
+ **/
+
+#ifndef TOR_SHUTDOWN_H
+#define TOR_SHUTDOWN_H
+
+void tor_cleanup(void);
+void tor_free_all(int postfork);
+
+#endif /* !defined(TOR_SHUTDOWN_H) */
diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c
new file mode 100644
index 0000000000..5aa4fd76c9
--- /dev/null
+++ b/src/app/main/subsysmgr.c
@@ -0,0 +1,252 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * True iff we have checked tor_subsystems for consistency.
+ **/
+static bool subsystem_array_validated = false;
+
+/**
+ * True if a given subsystem is initialized.  Expand this array if there
+ * are more than this number of subsystems.  (We'd rather not
+ * dynamically allocate in this module.)
+ **/
+static bool sys_initialized[128];
+
+/**
+ * Exit with a raw assertion if the subsystems list is inconsistent;
+ * initialize the subsystem_initialized array.
+ **/
+static void
+check_and_setup(void)
+{
+  if (subsystem_array_validated)
+    return;
+
+  raw_assert(ARRAY_LENGTH(sys_initialized) >= n_tor_subsystems);
+  memset(sys_initialized, 0, sizeof(sys_initialized));
+
+  int last_level = MIN_SUBSYS_LEVEL;
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) {
+      fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. "
+              "It is supposed to be between %d and %d (inclusive).\n",
+              sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    if (sys->level < last_level) {
+      fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. "
+              "Its level is %d; but the previous subsystem's level was %d.\n",
+              sys->name, i, sys->level, last_level);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    last_level = sys->level;
+  }
+
+  subsystem_array_validated = true;
+}
+
+/**
+ * Initialize all the subsystems; exit on failure.
+ **/
+int
+subsystems_init(void)
+{
+  return subsystems_init_upto(MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Initialize all the subsystems whose level is less than or equal to
+ * <b>target_level</b>; exit on failure.
+ **/
+int
+subsystems_init_upto(int target_level)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (sys_initialized[i])
+      continue;
+    int r = 0;
+    if (sys->initialize) {
+      // Note that the logging subsystem is designed so that it does no harm
+      // to log a message in an uninitialized state.  These messages will be
+      // discarded for now, however.
+      log_debug(LD_GENERAL, "Initializing %s", sys->name);
+      r = sys->initialize();
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n",
+              sys->name, i);
+      raw_assert_unreached_msg("A subsystem couldn't be initialized.");
+    }
+    sys_initialized[i] = true;
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems of level no more than <b>target_level</b>.
+ **/
+int
+subsystems_add_pubsub_upto(pubsub_builder_t *builder,
+                           int target_level)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (! sys_initialized[i])
+      continue;
+    int r = 0;
+    if (sys->add_pubsub) {
+      subsys_id_t sysid = get_subsys_id(sys->name);
+      raw_assert(sysid != ERROR_ID);
+      pubsub_connector_t *connector;
+      connector = pubsub_connector_for_subsystem(builder, sysid);
+      r = sys->add_pubsub(connector);
+      pubsub_connector_free(connector);
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) could not connect to "
+              "publish/subscribe system.", sys->name, sys->level);
+      raw_assert_unreached_msg("A subsystem couldn't be connected.");
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems.
+ **/
+int
+subsystems_add_pubsub(pubsub_builder_t *builder)
+{
+  return subsystems_add_pubsub_upto(builder, MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Shut down all the subsystems.
+ **/
+void
+subsystems_shutdown(void)
+{
+  subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1);
+}
+
+/**
+ * Shut down all the subsystems whose level is above <b>target_level</b>.
+ **/
+void
+subsystems_shutdown_downto(int target_level)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level <= target_level)
+      break;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->shutdown) {
+      log_debug(LD_GENERAL, "Shutting down %s", sys->name);
+      sys->shutdown();
+    }
+    sys_initialized[i] = false;
+  }
+}
+
+/**
+ * Run pre-fork code on all subsystems that declare any
+ **/
+void
+subsystems_prefork(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->prefork) {
+      log_debug(LD_GENERAL, "Pre-fork: %s", sys->name);
+      sys->prefork();
+    }
+  }
+}
+
+/**
+ * Run post-fork code on all subsystems that declare any
+ **/
+void
+subsystems_postfork(void)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->postfork) {
+      log_debug(LD_GENERAL, "Post-fork: %s", sys->name);
+      sys->postfork();
+    }
+  }
+}
+
+/**
+ * Run thread-cleanup code on all subsystems that declare any
+ **/
+void
+subsystems_thread_cleanup(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->thread_cleanup) {
+      log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name);
+      sys->thread_cleanup();
+    }
+  }
+}
diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h
new file mode 100644
index 0000000000..d4426614e3
--- /dev/null
+++ b/src/app/main/subsysmgr.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SUBSYSMGR_T
+#define TOR_SUBSYSMGR_T
+
+#include "lib/subsys/subsys.h"
+
+extern const struct subsys_fns_t *tor_subsystems[];
+extern const unsigned n_tor_subsystems;
+
+int subsystems_init(void);
+int subsystems_init_upto(int level);
+
+struct pubsub_builder_t;
+int subsystems_add_pubsub_upto(struct pubsub_builder_t *builder,
+                               int target_level);
+int subsystems_add_pubsub(struct pubsub_builder_t *builder);
+
+void subsystems_shutdown(void);
+void subsystems_shutdown_downto(int level);
+
+void subsystems_prefork(void);
+void subsystems_postfork(void);
+void subsystems_thread_cleanup(void);
+
+#endif /* !defined(TOR_SUBSYSMGR_T) */
diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c
new file mode 100644
index 0000000000..1af9340c1a
--- /dev/null
+++ b/src/app/main/subsystem_list.c
@@ -0,0 +1,71 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+#include "core/mainloop/mainloop_sys.h"
+#include "core/or/ocirc_event_sys.h"
+#include "core/or/or_sys.h"
+#include "core/or/orconn_event_sys.h"
+#include "feature/control/btrack_sys.h"
+#include "feature/relay/relay_sys.h"
+#include "lib/compress/compress_sys.h"
+#include "lib/crypt_ops/crypto_sys.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/log/log_sys.h"
+#include "lib/net/network_sys.h"
+#include "lib/process/process_sys.h"
+#include "lib/process/winprocess_sys.h"
+#include "lib/thread/thread_sys.h"
+#include "lib/time/time_sys.h"
+#include "lib/tls/tortls_sys.h"
+#include "lib/wallclock/wallclock_sys.h"
+#include "lib/evloop/evloop_sys.h"
+
+#include "feature/dirauth/dirauth_sys.h"
+
+#include <stddef.h>
+
+/**
+ * Global list of the subsystems in Tor, in the order of their initialization.
+ * Want to know the exact level numbers?
+ * We'll implement a level dump command in #31614.
+ **/
+const subsys_fns_t *tor_subsystems[] = {
+  &sys_winprocess,
+  &sys_torerr,
+
+  &sys_wallclock,
+  &sys_threads,
+  &sys_logging,
+
+  &sys_time,
+  &sys_network,
+
+  &sys_compress,
+  &sys_crypto,
+  &sys_tortls,
+  &sys_process,
+
+  &sys_orconn_event,
+  &sys_ocirc_event,
+  &sys_btrack,
+
+  &sys_evloop,
+
+  &sys_mainloop,
+  &sys_or,
+
+  &sys_relay,
+
+#ifdef HAVE_MODULE_DIRAUTH
+  &sys_dirauth,
+#endif
+};
+
+const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems);
diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py
index 3a454a3fc1..b861e9433e 100644
--- a/src/config/mmdb-convert.py
+++ b/src/config/mmdb-convert.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/python
 
 #   This software has been dedicated to the public domain under the CC0
 #   public domain dedication.
@@ -77,7 +77,7 @@ def to_int32(s):
 
 def to_int28(s):
     "Parse a pair of big-endian 28-bit integers from bytestring s."
-    a, b = unpack("!LL", s + b'\x00')
+    a, b = struct.unpack("!LL", s + b'\x00')
     return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8)
 
 class Tree(object):
diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging
index cb3adca35c..90bad7f7cc 100644
--- a/src/config/torrc.minimal.in-staging
+++ b/src/config/torrc.minimal.in-staging
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in
index c2ae707e93..51e1c3af4b 100644
--- a/src/config/torrc.sample.in
+++ b/src/config/torrc.sample.in
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
@@ -174,13 +177,11 @@
 
 ## Uncomment this if you want your relay to be an exit, with the default
 ## exit policy (or whatever exit policy you set below).
-## (If ReducedExitPolicy or ExitPolicy are set, relays are exits.
-## If neither exit policy option is set, relays are non-exits.)
+## (If ReducedExitPolicy, ExitPolicy, or IPv6Exit are set, relays are exits.
+## If none of these options are set, relays are non-exits.)
 #ExitRelay 1
 
 ## Uncomment this if you want your relay to allow IPv6 exit traffic.
-## You must also set ExitRelay, ReducedExitPolicy, or ExitPolicy to make your
-## relay into an exit.
 ## (Relays do not allow any exit traffic by default.)
 #IPv6Exit 1
 
diff --git a/src/core/crypto/.may_include b/src/core/crypto/.may_include
new file mode 100644
index 0000000000..5782a36797
--- /dev/null
+++ b/src/core/crypto/.may_include
@@ -0,0 +1,10 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/cc/*.h
+lib/log/*.h
+
+core/crypto/*.h
diff --git a/src/core/crypto/hs_ntor.c b/src/core/crypto/hs_ntor.c
index c34073690e..add8a2b8f2 100644
--- a/src/core/crypto/hs_ntor.c
+++ b/src/core/crypto/hs_ntor.c
@@ -176,7 +176,6 @@ get_introduce1_key_material(const uint8_t *secret_input,
   uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN];
   uint8_t info_blob[INFO_BLOB_LEN];
   uint8_t kdf_input[KDF_INPUT_LEN];
-  crypto_xof_t *xof;
   uint8_t *ptr;
 
   /* Let's build info */
@@ -193,10 +192,8 @@ get_introduce1_key_material(const uint8_t *secret_input,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Now we need to run kdf_input over SHAKE-256 */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ;
-  crypto_xof_free(xof);
+  crypto_xof(keystream, sizeof(keystream),
+             kdf_input, sizeof(kdf_input));
 
   { /* Get the keys */
     memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN);
@@ -594,7 +591,6 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
 {
   uint8_t *ptr;
   uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN];
-  crypto_xof_t *xof;
 
   /* Sanity checks on lengths to make sure we are good */
   if (BUG(seed_len != DIGEST256_LEN)) {
@@ -611,10 +607,8 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Generate the keys */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN);
-  crypto_xof_free(xof);
+  crypto_xof(keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN,
+             kdf_input, sizeof(kdf_input));
 
   return 0;
 }
diff --git a/src/core/crypto/onion_crypto.h b/src/core/crypto/onion_crypto.h
index 1cddde3610..7abdd6538e 100644
--- a/src/core/crypto/onion_crypto.h
+++ b/src/core/crypto/onion_crypto.h
@@ -44,4 +44,4 @@ void server_onion_keys_free_(server_onion_keys_t *keys);
 #define server_onion_keys_free(keys) \
   FREE_AND_NULL(server_onion_keys_t, server_onion_keys_free_, (keys))
 
-#endif
+#endif /* !defined(TOR_ONION_CRYPTO_H) */
diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c
index 0b83b2d0a5..8a285131a8 100644
--- a/src/core/crypto/relay_crypto.c
+++ b/src/core/crypto/relay_crypto.c
@@ -6,12 +6,14 @@
 
 #include "core/or/or.h"
 #include "core/or/circuitlist.h"
+#include "core/or/crypt_path.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "core/crypto/hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
+#include "core/or/sendme.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
@@ -20,7 +22,7 @@
 /** Update digest from the payload of cell. Assign integrity part to
  * cell.
  */
-static void
+void
 relay_set_digest(crypto_digest_t *digest, cell_t *cell)
 {
   char integrity[4];
@@ -84,12 +86,39 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
  *
  * Note that we use the same operation for encrypting and for decrypting.
  */
-static void
+void
 relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in)
 {
   crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
 }
 
+/** Return the sendme_digest within the <b>crypto</b> object. */
+uint8_t *
+relay_crypto_get_sendme_digest(relay_crypto_t *crypto)
+{
+  tor_assert(crypto);
+  return crypto->sendme_digest;
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                  bool is_foward_digest)
+{
+  struct crypto_digest_t *digest;
+
+  tor_assert(crypto);
+
+  digest = crypto->b_digest;
+  if (is_foward_digest) {
+    digest = crypto->f_digest;
+  }
+
+  crypto_digest_get_digest(digest, (char *) crypto->sendme_digest,
+                           sizeof(crypto->sendme_digest));
+}
+
 /** Do the appropriate en/decryptions for <b>cell</b> arriving on
  * <b>circ</b> in direction <b>cell_direction</b>.
  *
@@ -134,12 +163,12 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell,
         tor_assert(thishop);
 
         /* decrypt one layer */
-        relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload);
+        cpath_crypt_cell(thishop, cell->payload, true);
 
         relay_header_unpack(&rh, cell->payload);
         if (rh.recognized == 0) {
           /* it's possibly recognized. have to check digest to be sure. */
-          if (relay_digest_matches(thishop->crypto.b_digest, cell)) {
+          if (relay_digest_matches(cpath_get_incoming_digest(thishop), cell)) {
             *recognized = 1;
             *layer_hint = thishop;
             return 0;
@@ -187,14 +216,17 @@ relay_encrypt_cell_outbound(cell_t *cell,
                             crypt_path_t *layer_hint)
 {
   crypt_path_t *thishop; /* counter for repeated crypts */
-  relay_set_digest(layer_hint->crypto.f_digest, cell);
+  cpath_set_cell_forward_digest(layer_hint, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(circ), layer_hint);
 
   thishop = layer_hint;
   /* moving from farthest to nearest hop */
   do {
     tor_assert(thishop);
     log_debug(LD_OR,"encrypting a layer of the relay cell.");
-    relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload);
+    cpath_crypt_cell(thishop, cell->payload, false);
 
     thishop = thishop->prev;
   } while (thishop != circ->cpath->prev);
@@ -212,6 +244,10 @@ relay_encrypt_cell_inbound(cell_t *cell,
                            or_circuit_t *or_circ)
 {
   relay_set_digest(or_circ->crypto.b_digest, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(or_circ), NULL);
+
   /* encrypt one layer */
   relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload);
 }
diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h
index 45a21d14ab..9478f8d359 100644
--- a/src/core/crypto/relay_crypto.h
+++ b/src/core/crypto/relay_crypto.h
@@ -27,5 +27,16 @@ void relay_crypto_clear(relay_crypto_t *crypto);
 
 void relay_crypto_assert_ok(const relay_crypto_t *crypto);
 
+uint8_t *relay_crypto_get_sendme_digest(relay_crypto_t *crypto);
+
+void relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                       bool is_foward_digest);
+
+void
+relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in);
+
+void
+relay_set_digest(crypto_digest_t *digest, cell_t *cell);
+
 #endif /* !defined(TOR_RELAY_CRYPTO_H) */
 
diff --git a/src/core/include.am b/src/core/include.am
index 1b8ef2ac58..9b4b251c81 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -6,11 +6,14 @@ noinst_LIBRARIES += \
 	src/core/libtor-app-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 LIBTOR_APP_A_SOURCES = 				\
 	src/app/config/config.c			\
-	src/app/config/confparse.c		\
 	src/app/config/statefile.c		\
 	src/app/main/main.c			\
+	src/app/main/shutdown.c			\
+	src/app/main/subsystem_list.c		\
+	src/app/main/subsysmgr.c		\
 	src/core/crypto/hs_ntor.c		\
 	src/core/crypto/onion_crypto.c		\
 	src/core/crypto/onion_fast.c		\
@@ -20,6 +23,8 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/mainloop/connection.c		\
 	src/core/mainloop/cpuworker.c		\
 	src/core/mainloop/mainloop.c		\
+	src/core/mainloop/mainloop_pubsub.c	\
+	src/core/mainloop/mainloop_sys.c	\
 	src/core/mainloop/netstatus.c		\
 	src/core/mainloop/periodic.c		\
 	src/core/or/address_set.c		\
@@ -30,13 +35,20 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/or/circuitlist.c		\
 	src/core/or/circuitmux.c		\
 	src/core/or/circuitmux_ewma.c		\
+	src/core/or/circuitpadding.c		\
+	src/core/or/circuitpadding_machines.c	\
 	src/core/or/circuitstats.c		\
 	src/core/or/circuituse.c		\
+	src/core/or/crypt_path.c		\
 	src/core/or/command.c			\
 	src/core/or/connection_edge.c		\
 	src/core/or/connection_or.c		\
 	src/core/or/dos.c			\
 	src/core/or/onion.c			\
+	src/core/or/ocirc_event.c		\
+	src/core/or/or_periodic.c		\
+	src/core/or/or_sys.c			\
+	src/core/or/orconn_event.c		\
 	src/core/or/policies.c			\
 	src/core/or/protover.c			\
 	src/core/or/protover_rust.c		\
@@ -45,6 +57,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/or/scheduler.c			\
 	src/core/or/scheduler_kist.c		\
 	src/core/or/scheduler_vanilla.c		\
+	src/core/or/sendme.c			\
 	src/core/or/status.c			\
 	src/core/or/versions.c			\
 	src/core/proto/proto_cell.c		\
@@ -59,10 +72,21 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/client/dnsserv.c		\
 	src/feature/client/entrynodes.c		\
 	src/feature/client/transports.c		\
+	src/feature/control/btrack.c		\
+	src/feature/control/btrack_circuit.c	\
+	src/feature/control/btrack_orconn.c	\
+	src/feature/control/btrack_orconn_cevent.c	\
+	src/feature/control/btrack_orconn_maps.c	\
 	src/feature/control/control.c		\
+	src/feature/control/control_auth.c	\
+	src/feature/control/control_bootstrap.c	\
+	src/feature/control/control_cmd.c	\
+	src/feature/control/control_events.c	\
+	src/feature/control/control_fmt.c	\
+	src/feature/control/control_getinfo.c	\
+	src/feature/control/control_proto.c	\
 	src/feature/control/fmt_serverstatus.c  \
 	src/feature/control/getinfo_geoip.c	\
-	src/feature/dirauth/keypin.c		\
 	src/feature/dircache/conscache.c	\
 	src/feature/dircache/consdiffmgr.c	\
 	src/feature/dircache/dircache.c		\
@@ -92,6 +116,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/hs/hs_config.c		\
 	src/feature/hs/hs_control.c		\
 	src/feature/hs/hs_descriptor.c		\
+	src/feature/hs/hs_dos.c			\
 	src/feature/hs/hs_ident.c		\
 	src/feature/hs/hs_intropoint.c		\
 	src/feature/hs/hs_service.c		\
@@ -99,13 +124,13 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/hs_common/replaycache.c	\
 	src/feature/hs_common/shared_random_client.c	\
 	src/feature/keymgt/loadkey.c		\
-	src/feature/dirauth/keypin.c		\
 	src/feature/nodelist/authcert.c		\
 	src/feature/nodelist/describe.c		\
 	src/feature/nodelist/dirlist.c		\
 	src/feature/nodelist/microdesc.c	\
 	src/feature/nodelist/networkstatus.c	\
 	src/feature/nodelist/nickname.c		\
+	src/feature/nodelist/nodefamily.c	\
 	src/feature/nodelist/nodelist.c		\
 	src/feature/nodelist/node_select.c	\
 	src/feature/nodelist/routerinfo.c	\
@@ -116,6 +141,8 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/relay/dns.c			\
 	src/feature/relay/ext_orport.c		\
 	src/feature/relay/onion_queue.c		\
+	src/feature/relay/relay_periodic.c	\
+	src/feature/relay/relay_sys.c		\
 	src/feature/relay/router.c		\
 	src/feature/relay/routerkeys.c		\
 	src/feature/relay/routermode.c		\
@@ -130,17 +157,6 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/stats/rephist.c		\
 	src/feature/stats/predict_ports.c
 
-# These should eventually move into module_dirauth_sources, but for now
-# the separation is only in the code location.
-LIBTOR_APP_A_SOURCES += 			\
-	src/feature/dirauth/bwauth.c		\
-	src/feature/dirauth/dsigs_parse.c	\
-	src/feature/dirauth/guardfraction.c	\
-	src/feature/dirauth/reachability.c	\
-	src/feature/dirauth/recommend_pkg.c	\
-	src/feature/dirauth/process_descs.c	\
-	src/feature/dirauth/voteflags.c
-
 if BUILD_NT_SERVICES
 LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
 endif
@@ -156,10 +172,21 @@ LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES)
 # The Directory Authority module.
 MODULE_DIRAUTH_SOURCES = 					\
 	src/feature/dirauth/authmode.c				\
+	src/feature/dirauth/bridgeauth.c			\
+	src/feature/dirauth/bwauth.c				\
+	src/feature/dirauth/dirauth_periodic.c			\
+	src/feature/dirauth/dirauth_sys.c			\
 	src/feature/dirauth/dircollate.c			\
 	src/feature/dirauth/dirvote.c				\
+	src/feature/dirauth/dsigs_parse.c			\
+	src/feature/dirauth/guardfraction.c			\
+	src/feature/dirauth/keypin.c				\
+	src/feature/dirauth/process_descs.c			\
+	src/feature/dirauth/reachability.c			\
+	src/feature/dirauth/recommend_pkg.c			\
 	src/feature/dirauth/shared_random.c			\
-	src/feature/dirauth/shared_random_state.c
+	src/feature/dirauth/shared_random_state.c		\
+	src/feature/dirauth/voteflags.c
 
 if BUILD_MODULE_DIRAUTH
 LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
@@ -183,14 +210,16 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\""		\
 src_core_libtor_app_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_core_libtor_app_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/app/config/config.h				\
-	src/app/config/confparse.h			\
 	src/app/config/or_options_st.h			\
 	src/app/config/or_state_st.h			\
 	src/app/config/statefile.h			\
 	src/app/main/main.h				\
 	src/app/main/ntmain.h				\
+	src/app/main/shutdown.h 			\
+	src/app/main/subsysmgr.h			\
 	src/core/crypto/hs_ntor.h			\
 	src/core/crypto/onion_crypto.h	        	\
 	src/core/crypto/onion_fast.h			\
@@ -200,6 +229,8 @@ noinst_HEADERS +=					\
 	src/core/mainloop/connection.h			\
 	src/core/mainloop/cpuworker.h			\
 	src/core/mainloop/mainloop.h			\
+	src/core/mainloop/mainloop_pubsub.h		\
+	src/core/mainloop/mainloop_sys.h		\
 	src/core/mainloop/netstatus.h			\
 	src/core/mainloop/periodic.h			\
 	src/core/or/addr_policy_st.h			\
@@ -215,28 +246,37 @@ noinst_HEADERS +=					\
 	src/core/or/circuitmux.h			\
 	src/core/or/circuitmux_ewma.h			\
 	src/core/or/circuitstats.h			\
+	src/core/or/circuitpadding.h			\
+	src/core/or/circuitpadding_machines.h		\
 	src/core/or/circuituse.h			\
 	src/core/or/command.h				\
 	src/core/or/connection_edge.h			\
 	src/core/or/connection_or.h			\
 	src/core/or/connection_st.h			\
+	src/core/or/crypt_path.h			\
 	src/core/or/cpath_build_state_st.h		\
 	src/core/or/crypt_path_reference_st.h		\
 	src/core/or/crypt_path_st.h			\
 	src/core/or/destroy_cell_queue_st.h		\
 	src/core/or/dos.h				\
 	src/core/or/edge_connection_st.h		\
-	src/core/or/half_edge_st.h		\
+	src/core/or/half_edge_st.h			\
 	src/core/or/entry_connection_st.h		\
 	src/core/or/entry_port_cfg_st.h			\
 	src/core/or/extend_info_st.h			\
 	src/core/or/listener_connection_st.h		\
 	src/core/or/onion.h				\
 	src/core/or/or.h				\
+	src/core/or/or_periodic.h			\
+	src/core/or/or_sys.h				\
+	src/core/or/orconn_event.h			\
+	src/core/or/orconn_event_sys.h			\
 	src/core/or/or_circuit_st.h			\
 	src/core/or/or_connection_st.h			\
 	src/core/or/or_handshake_certs_st.h		\
 	src/core/or/or_handshake_state_st.h		\
+	src/core/or/ocirc_event.h			\
+	src/core/or/ocirc_event_sys.h			\
 	src/core/or/origin_circuit_st.h			\
 	src/core/or/policies.h				\
 	src/core/or/port_cfg_st.h			\
@@ -245,6 +285,7 @@ noinst_HEADERS +=					\
 	src/core/or/relay.h				\
 	src/core/or/relay_crypto_st.h			\
 	src/core/or/scheduler.h				\
+	src/core/or/sendme.h				\
 	src/core/or/server_port_cfg_st.h		\
 	src/core/or/socks_request_st.h			\
 	src/core/or/status.h				\
@@ -263,12 +304,27 @@ noinst_HEADERS +=					\
 	src/feature/client/dnsserv.h			\
 	src/feature/client/entrynodes.h			\
 	src/feature/client/transports.h			\
+	src/feature/control/btrack_circuit.h		\
+	src/feature/control/btrack_orconn.h		\
+	src/feature/control/btrack_orconn_cevent.h	\
+	src/feature/control/btrack_orconn_maps.h	\
+	src/feature/control/btrack_sys.h		\
 	src/feature/control/control.h			\
+	src/feature/control/control_auth.h		\
+	src/feature/control/control_cmd.h		\
+	src/feature/control/control_cmd_args_st.h	\
 	src/feature/control/control_connection_st.h	\
+	src/feature/control/control_events.h		\
+	src/feature/control/control_fmt.h		\
+	src/feature/control/control_getinfo.h		\
+	src/feature/control/control_proto.h		\
 	src/feature/control/fmt_serverstatus.h		\
 	src/feature/control/getinfo_geoip.h		\
 	src/feature/dirauth/authmode.h			\
+        src/feature/dirauth/bridgeauth.h		\
 	src/feature/dirauth/bwauth.h			\
+	src/feature/dirauth/dirauth_periodic.h		\
+	src/feature/dirauth/dirauth_sys.h		\
 	src/feature/dirauth/dircollate.h		\
 	src/feature/dirauth/dirvote.h			\
 	src/feature/dirauth/dsigs_parse.h		\
@@ -317,6 +373,7 @@ noinst_HEADERS +=					\
 	src/feature/hs/hs_config.h			\
 	src/feature/hs/hs_control.h			\
 	src/feature/hs/hs_descriptor.h			\
+	src/feature/hs/hs_dos.h				\
 	src/feature/hs/hs_ident.h			\
 	src/feature/hs/hs_intropoint.h			\
 	src/feature/hs/hs_service.h			\
@@ -340,6 +397,8 @@ noinst_HEADERS +=					\
 	src/feature/nodelist/networkstatus_voter_info_st.h	\
 	src/feature/nodelist/nickname.h			\
 	src/feature/nodelist/node_st.h			\
+	src/feature/nodelist/nodefamily.h		\
+	src/feature/nodelist/nodefamily_st.h		\
 	src/feature/nodelist/nodelist.h			\
 	src/feature/nodelist/node_select.h		\
 	src/feature/nodelist/routerinfo.h		\
@@ -356,6 +415,8 @@ noinst_HEADERS +=					\
 	src/feature/relay/dns_structs.h			\
 	src/feature/relay/ext_orport.h			\
 	src/feature/relay/onion_queue.h			\
+	src/feature/relay/relay_periodic.h		\
+	src/feature/relay/relay_sys.h			\
 	src/feature/relay/router.h			\
 	src/feature/relay/routerkeys.h			\
 	src/feature/relay/routermode.h			\
@@ -374,9 +435,10 @@ noinst_HEADERS +=					\
 	src/feature/stats/rephist.h			\
 	src/feature/stats/predict_ports.h
 
-noinst_HEADERS +=			\
-	src/app/config/auth_dirs.inc	\
-	src/app/config/fallback_dirs.inc
+noinst_HEADERS +=				\
+	src/app/config/auth_dirs.inc		\
+	src/app/config/fallback_dirs.inc	\
+	src/app/config/testnet.inc
 
 # This may someday want to be an installed file?
 noinst_HEADERS += src/feature/api/tor_api.h
diff --git a/src/core/mainloop/.may_include b/src/core/mainloop/.may_include
new file mode 100644
index 0000000000..79d6a130a4
--- /dev/null
+++ b/src/core/mainloop/.may_include
@@ -0,0 +1,20 @@
+!advisory
+
+orconfig.h
+
+lib/container/*.h
+lib/dispatch/*.h
+lib/evloop/*.h
+lib/pubsub/*.h
+lib/subsys/*.h
+lib/buf/*.h
+lib/crypt_ops/*.h
+lib/err/*.h
+lib/tls/*.h
+lib/net/*.h
+lib/evloop/*.h
+lib/geoip/*.h
+lib/sandbox/*.h
+lib/compress/*.h
+
+core/mainloop/*.h
\ No newline at end of file
diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c
index 3595bba85c..50cd3810a4 100644
--- a/src/core/mainloop/connection.c
+++ b/src/core/mainloop/connection.c
@@ -57,7 +57,7 @@
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/err/backtrace.h"
 
@@ -82,12 +82,14 @@
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/dnsserv.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
@@ -696,6 +698,7 @@ connection_free_minimal(connection_t *conn)
     control_connection_t *control_conn = TO_CONTROL_CONN(conn);
     tor_free(control_conn->safecookie_client_hash);
     tor_free(control_conn->incoming_cmd);
+    tor_free(control_conn->current_cmd);
     if (control_conn->ephemeral_onion_services) {
       SMARTLIST_FOREACH(control_conn->ephemeral_onion_services, char *, cp, {
         memwipe(cp, 0, strlen(cp));
@@ -1195,7 +1198,7 @@ make_win32_socket_exclusive(tor_socket_t sock)
     return -1;
   }
   return 0;
-#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */
+#else /* !defined(SO_EXCLUSIVEADDRUSE) */
   (void) sock;
   return 0;
 #endif /* defined(SO_EXCLUSIVEADDRUSE) */
@@ -1466,6 +1469,20 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                tor_socket_strerror(tor_socket_errno(s)));
       goto err;
     }
+
+#ifndef __APPLE__
+    /* This code was introduced to help debug #28229. */
+    int value;
+    socklen_t len = sizeof(value);
+
+    if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) {
+      if (value == 0) {
+        log_err(LD_NET, "Could not listen on %s - "
+                        "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address);
+        goto err;
+      }
+    }
+#endif /* !defined(__APPLE__) */
 #endif /* defined(HAVE_SYS_UN_H) */
   } else {
     log_err(LD_BUG, "Got unexpected address family %d.",
@@ -1644,7 +1661,7 @@ check_sockaddr(const struct sockaddr *sa, int len, int level)
              len,(int)sizeof(struct sockaddr_in6));
       ok = 0;
     }
-    if (tor_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
+    if (fast_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
         sin6->sin6_port == 0) {
       log_fn(level, LD_NET,
              "Address for new connection has address/port equal to zero.");
@@ -1867,7 +1884,7 @@ connection_init_accepted_conn(connection_t *conn,
       /* Initiate Extended ORPort authentication. */
       return connection_ext_or_start_auth(TO_OR_CONN(conn));
     case CONN_TYPE_OR:
-      control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
+      connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
       rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1);
       if (rv < 0) {
         connection_or_close_for_error(TO_OR_CONN(conn), 0);
@@ -1880,6 +1897,9 @@ connection_init_accepted_conn(connection_t *conn,
       TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch();
       TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type;
 
+      /* Any incoming connection on an entry port counts as user activity. */
+      note_user_activity(approx_time());
+
       switch (TO_CONN(listener)->type) {
         case CONN_TYPE_AP_LISTENER:
           conn->state = AP_CONN_STATE_SOCKS_WAIT;
@@ -2079,6 +2099,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn)
     return;
   }
 
+  if (fascist_firewall_use_ipv6(options)) {
+    log_info(LD_NET, "Our outgoing connection is using IPv%d.",
+             tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4);
+  }
+
   /* Check if we couldn't satisfy an address family preference */
   if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6)
       || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) {
@@ -2837,7 +2862,7 @@ retry_listener_ports(smartlist_t *old_conns,
           SMARTLIST_DEL_CURRENT(old_conns, conn);
           break;
         }
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
       }
     } SMARTLIST_FOREACH_END(wanted);
 
@@ -2901,6 +2926,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
     retval = -1;
 
 #ifdef ENABLE_LISTENER_REBIND
+  if (smartlist_len(replacements))
+    log_debug(LD_NET, "%d replacements - starting rebinding loop.",
+              smartlist_len(replacements));
+
   SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) {
     int addr_in_use = 0;
     int skip = 0;
@@ -2912,8 +2941,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
       connection_listener_new_for_port(r->new_port, &skip, &addr_in_use);
     connection_t *old_conn = r->old_conn;
 
-    if (skip)
+    if (skip) {
+      log_debug(LD_NET, "Skipping creating new listener for %s:%d",
+                old_conn->address, old_conn->port);
       continue;
+    }
 
     connection_close_immediate(old_conn);
     connection_mark_for_close(old_conn);
@@ -2932,7 +2964,7 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
                conn_type_to_string(old_conn->type), old_conn->address,
                old_conn->port, new_conn->address, new_conn->port);
   } SMARTLIST_FOREACH_END(r);
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
 
   /* Any members that were still in 'listeners' don't correspond to
    * any configured port.  Kill 'em. */
@@ -3019,7 +3051,7 @@ connection_mark_all_noncontrol_connections(void)
  * uses pluggable transports, since we should then limit it even if it
  * comes from an internal IP address. */
 static int
-connection_is_rate_limited(connection_t *conn)
+connection_is_rate_limited(const connection_t *conn)
 {
   const or_options_t *options = get_options();
   if (conn->linked)
@@ -3154,14 +3186,14 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
                                      global_bucket_val, conn_bucket);
 }
 
-/** Return 1 if the global write buckets are low enough that we
+/** Return true iff the global write buckets are low enough that we
  * shouldn't send <b>attempt</b> bytes of low-priority directory stuff
- * out to <b>conn</b>. Else return 0.
-
- * Priority was 1 for v1 requests (directories and running-routers),
- * and 2 for v2 requests and later (statuses and descriptors).
+ * out to <b>conn</b>.
+ *
+ * If we are a directory authority, always answer dir requests thus true is
+ * always returned.
  *
- * There are a lot of parameters we could use here:
+ * Note: There are a lot of parameters we could use here:
  * - global_relayed_write_bucket. Low is bad.
  * - global_write_bucket. Low is bad.
  * - bandwidthrate. Low is bad.
@@ -3173,39 +3205,40 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
  *   mean is "total directory bytes added to outbufs recently", but
  *   that's harder to quantify and harder to keep track of.
  */
-int
-global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
+bool
+connection_dir_is_global_write_low(const connection_t *conn, size_t attempt)
 {
   size_t smaller_bucket =
     MIN(token_bucket_rw_get_write(&global_bucket),
         token_bucket_rw_get_write(&global_relayed_bucket));
-  if (authdir_mode(get_options()) && priority>1)
-    return 0; /* there's always room to answer v2 if we're an auth dir */
+
+  /* Special case for authorities (directory only). */
+  if (authdir_mode_v3(get_options())) {
+    /* Are we configured to possibly reject requests under load? */
+    if (!get_options()->AuthDirRejectRequestsUnderLoad) {
+      /* Answer request no matter what. */
+      return false;
+    }
+    /* Always answer requests from a known relay which includes the other
+     * authorities. The following looks up the addresses for relays that we
+     * have their descriptor _and_ any configured trusted directories. */
+    if (nodelist_probably_contains_address(&conn->addr)) {
+      return false;
+    }
+  }
 
   if (!connection_is_rate_limited(conn))
-    return 0; /* local conns don't get limited */
+    return false; /* local conns don't get limited */
 
   if (smaller_bucket < attempt)
-    return 1; /* not enough space no matter the priority */
+    return true; /* not enough space. */
 
   {
     const time_t diff = approx_time() - write_buckets_last_empty_at;
     if (diff <= 1)
-      return 1; /* we're already hitting our limits, no more please */
+      return true; /* we're already hitting our limits, no more please */
   }
-
-  if (priority == 1) { /* old-style v1 query */
-    /* Could we handle *two* of these requests within the next two seconds? */
-    const or_options_t *options = get_options();
-    size_t can_write = (size_t) (smaller_bucket
-      + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate :
-           options->BandwidthRate));
-    if (can_write < 2*attempt)
-      return 1;
-  } else { /* v2 query */
-    /* no further constraints yet */
-  }
-  return 0;
+  return false;
 }
 
 /** When did we last tell the accounting subsystem about transmitted
@@ -3931,9 +3964,9 @@ update_send_buffer_size(tor_socket_t sock)
       &isb, sizeof(isb), &bytesReturned, NULL, NULL)) {
     setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb));
   }
-#else
+#else /* !defined(_WIN32) */
   (void) sock;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Try to flush more bytes onto <b>conn</b>-\>s.
@@ -4331,6 +4364,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
   connection_write_to_buf_commit(conn, written);
 }
 
+/**
+ * Write a <b>string</b> (of size <b>len</b> to directory connection
+ * <b>dir_conn</b>. Apply compression if connection is configured to use
+ * it and finalize it if <b>done</b> is true.
+ */
+void
+connection_dir_buf_add(const char *string, size_t len,
+                       dir_connection_t *dir_conn, int done)
+{
+  if (dir_conn->compress_state != NULL) {
+    connection_buf_add_compress(string, len, dir_conn, done);
+    return;
+  }
+
+  connection_buf_add(string, len, TO_CONN(dir_conn));
+}
+
 void
 connection_buf_add_compress(const char *string, size_t len,
                             dir_connection_t *conn, int done)
@@ -4445,6 +4495,16 @@ connection_get_by_type_state(int type, int state)
   CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state);
 }
 
+/**
+ * Return a connection of type <b>type</b> that is not an internally linked
+ * connection, and is not marked for close.
+ **/
+MOCK_IMPL(connection_t *,
+connection_get_by_type_nonlinked,(int type))
+{
+  CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked);
+}
+
 /** Return a connection of type <b>type</b> that has rendquery equal
  * to <b>rendquery</b>, and that is not marked for close. If state
  * is non-zero, conn must be of that state too.
@@ -5287,7 +5347,7 @@ assert_connection_ok(connection_t *conn, time_t now)
         tor_assert(entry_conn->socks_request->has_finished);
         if (!conn->marked_for_close) {
           tor_assert(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
-          assert_cpath_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
+          cpath_assert_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
         }
       }
     }
@@ -5341,17 +5401,20 @@ assert_connection_ok(connection_t *conn, time_t now)
 }
 
 /** Fills <b>addr</b> and <b>port</b> with the details of the global
- *  proxy server we are using.
- *  <b>conn</b> contains the connection we are using the proxy for.
+ *  proxy server we are using. Store a 1 to the int pointed to by
+ *  <b>is_put_out</b> if the connection is using a pluggable
+ *  transport; store 0 otherwise. <b>conn</b> contains the connection
+ *  we are using the proxy for.
  *
  *  Return 0 on success, -1 on failure.
  */
 int
 get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                   const connection_t *conn)
+                   int *is_pt_out, const connection_t *conn)
 {
   const or_options_t *options = get_options();
 
+  *is_pt_out = 0;
   /* Client Transport Plugins can use another proxy, but that should be hidden
    * from the rest of tor (as the plugin is responsible for dealing with the
    * proxy), check it first, then check the rest of the proxy types to allow
@@ -5367,6 +5430,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
       tor_addr_copy(addr, &transport->addr);
       *port = transport->port;
       *proxy_type = transport->socks_version;
+      *is_pt_out = 1;
       return 0;
     }
 
@@ -5403,11 +5467,13 @@ log_failed_proxy_connection(connection_t *conn)
 {
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt;
 
-  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, conn) != 0)
+  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         conn) != 0)
     return; /* if we have no proxy set up, leave this function. */
 
+  (void)is_pt;
   log_warn(LD_NET,
            "The connection to the %s proxy server at %s just failed. "
            "Make sure that the proxy server is up and running.",
diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h
index 8ecdd6f06f..668c740042 100644
--- a/src/core/mainloop/connection.h
+++ b/src/core/mainloop/connection.h
@@ -187,7 +187,7 @@ int connection_proxy_connect(connection_t *conn, int type);
 int connection_read_proxy_handshake(connection_t *conn);
 void log_failed_proxy_connection(connection_t *conn);
 int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                       const connection_t *conn);
+                       int *is_pt_out, const connection_t *conn);
 
 int retry_all_listeners(smartlist_t *new_conns,
                         int close_all_noncontrol);
@@ -195,8 +195,9 @@ int retry_all_listeners(smartlist_t *new_conns,
 void connection_mark_all_noncontrol_listeners(void);
 void connection_mark_all_noncontrol_connections(void);
 
-ssize_t connection_bucket_write_limit(connection_t *conn, time_t now);
-int global_write_bucket_low(connection_t *conn, size_t attempt, int priority);
+ssize_t connection_bucket_write_limit(struct connection_t *conn, time_t now);
+bool connection_dir_is_global_write_low(const struct connection_t *conn,
+                                        size_t attempt);
 void connection_bucket_init(void);
 void connection_bucket_adjust(const or_options_t *options);
 void connection_bucket_refill_all(time_t now,
@@ -226,6 +227,8 @@ MOCK_DECL(void, connection_write_to_buf_impl_,
 /* DOCDOC connection_write_to_buf */
 static void connection_buf_add(const char *string, size_t len,
                                     connection_t *conn);
+void connection_dir_buf_add(const char *string, size_t len,
+                            dir_connection_t *dir_conn, int done);
 static inline void
 connection_buf_add(const char *string, size_t len, connection_t *conn)
 {
@@ -240,6 +243,7 @@ size_t connection_get_outbuf_len(connection_t *conn);
 connection_t *connection_get_by_global_id(uint64_t id);
 
 connection_t *connection_get_by_type(int type);
+MOCK_DECL(connection_t *,connection_get_by_type_nonlinked,(int type));
 MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type,
                                                   const tor_addr_t *addr,
                                                   uint16_t port, int purpose));
diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c
index e704d55642..436fcd28c3 100644
--- a/src/core/mainloop/cpuworker.c
+++ b/src/core/mainloop/cpuworker.c
@@ -34,7 +34,6 @@
 #include "core/crypto/onion_crypto.h"
 
 #include "core/or/or_circuit_st.h"
-#include "lib/intmath/weakrng.h"
 
 static void queue_pending_tasks(void);
 
@@ -74,8 +73,6 @@ worker_state_free_void(void *arg)
 static replyqueue_t *replyqueue = NULL;
 static threadpool_t *threadpool = NULL;
 
-static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT;
-
 static int total_pending_tasks = 0;
 static int max_pending_tasks = 128;
 
@@ -109,7 +106,6 @@ cpu_init(void)
 
   /* Total voodoo. Can we make this more sensible? */
   max_pending_tasks = get_num_cpus(get_options()) * 64;
-  crypto_seed_weak_rng(&request_sample_rng);
 }
 
 /** Magic numbers to make sure our cpuworker_requests don't grow any
@@ -235,9 +231,10 @@ should_time_request(uint16_t onionskin_type)
    * sample */
   if (onionskins_n_processed[onionskin_type] < 4096)
     return 1;
+
   /** Otherwise, measure with P=1/128.  We avoid doing this for every
    * handshake, since the measurement itself can take a little time. */
-  return tor_weak_random_one_in_n(&request_sample_rng, 128);
+  return crypto_fast_rng_one_in_n(get_thread_fast_rng(), 128);
 }
 
 /** Return an estimate of how many microseconds we will need for a single
diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c
index f0aa37e8da..c47e440774 100644
--- a/src/core/mainloop/mainloop.c
+++ b/src/core/mainloop/mainloop.c
@@ -73,8 +73,8 @@
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
-#include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
@@ -95,7 +95,7 @@
 #include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/err/backtrace.h"
 #include "lib/tls/buffers_tls.h"
@@ -105,9 +105,6 @@
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/authmode.h"
-
 #include "core/or/cell_st.h"
 #include "core/or/entry_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -200,12 +197,10 @@ static int can_complete_circuits = 0;
 #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60)
 
 static int conn_close_if_marked(int i);
-static int run_main_loop_until_done(void);
 static void connection_start_reading_from_linked_conn(connection_t *conn);
 static int connection_should_read_from_linked_conn(connection_t *conn);
 static void conn_read_callback(evutil_socket_t fd, short event, void *_conn);
 static void conn_write_callback(evutil_socket_t fd, short event, void *_conn);
-static void second_elapsed_callback(periodic_timer_t *timer, void *args);
 static void shutdown_did_not_work_callback(evutil_socket_t fd, short event,
                                            void *arg) ATTR_NORETURN;
 
@@ -759,7 +754,7 @@ tor_shutdown_event_loop_for_restart_cb(
   tor_event_free(tor_shutdown_event_loop_for_restart_event);
   tor_shutdown_event_loop_and_exit(0);
 }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
 /**
  * After finishing the current callback (if any), shut down the main loop,
@@ -1361,119 +1356,92 @@ static int periodic_events_initialized = 0;
 #define CALLBACK(name) \
   static int name ## _callback(time_t, const or_options_t *)
 CALLBACK(add_entropy);
-CALLBACK(check_authority_cert);
-CALLBACK(check_canonical_channels);
-CALLBACK(check_descriptor);
-CALLBACK(check_dns_honesty);
-CALLBACK(check_ed_keys);
 CALLBACK(check_expired_networkstatus);
-CALLBACK(check_for_reachability_bw);
-CALLBACK(check_onion_keys_expiry_time);
 CALLBACK(clean_caches);
 CALLBACK(clean_consdiffmgr);
-CALLBACK(dirvote);
-CALLBACK(downrate_stability);
-CALLBACK(expire_old_ciruits_serverside);
 CALLBACK(fetch_networkstatus);
 CALLBACK(heartbeat);
 CALLBACK(hs_service);
 CALLBACK(launch_descriptor_fetches);
-CALLBACK(launch_reachability_tests);
-CALLBACK(reachability_warnings);
+CALLBACK(prune_old_routers);
 CALLBACK(record_bridge_stats);
 CALLBACK(rend_cache_failure_clean);
 CALLBACK(reset_padding_counts);
-CALLBACK(retry_dns);
 CALLBACK(retry_listeners);
-CALLBACK(rotate_onion_key);
 CALLBACK(rotate_x509_certificate);
-CALLBACK(save_stability);
 CALLBACK(save_state);
-CALLBACK(write_bridge_ns);
 CALLBACK(write_stats_file);
+CALLBACK(control_per_second_events);
+CALLBACK(second_elapsed);
 
 #undef CALLBACK
 
 /* Now we declare an array of periodic_event_item_t for each periodic event */
-#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f)
-
-STATIC periodic_event_item_t periodic_events[] = {
-  /* Everyone needs to run those. */
-  CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0),
-
-  /* Routers (bridge and relay) only. */
-  CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0),
-
-  /* Authorities (bridge and directory) only. */
-  CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-  CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-
-  /* Directory authority only. */
-  CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0),
-  CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET),
-
-  /* Relay only. */
-  CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+#define CALLBACK(name, r, f) \
+  PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f)
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+
+STATIC periodic_event_item_t mainloop_periodic_events[] = {
+
+  /* Everyone needs to run these. They need to have very long timeouts for
+   * that to be safe. */
+  CALLBACK(add_entropy, ALL, 0),
+  CALLBACK(heartbeat, ALL, 0),
+  CALLBACK(reset_padding_counts, ALL, 0),
+
+  /* This is a legacy catch-all callback that runs once per second if
+   * we are online and active. */
+  CALLBACK(second_elapsed, NET_PARTICIPANT,
+           FL(RUN_ON_DISABLE)),
+
+  /* XXXX Do we have a reason to do this on a callback? Does it do any good at
+   * all?  For now, if we're dormant, we can let our listeners decay. */
+  CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)),
+
+  /* We need to do these if we're participating in the Tor network. */
+  CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)),
+  CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0),
+  CALLBACK(check_network_participation, NET_PARTICIPANT, 0),
+
+  /* We need to do these if we're participating in the Tor network, and
+   * immediately before we stop. */
+  CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Hidden Service service only. */
-  CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more
 
   /* Bridge only. */
-  CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0),
+  CALLBACK(record_bridge_stats, BRIDGE, 0),
 
   /* Client only. */
-  CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0),
-
-  /* Bridge Authority only. */
-  CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0),
+  /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */
+  CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Directory server only. */
-  CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0),
+  CALLBACK(clean_consdiffmgr, DIRSERVER, 0),
+
+  /* Controller with per-second events only. */
+  CALLBACK(control_per_second_events, CONTROLEV, 0),
 
   END_OF_PERIODIC_EVENTS
 };
 #undef CALLBACK
+#undef FL
 
 /* These are pointers to members of periodic_events[] that are used to
  * implement particular callbacks.  We keep them separate here so that we
  * can access them by name.  We also keep them inside periodic_events[]
  * so that we can implement "reset all timers" in a reasonable way. */
-static periodic_event_item_t *check_descriptor_event=NULL;
-static periodic_event_item_t *dirvote_event=NULL;
 static periodic_event_item_t *fetch_networkstatus_event=NULL;
 static periodic_event_item_t *launch_descriptor_fetches_event=NULL;
 static periodic_event_item_t *check_dns_honesty_event=NULL;
 static periodic_event_item_t *save_state_event=NULL;
+static periodic_event_item_t *prune_old_routers_event=NULL;
 
 /** Reset all the periodic events so we'll do all our actions again as if we
  * just started up.
@@ -1483,24 +1451,7 @@ static periodic_event_item_t *save_state_event=NULL;
 void
 reset_all_main_loop_timers(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_reschedule(&periodic_events[i]);
-  }
-}
-
-/** Return the member of periodic_events[] whose name is <b>name</b>.
- * Return NULL if no such event is found.
- */
-static periodic_event_item_t *
-find_periodic_event(const char *name)
-{
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    if (strcmp(name, periodic_events[i].name) == 0)
-      return &periodic_events[i];
-  }
-  return NULL;
+  periodic_events_reset_all();
 }
 
 /** Return a bitmask of the roles this tor instance is configured for using
@@ -1510,7 +1461,7 @@ get_my_roles(const or_options_t *options)
 {
   tor_assert(options);
 
-  int roles = 0;
+  int roles = PERIODIC_EVENT_ROLE_ALL;
   int is_bridge = options->BridgeRelay;
   int is_relay = server_mode(options);
   int is_dirauth = authdir_mode_v3(options);
@@ -1518,6 +1469,8 @@ get_my_roles(const or_options_t *options)
   int is_hidden_service = !!hs_service_get_num_services() ||
                           !!rend_num_services();
   int is_dirserver = dir_server_mode(options);
+  int sending_control_events = control_any_per_second_event_enabled();
+
   /* We also consider tor to have the role of a client if the ControlPort is
    * set because a lot of things can be done over the control port which
    * requires tor to have basic functionnalities. */
@@ -1525,6 +1478,9 @@ get_my_roles(const or_options_t *options)
                   options->ControlPort_set ||
                   options->OwningControllerFD != UINT64_MAX;
 
+  int is_net_participant = is_participating_on_network() ||
+    is_relay || is_hidden_service;
+
   if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE;
   if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT;
   if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY;
@@ -1532,6 +1488,8 @@ get_my_roles(const or_options_t *options)
   if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH;
   if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE;
   if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER;
+  if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV;
 
   return roles;
 }
@@ -1556,9 +1514,9 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data)
   rescan_periodic_events(get_options());
 }
 
-/** Set up all the members of periodic_events[], and configure them all to be
- * launched from a callback. */
-STATIC void
+/** Set up all the members of mainloop_periodic_events[], and configure them
+ * all to be launched from a callback. */
+void
 initialize_periodic_events(void)
 {
   if (periodic_events_initialized)
@@ -1566,39 +1524,58 @@ initialize_periodic_events(void)
 
   periodic_events_initialized = 1;
 
-  /* Set up all periodic events. We'll launch them by roles. */
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_setup(&periodic_events[i]);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_events_register(&mainloop_periodic_events[i]);
   }
 
+  /* Set up all periodic events. We'll launch them by roles. */
+
 #define NAMED_CALLBACK(name) \
-  STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END
+  STMT_BEGIN name ## _event = periodic_events_find( #name ); STMT_END
 
-  NAMED_CALLBACK(check_descriptor);
-  NAMED_CALLBACK(dirvote);
+  NAMED_CALLBACK(prune_old_routers);
   NAMED_CALLBACK(fetch_networkstatus);
   NAMED_CALLBACK(launch_descriptor_fetches);
   NAMED_CALLBACK(check_dns_honesty);
   NAMED_CALLBACK(save_state);
-
-  struct timeval one_second = { 1, 0 };
-  initialize_periodic_events_event = tor_evtimer_new(
-                  tor_libevent_get_base(),
-                  initialize_periodic_events_cb, NULL);
-  event_add(initialize_periodic_events_event, &one_second);
 }
 
 STATIC void
 teardown_periodic_events(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_destroy(&periodic_events[i]);
-  }
+  periodic_events_disconnect_all();
+  fetch_networkstatus_event = NULL;
+  launch_descriptor_fetches_event = NULL;
+  check_dns_honesty_event = NULL;
+  save_state_event = NULL;
+  prune_old_routers_event = NULL;
   periodic_events_initialized = 0;
 }
 
+static mainloop_event_t *rescan_periodic_events_ev = NULL;
+
+/** Callback: rescan the periodic event list. */
+static void
+rescan_periodic_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+  rescan_periodic_events(get_options());
+}
+
+/**
+ * Schedule an event that will rescan which periodic events should run.
+ **/
+MOCK_IMPL(void,
+schedule_rescan_periodic_events,(void))
+{
+  if (!rescan_periodic_events_ev) {
+    rescan_periodic_events_ev =
+      mainloop_event_new(rescan_periodic_events_cb, NULL);
+  }
+  mainloop_event_activate(rescan_periodic_events_ev);
+}
+
 /** Do a pass at all our periodic events, disable those we don't need anymore
  * and enable those we need now using the given options. */
 void
@@ -1606,36 +1583,7 @@ rescan_periodic_events(const or_options_t *options)
 {
   tor_assert(options);
 
-  /* Avoid scanning the event list if we haven't initialized it yet. This is
-   * particularly useful for unit tests in order to avoid initializing main
-   * loop events everytime. */
-  if (!periodic_events_initialized) {
-    return;
-  }
-
-  int roles = get_my_roles(options);
-
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-
-    int enable = !!(item->roles & roles);
-
-    /* Handle the event flags. */
-    if (net_is_disabled() &&
-        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
-      enable = 0;
-    }
-
-    /* Enable the event if needed. It is safe to enable an event that was
-     * already enabled. Same goes for disabling it. */
-    if (enable) {
-      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
-      periodic_event_enable(item);
-    } else {
-      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
-      periodic_event_disable(item);
-    }
-  }
+  periodic_events_rescan_by_roles(get_my_roles(options), net_is_disabled());
 }
 
 /* We just got new options globally set, see if we need to enabled or disable
@@ -1643,26 +1591,7 @@ rescan_periodic_events(const or_options_t *options)
 void
 periodic_events_on_new_options(const or_options_t *options)
 {
-  /* Only if we've already initialized the events, rescan the list which will
-   * enable or disable events depending on our roles. This will be called at
-   * bootup and we don't want this function to initialize the events because
-   * they aren't set up at this stage. */
-  if (periodic_events_initialized) {
-    rescan_periodic_events(options);
-  }
-}
-
-/**
- * Update our schedule so that we'll check whether we need to update our
- * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
- * seconds.
- */
-void
-reschedule_descriptor_update_check(void)
-{
-  if (check_descriptor_event) {
-    periodic_event_reschedule(check_descriptor_event);
-  }
+  rescan_periodic_events(options);
 }
 
 /**
@@ -1708,40 +1637,41 @@ mainloop_schedule_postloop_cleanup(void)
   mainloop_event_activate(postloop_cleanup_ev);
 }
 
-#define LONGEST_TIMER_PERIOD (30 * 86400)
-/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
- * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */
-static inline int
-safe_timer_diff(time_t now, time_t next)
-{
-  if (next > now) {
-    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
-     * and nothing that could run Tor. It's a bug if 'next' is around then.
-     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
-     * Bang. We cannot extrapolate past a singularity, but there was probably
-     * nothing that could run Tor then, either.
-     **/
-    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
-
-    if (next - LONGEST_TIMER_PERIOD > now)
-      return LONGEST_TIMER_PERIOD;
-    return (int)(next - now);
-  } else {
-    return 1;
+/** Event to run 'scheduled_shutdown_cb' */
+static mainloop_event_t *scheduled_shutdown_ev=NULL;
+
+/** Callback: run a scheduled shutdown */
+static void
+scheduled_shutdown_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+  tor_shutdown_event_loop_and_exit(0);
+}
+
+/** Schedule the mainloop to exit after <b>delay_sec</b> seconds. */
+void
+mainloop_schedule_shutdown(int delay_sec)
+{
+  const struct timeval delay_tv = { delay_sec, 0 };
+  if (! scheduled_shutdown_ev) {
+    scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL);
   }
+  mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv);
 }
 
 /** Perform regular maintenance tasks.  This function gets run once per
- * second by second_elapsed_callback().
+ * second.
  */
-static void
-run_scheduled_events(time_t now)
+static int
+second_elapsed_callback(time_t now, const or_options_t *options)
 {
-  const or_options_t *options = get_options();
-
-  /* 0. See if we've been asked to shut down and our timeout has
-   * expired; or if our bandwidth limits are exhausted and we
-   * should hibernate; or if it's time to wake up from hibernation.
+  /* 0. See if our bandwidth limits are exhausted and we should hibernate
+   *
+   * Note: we have redundant mechanisms to handle the case where it's
+   * time to wake up from hibernation; or where we have a scheduled
+   * shutdown and it's time to run it, but this will also handle those.
    */
   consider_hibernation(now);
 
@@ -1751,10 +1681,13 @@ run_scheduled_events(time_t now)
   if (options->UseBridges && !net_is_disabled()) {
     /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches()
      * -- the latter is only for fetching consensus-derived directory info. */
+    // TODO: client
+    //     Also, schedule this rather than probing 1x / sec
     fetch_bridge_descriptors(options, now);
   }
 
   if (accounting_is_enabled(options)) {
+    // TODO: refactor or rewrite?
     accounting_run_housekeeping(now);
   }
 
@@ -1765,6 +1698,7 @@ run_scheduled_events(time_t now)
    */
   /* (If our circuit build timeout can ever become lower than a second (which
    * it can't, currently), we should do this more often.) */
+  // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE
   circuit_expire_building();
   circuit_expire_waiting_for_better_guard();
 
@@ -1798,80 +1732,8 @@ run_scheduled_events(time_t now)
     run_connection_housekeeping(i, now);
   }
 
-  /* 11b. check pending unconfigured managed proxies */
-  if (!net_is_disabled() && pt_proxies_configuration_pending())
-    pt_configure_remaining_proxies();
-}
-
-/* Periodic callback: rotate the onion keys after the period defined by the
- * "onion-key-rotation-days" consensus parameter, shut down and restart all
- * cpuworkers, and update our descriptor if necessary.
- */
-static int
-rotate_onion_key_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_lifetime = get_onion_key_lifetime();
-    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
-    if (rotation_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL,"Rotating onion key.");
-    rotate_onion_key();
-    cpuworkers_rotate_keyinfo();
-    if (router_rebuild_descriptor(1)<0) {
-      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
-    }
-    if (advertised_server_mode() && !net_is_disabled())
-      router_upload_dir_desc_to_dirservers(0);
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Period callback: Check if our old onion keys are still valid after the
- * period of time defined by the consensus parameter
- * "onion-key-grace-period-days", otherwise expire them by setting them to
- * NULL.
- */
-static int
-check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_grace_period = get_onion_key_grace_period();
-    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
-    if (expiry_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL, "Expiring old onion keys.");
-    expire_old_onion_keys();
-    cpuworkers_rotate_keyinfo();
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Periodic callback: Every 30 seconds, check whether it's time to make new
- * Ed25519 subkeys.
- */
-static int
-check_ed_keys_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    if (should_make_new_ed_keys(options, now)) {
-      int new_signing_key = load_ed_keys(options, now);
-      if (new_signing_key < 0 ||
-          generate_ed_link_cert(options, now, new_signing_key > 0)) {
-        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
-        tor_shutdown_event_loop_and_exit(1);
-      }
-    }
-    return 30;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
+  /* Run again in a second. */
+  return 1;
 }
 
 /**
@@ -1947,100 +1809,53 @@ add_entropy_callback(time_t now, const or_options_t *options)
   return ENTROPY_INTERVAL;
 }
 
-/**
- * Periodic callback: if we're an authority, make sure we test
- * the routers on the network for reachability.
- */
-static int
-launch_reachability_tests_callback(time_t now, const or_options_t *options)
+/** Periodic callback: if there has been no network usage in a while,
+ * enter a dormant state. */
+STATIC int
+check_network_participation_callback(time_t now, const or_options_t *options)
 {
-  if (authdir_mode_tests_reachability(options) &&
-      !net_is_disabled()) {
-    /* try to determine reachability of the other Tor relays */
-    dirserv_test_reachability(now);
+  /* If we're a server, we can't become dormant. */
+  if (server_mode(options)) {
+    goto found_activity;
   }
-  return REACHABILITY_TEST_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, discount the stability
- * information (and other rephist information) that's older.
- */
-static int
-downrate_stability_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* 1d. Periodically, we discount older stability information so that new
-   * stability info counts more, and save the stability information to disk as
-   * appropriate. */
-  time_t next = rep_hist_downrate_old_runs(now);
-  return safe_timer_diff(now, next);
-}
+  /* If we're running an onion service, we can't become dormant. */
+  /* XXXX this would be nice to change, so that we can be dormant with a
+   * service. */
+  if (hs_service_get_num_services() || rend_num_services()) {
+    goto found_activity;
+  }
 
-/**
- * Periodic callback: if we're an authority, record our measured stability
- * information from rephist in an mtbf file.
- */
-static int
-save_stability_callback(time_t now, const or_options_t *options)
-{
-  if (authdir_mode_tests_reachability(options)) {
-    if (rep_hist_record_mtbf_data(now, 1)<0) {
-      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+  /* If we have any currently open entry streams other than "linked"
+   * connections used for directory requests, those count as user activity.
+   */
+  if (options->DormantTimeoutDisabledByIdleStreams) {
+    if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) {
+      goto found_activity;
     }
   }
-#define SAVE_STABILITY_INTERVAL (30*60)
-  return SAVE_STABILITY_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, check on our authority
- * certificate (the one that authenticates our authority signing key).
- */
-static int
-check_authority_cert_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  (void)options;
-  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
-   * close to expiring and warn the admin if it is. */
-  v3_authority_check_key_expiry();
-#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
-  return CHECK_V3_CERTIFICATE_INTERVAL;
-}
+  /* XXXX Make this configurable? */
+/** How often do we check whether we have had network activity? */
+#define CHECK_PARTICIPATION_INTERVAL (5*60)
 
-/**
- * Scheduled callback: Run directory-authority voting functionality.
- *
- * The schedule is a bit complicated here, so dirvote_act() manages the
- * schedule itself.
- **/
-static int
-dirvote_callback(time_t now, const or_options_t *options)
-{
-  if (!authdir_mode_v3(options)) {
-    tor_assert_nonfatal_unreached();
-    return 3600;
+  /* Become dormant if there has been no user activity in a long time.
+   * (The funny checks below are in order to prevent overflow.) */
+  time_t time_since_last_activity = 0;
+  if (get_last_user_activity_time() < now)
+    time_since_last_activity = now - get_last_user_activity_time();
+  if (time_since_last_activity >= options->DormantClientTimeout) {
+    log_notice(LD_GENERAL, "No user activity in a long time: becoming"
+               " dormant.");
+    set_network_participation(false);
+    rescan_periodic_events(options);
   }
 
-  time_t next = dirvote_act(options, now);
-  if (BUG(next == TIME_MAX)) {
-    /* This shouldn't be returned unless we called dirvote_act() without
-     * being an authority.  If it happens, maybe our configuration will
-     * fix itself in an hour or so? */
-    return 3600;
-  }
-  return safe_timer_diff(now, next);
-}
+  return CHECK_PARTICIPATION_INTERVAL;
 
-/** Reschedule the directory-authority voting event.  Run this whenever the
- * schedule has changed. */
-void
-reschedule_dirvote(const or_options_t *options)
-{
-  if (periodic_events_initialized && authdir_mode_v3(options)) {
-    periodic_event_reschedule(dirvote_event);
-  }
+ found_activity:
+  note_user_activity(now);
+  return CHECK_PARTICIPATION_INTERVAL;
 }
 
 /**
@@ -2053,11 +1868,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   (void)options;
   /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
-  /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
-   * networkstatus_get_reasonably_live_consensus(), but that value is way
-   * way too high.  Arma: is the bridge issue there resolved yet? -NM */
-#define NS_EXPIRY_SLOP (24*60*60)
-  if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) &&
+  /* Use reasonably live consensuses until they are no longer reasonably live.
+   */
+  if (ns && !networkstatus_consensus_reasonably_live(ns, now) &&
       router_have_minimum_dir_info()) {
     router_dir_info_changed();
   }
@@ -2143,17 +1956,6 @@ write_stats_file_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next_time_to_write_stats_files);
 }
 
-#define CHANNEL_CHECK_INTERVAL (60*60)
-static int
-check_canonical_channels_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  if (public_server_mode(options))
-    channel_check_for_duplicates();
-
-  return CHANNEL_CHECK_INTERVAL;
-}
-
 static int
 reset_padding_counts_callback(time_t now, const or_options_t *options)
 {
@@ -2228,87 +2030,24 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
 }
 
 /**
- * Periodic callback: If we're a server and initializing dns failed, retry.
+ * Periodic callback: prune routerlist of old information about Tor network.
  */
 static int
-retry_dns_callback(time_t now, const or_options_t *options)
+prune_old_routers_callback(time_t now, const or_options_t *options)
 {
+#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour.
   (void)now;
-#define RETRY_DNS_INTERVAL (10*60)
-  if (server_mode(options) && has_dns_init_failed())
-    dns_init();
-  return RETRY_DNS_INTERVAL;
-}
-
-/** Periodic callback: consider rebuilding or and re-uploading our descriptor
- * (if we've passed our internal checks). */
-static int
-check_descriptor_callback(time_t now, const or_options_t *options)
-{
-/** How often do we check whether part of our router info has changed in a
- * way that would require an upload? That includes checking whether our IP
- * address has changed. */
-#define CHECK_DESCRIPTOR_INTERVAL (60)
-
   (void)options;
 
-  /* 2b. Once per minute, regenerate and upload the descriptor if the old
-   * one is inaccurate. */
   if (!net_is_disabled()) {
-    check_descriptor_bandwidth_changed(now);
-    check_descriptor_ipaddress_changed(now);
-    mark_my_descriptor_dirty_if_too_old(now);
-    consider_publishable_server(0);
     /* If any networkstatus documents are no longer recent, we need to
      * update all the descriptors' running status. */
     /* Remove dead routers. */
-    /* XXXX This doesn't belong here, but it was here in the pre-
-     * XXXX refactoring code. */
+    log_debug(LD_GENERAL, "Pruning routerlist...");
     routerlist_remove_old_routers();
   }
 
-  return CHECK_DESCRIPTOR_INTERVAL;
-}
-
-/**
- * Periodic callback: check whether we're reachable (as a relay), and
- * whether our bandwidth has changed enough that we need to
- * publish a new descriptor.
- */
-static int
-check_for_reachability_bw_callback(time_t now, const or_options_t *options)
-{
-  /* XXXX This whole thing was stuck in the middle of what is now
-   * XXXX check_descriptor_callback.  I'm not sure it's right. */
-
-  static int dirport_reachability_count = 0;
-  /* also, check religiously for reachability, if it's within the first
-   * 20 minutes of our uptime. */
-  if (server_mode(options) &&
-      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
-      !net_is_disabled()) {
-    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-      router_do_reachability_checks(1, dirport_reachability_count==0);
-      if (++dirport_reachability_count > 5)
-        dirport_reachability_count = 0;
-      return 1;
-    } else {
-      /* If we haven't checked for 12 hours and our bandwidth estimate is
-       * low, do another bandwidth test. This is especially important for
-       * bridges, since they might go long periods without much use. */
-      const routerinfo_t *me = router_get_my_routerinfo();
-      static int first_time = 1;
-      if (!first_time && me &&
-          me->bandwidthcapacity < me->bandwidthrate &&
-          me->bandwidthcapacity < 51200) {
-        reset_bandwidth_test();
-      }
-      first_time = 0;
-#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
-      return BANDWIDTH_RECHECK_INTERVAL;
-    }
-  }
-  return CHECK_DESCRIPTOR_INTERVAL;
+  return ROUTERLIST_PRUNING_INTERVAL;
 }
 
 /**
@@ -2353,109 +2092,6 @@ retry_listeners_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
-/**
- * Periodic callback: as a server, see if we have any old unused circuits
- * that should be expired */
-static int
-expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* every 11 seconds, so not usually the same second as other such events */
-  circuit_expire_old_circuits_serverside(now);
-  return 11;
-}
-
-/**
- * Callback: Send warnings if Tor doesn't find its ports reachable.
- */
-static int
-reachability_warnings_callback(time_t now, const or_options_t *options)
-{
-  (void) now;
-
-  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
-  }
-
-  if (server_mode(options) &&
-      !net_is_disabled() &&
-      have_completed_a_circuit()) {
-    /* every 20 minutes, check and complain if necessary */
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (me && !check_whether_orport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
-               "its ORPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->or_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
-                                  address, me->or_port);
-      tor_free(address);
-    }
-
-    if (me && !check_whether_dirport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,
-               "Your server (%s:%d) has not managed to confirm that its "
-               "DirPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->dir_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
-                                  address, me->dir_port);
-      tor_free(address);
-    }
-  }
-
-  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
-}
-
-static int dns_honesty_first_time = 1;
-
-/**
- * Periodic event: if we're an exit, see if our DNS server is telling us
- * obvious lies.
- */
-static int
-check_dns_honesty_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  /* 9. and if we're an exit node, check whether our DNS is telling stories
-   * to us. */
-  if (net_is_disabled() ||
-      ! public_server_mode(options) ||
-      router_my_exit_policy_is_reject_star())
-    return PERIODIC_EVENT_NO_UPDATE;
-
-  if (dns_honesty_first_time) {
-    /* Don't launch right when we start */
-    dns_honesty_first_time = 0;
-    return crypto_rand_int_range(60, 180);
-  }
-
-  dns_launch_correctness_checks();
-  return 12*3600 + crypto_rand_int(12*3600);
-}
-
-/**
- * Periodic callback: if we're the bridge authority, write a networkstatus
- * file to disk.
- */
-static int
-write_bridge_ns_callback(time_t now, const or_options_t *options)
-{
-  /* 10. write bridge networkstatus file to disk */
-  if (options->BridgeAuthoritativeDir) {
-    networkstatus_dump_bridge_status_to_file(now);
-#define BRIDGE_STATUSFILE_INTERVAL (30*60)
-     return BRIDGE_STATUSFILE_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
 static int heartbeat_callback_first_time = 1;
 
 /**
@@ -2522,36 +2158,19 @@ hs_service_callback(time_t now, const or_options_t *options)
   return 1;
 }
 
-/** Timer: used to invoke second_elapsed_callback() once per second. */
-static periodic_timer_t *second_timer = NULL;
-
-/**
- * Enable or disable the per-second timer as appropriate, creating it if
- * necessary.
+/*
+ * Periodic callback: Send once-per-second events to the controller(s).
+ * This is called every second.
  */
-void
-reschedule_per_second_timer(void)
+static int
+control_per_second_events_callback(time_t now, const or_options_t *options)
 {
-  struct timeval one_second;
-  one_second.tv_sec = 1;
-  one_second.tv_usec = 0;
-
-  if (! second_timer) {
-    second_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &one_second,
-                                      second_elapsed_callback,
-                                      NULL);
-    tor_assert(second_timer);
-  }
+  (void) options;
+  (void) now;
 
-  const bool run_per_second_events =
-    control_any_per_second_event_enabled() || ! net_is_completely_disabled();
+  control_per_second_events();
 
-  if (run_per_second_events) {
-    periodic_timer_launch(second_timer, &one_second);
-  } else {
-    periodic_timer_disable(second_timer);
-  }
+  return 1;
 }
 
 /** Last time that update_current_time was called. */
@@ -2581,6 +2200,17 @@ update_current_time(time_t now)
   memcpy(&last_updated, &current_second_last_changed, sizeof(last_updated));
   monotime_coarse_get(&current_second_last_changed);
 
+  /** How much clock jumping means that we should adjust our idea of when
+   * to go dormant? */
+#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20
+
+  /* Don't go dormant early or late just because we jumped in time. */
+  if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) {
+    if (is_participating_on_network()) {
+      netstatus_note_clock_jumped(seconds_elapsed);
+    }
+  }
+
   /** How much clock jumping do we tolerate? */
 #define NUM_JUMPED_SECONDS_BEFORE_WARN 100
 
@@ -2590,6 +2220,7 @@ update_current_time(time_t now)
   if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) {
     // moving back in time is always a bad sign.
     circuit_note_clock_jumped(seconds_elapsed, false);
+
   } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
     /* Compare the monotonic clock to the result of time(). */
     const int32_t monotime_msec_passed =
@@ -2619,31 +2250,6 @@ update_current_time(time_t now)
   current_second = now;
 }
 
-/** Libevent callback: invoked once every second. */
-static void
-second_elapsed_callback(periodic_timer_t *timer, void *arg)
-{
-  /* XXXX This could be sensibly refactored into multiple callbacks, and we
-   * could use Libevent's timers for this rather than checking the current
-   * time against a bunch of timeouts every second. */
-  time_t now;
-  (void)timer;
-  (void)arg;
-
-  now = time(NULL);
-
-  /* We don't need to do this once-per-second any more: time-updating is
-   * only in this callback _because it is a callback_. It should be fine
-   * to disable this callback, and the time will still get updated.
-   */
-  update_current_time(now);
-
-  /* Maybe some controller events are ready to fire */
-  control_per_second_events();
-
-  run_scheduled_events(now);
-}
-
 #ifdef HAVE_SYSTEMD_209
 static periodic_timer_t *systemd_watchdog_timer = NULL;
 
@@ -2701,8 +2307,7 @@ dns_servers_relaunch_checks(void)
 {
   if (server_mode(get_options())) {
     dns_reset_correctness_checks();
-    if (periodic_events_initialized) {
-      tor_assert(check_dns_honesty_event);
+    if (check_dns_honesty_event) {
       periodic_event_reschedule(check_dns_honesty_event);
     }
   }
@@ -2712,8 +2317,6 @@ dns_servers_relaunch_checks(void)
 void
 initialize_mainloop_events(void)
 {
-  initialize_periodic_events();
-
   if (!schedule_active_linked_connections_event) {
     schedule_active_linked_connections_event =
       mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL);
@@ -2731,11 +2334,16 @@ do_main_loop(void)
   /* initialize the periodic events first, so that code that depends on the
    * events being present does not assert.
    */
-  initialize_periodic_events();
+  tor_assert(periodic_events_initialized);
   initialize_mainloop_events();
 
-  /* set up once-a-second callback. */
-  reschedule_per_second_timer();
+  periodic_events_connect_all();
+
+  struct timeval one_second = { 1, 0 };
+  initialize_periodic_events_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  initialize_periodic_events_cb, NULL);
+  event_add(initialize_periodic_events_event, &one_second);
 
 #ifdef HAVE_SYSTEMD_209
   uint64_t watchdog_delay;
@@ -2758,10 +2366,6 @@ do_main_loop(void)
     }
   }
 #endif /* defined(HAVE_SYSTEMD_209) */
-
-  main_loop_should_exit = 0;
-  main_loop_exit_value = 0;
-
 #ifdef ENABLE_RESTART_DEBUGGING
   {
     static int first_time = 1;
@@ -2785,7 +2389,7 @@ do_main_loop(void)
       event_add(tor_shutdown_event_loop_for_restart_event, &restart_after);
     }
   }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
   return run_main_loop_until_done();
 }
@@ -2887,10 +2491,14 @@ run_main_loop_once(void)
  *
  * Shadow won't invoke this function, so don't fill it up with things.
  */
-static int
+STATIC int
 run_main_loop_until_done(void)
 {
   int loop_result = 1;
+
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
+
   do {
     loop_result = run_main_loop_once();
   } while (loop_result == 1);
@@ -2921,7 +2529,6 @@ tor_mainloop_free_all(void)
   smartlist_free(connection_array);
   smartlist_free(closeable_connection_lst);
   smartlist_free(active_linked_connection_lst);
-  periodic_timer_free(second_timer);
   teardown_periodic_events();
   tor_event_free(shutdown_did_not_work_event);
   tor_event_free(initialize_periodic_events_event);
@@ -2929,6 +2536,8 @@ tor_mainloop_free_all(void)
   mainloop_event_free(schedule_active_linked_connections_event);
   mainloop_event_free(postloop_cleanup_ev);
   mainloop_event_free(handle_deferred_signewnym_ev);
+  mainloop_event_free(scheduled_shutdown_ev);
+  mainloop_event_free(rescan_periodic_events_ev);
 
 #ifdef HAVE_SYSTEMD_209
   periodic_timer_free(systemd_watchdog_timer);
@@ -2948,7 +2557,6 @@ tor_mainloop_free_all(void)
   can_complete_circuits = 0;
   quiet_level = 0;
   should_init_bridge_stats = 1;
-  dns_honesty_first_time = 1;
   heartbeat_callback_first_time = 1;
   current_second = 0;
   memset(&current_second_last_changed, 0,
diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h
index c5669fc4e0..caef736c15 100644
--- a/src/core/mainloop/mainloop.h
+++ b/src/core/mainloop/mainloop.h
@@ -59,12 +59,11 @@ void directory_info_has_arrived(time_t now, int from_cache, int suppress_logs);
 void ip_address_changed(int at_interface);
 void dns_servers_relaunch_checks(void);
 void reset_all_main_loop_timers(void);
-void reschedule_descriptor_update_check(void);
 void reschedule_directory_downloads(void);
 void reschedule_or_state_save(void);
-void reschedule_dirvote(const or_options_t *options);
 void mainloop_schedule_postloop_cleanup(void);
 void rescan_periodic_events(const or_options_t *options);
+MOCK_DECL(void, schedule_rescan_periodic_events,(void));
 
 void update_current_time(time_t now);
 
@@ -81,13 +80,15 @@ uint64_t get_main_loop_error_count(void);
 uint64_t get_main_loop_idle_count(void);
 
 void periodic_events_on_new_options(const or_options_t *options);
-void reschedule_per_second_timer(void);
 
 void do_signewnym(time_t);
 time_t get_last_signewnym_time(void);
 
+void mainloop_schedule_shutdown(int delay_sec);
+
 void tor_init_connection_lists(void);
 void initialize_mainloop_events(void);
+void initialize_periodic_events(void);
 void tor_mainloop_free_all(void);
 
 struct token_bucket_rw_t;
@@ -98,17 +99,20 @@ extern struct token_bucket_rw_t global_bucket;
 extern struct token_bucket_rw_t global_relayed_bucket;
 
 #ifdef MAINLOOP_PRIVATE
+STATIC int run_main_loop_until_done(void);
 STATIC void close_closeable_connections(void);
-STATIC void initialize_periodic_events(void);
 STATIC void teardown_periodic_events(void);
 STATIC int get_my_roles(const or_options_t *);
+STATIC int check_network_participation_callback(time_t now,
+                                                const or_options_t *options);
+
 #ifdef TOR_UNIT_TESTS
 extern smartlist_t *connection_array;
 
 /* We need the periodic_event_item_t definition. */
 #include "core/mainloop/periodic.h"
-extern periodic_event_item_t periodic_events[];
-#endif
-#endif /* defined(MAIN_PRIVATE) */
+extern periodic_event_item_t mainloop_periodic_events[];
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(MAINLOOP_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_MAINLOOP_H) */
diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c
new file mode 100644
index 0000000000..53275d8119
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.c
@@ -0,0 +1,170 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/pubsub/pubsub_build.h"
+
+/**
+ * Dispatcher to use for delivering messages.
+ **/
+static dispatch_t *the_dispatcher = NULL;
+static pubsub_items_t *the_pubsub_items = NULL;
+/**
+ * A list of mainloop_event_t, indexed by channel ID, to flush the messages
+ * on a channel.
+ **/
+static smartlist_t *alert_events = NULL;
+
+/**
+ * Mainloop event callback: flush all the messages in a channel.
+ *
+ * The channel is encoded as a pointer, and passed via arg.
+ **/
+static void
+flush_channel_event(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  if (!the_dispatcher)
+    return;
+
+  channel_id_t chan = (channel_id_t)(uintptr_t)(arg);
+  dispatch_flush(the_dispatcher, chan, INT_MAX);
+}
+
+/**
+ * Construct our global pubsub object from <b>builder</b>. Return 0 on
+ * success, -1 on failure. */
+int
+tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder)
+{
+  int rv = -1;
+  tor_mainloop_disconnect_pubsub();
+
+  the_dispatcher = pubsub_builder_finalize(builder, &the_pubsub_items);
+  if (! the_dispatcher)
+    goto err;
+
+  rv = 0;
+  goto done;
+ err:
+  tor_mainloop_disconnect_pubsub();
+ done:
+  return rv;
+}
+
+/**
+ * Install libevent events for all of the pubsub channels.
+ *
+ * Invoke this after tor_mainloop_connect_pubsub, and after libevent has been
+ * initialized.
+ */
+void
+tor_mainloop_connect_pubsub_events(void)
+{
+  tor_assert(the_dispatcher);
+  tor_assert(! alert_events);
+
+  const size_t num_channels = get_num_channel_ids();
+  alert_events = smartlist_new();
+  for (size_t i = 0; i < num_channels; ++i) {
+    smartlist_add(alert_events,
+                  mainloop_event_postloop_new(flush_channel_event,
+                                              (void*)(uintptr_t)(i)));
+  }
+}
+
+/**
+ * Dispatch alertfn callback: do nothing. Implements DELIV_NEVER.
+ **/
+static void
+alertfn_never(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  (void)arg;
+}
+
+/**
+ * Dispatch alertfn callback: activate a mainloop event. Implements
+ * DELIV_PROMPT.
+ **/
+static void
+alertfn_prompt(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  mainloop_event_t *event = arg;
+  mainloop_event_activate(event);
+}
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Set the strategy to be used for delivering messages on the named channel.
+ *
+ * This function needs to be called once globally for each channel, to
+ * set up how messages are delivered.
+ **/
+int
+tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                   deliv_strategy_t strategy)
+{
+  channel_id_t chan = get_channel_id(msg_channel_name);
+  if (BUG(chan == ERROR_ID) ||
+      BUG(chan >= smartlist_len(alert_events)))
+    return -1;
+
+  switch (strategy) {
+    case DELIV_NEVER:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_never, NULL);
+      break;
+    case DELIV_PROMPT:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_prompt,
+                            smartlist_get(alert_events, chan));
+      break;
+    case DELIV_IMMEDIATE:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_immediate, NULL);
+      break;
+  }
+  return 0;
+}
+
+/**
+ * Remove all pubsub dispatchers and events from the mainloop.
+ **/
+void
+tor_mainloop_disconnect_pubsub(void)
+{
+  if (the_pubsub_items) {
+    pubsub_items_clear_bindings(the_pubsub_items);
+    pubsub_items_free(the_pubsub_items);
+  }
+  if (alert_events) {
+    SMARTLIST_FOREACH(alert_events, mainloop_event_t *, ev,
+                      mainloop_event_free(ev));
+    smartlist_free(alert_events);
+  }
+  dispatch_free(the_dispatcher);
+}
diff --git a/src/core/mainloop/mainloop_pubsub.h b/src/core/mainloop/mainloop_pubsub.h
new file mode 100644
index 0000000000..365a3dd565
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_MAINLOOP_PUBSUB_H
+#define TOR_MAINLOOP_PUBSUB_H
+
+struct pubsub_builder_t;
+
+typedef enum {
+   DELIV_NEVER=0,
+   DELIV_PROMPT,
+   DELIV_IMMEDIATE,
+} deliv_strategy_t;
+
+int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder);
+void tor_mainloop_connect_pubsub_events(void);
+int tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                        deliv_strategy_t strategy);
+void tor_mainloop_disconnect_pubsub(void);
+
+#endif /* !defined(TOR_MAINLOOP_PUBSUB_H) */
diff --git a/src/core/mainloop/mainloop_sys.c b/src/core/mainloop/mainloop_sys.c
new file mode 100644
index 0000000000..fbd5a40327
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.c
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop_sys.h"
+#include "core/mainloop/mainloop.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_mainloop_initialize(void)
+{
+  initialize_periodic_events();
+  return 0;
+}
+
+static void
+subsys_mainloop_shutdown(void)
+{
+  tor_mainloop_free_all();
+}
+
+const struct subsys_fns_t sys_mainloop = {
+  .name = "mainloop",
+  .supported = true,
+  .level = 5,
+  .initialize = subsys_mainloop_initialize,
+  .shutdown = subsys_mainloop_shutdown,
+};
diff --git a/src/core/mainloop/mainloop_sys.h b/src/core/mainloop/mainloop_sys.h
new file mode 100644
index 0000000000..fa74fe5d4b
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef MAINLOOP_SYS_H
+#define MAINLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_mainloop;
+
+#endif /* !defined(MAINLOOP_SYS_H) */
diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c
index 1444ca5db2..4924888598 100644
--- a/src/core/mainloop/netstatus.c
+++ b/src/core/mainloop/netstatus.c
@@ -6,9 +6,12 @@
 
 #include "core/or/or.h"
 #include "core/mainloop/netstatus.h"
+#include "core/mainloop/mainloop.h"
 #include "app/config/config.h"
 #include "feature/hibernate/hibernate.h"
 
+#include "app/config/or_state_st.h"
+
 /** Return true iff our network is in some sense disabled or shutting down:
  * either we're hibernating, entering hibernation, or the network is turned
  * off with DisableNetwork. */
@@ -26,3 +29,136 @@ net_is_completely_disabled(void)
 {
   return get_options()->DisableNetwork || we_are_fully_hibernating();
 }
+
+/**
+ * The time at which we've last seen "user activity" -- that is, any activity
+ * that should keep us as a participant on the network.
+ *
+ * This is not actually the true time.  We will adjust this forward if
+ * our clock jumps, or if Tor is shut down for a while, so that the time
+ * since our last activity remains as it was before the jump or shutdown.
+ */
+static time_t last_user_activity_seen = 0;
+
+/**
+ * True iff we are currently a "network participant" -- that is, we
+ * are building circuits, fetching directory information, and so on.
+ **/
+static bool participating_on_network = false;
+
+/**
+ * Record the fact that we have seen "user activity" at the time now.  Move
+ * "last activity seen" time forwards, but never backwards.
+ *
+ * If we were previously not participating on the network, set our
+ * participation status to true, and launch periodic events as appropriate.
+ **/
+void
+note_user_activity(time_t now)
+{
+  last_user_activity_seen = MAX(now, last_user_activity_seen);
+
+  if (! participating_on_network) {
+    log_notice(LD_GENERAL, "Tor is no longer dormant.");
+    set_network_participation(true);
+    schedule_rescan_periodic_events();
+  }
+}
+
+/**
+ * Change the time at which "user activitiy" was last seen to <b>now</b>.
+ *
+ * Unlike note_user_actity, this function sets the time without checking
+ * whether it is in the past, and without causing any rescan of periodic events
+ * or change in participation status.
+ */
+void
+reset_user_activity(time_t now)
+{
+  last_user_activity_seen = now;
+}
+
+/**
+ * Return the most recent time at which we recorded "user activity".
+ **/
+time_t
+get_last_user_activity_time(void)
+{
+  return last_user_activity_seen;
+}
+
+/**
+ * Set the field that remembers whether we are currently participating on the
+ * network.  Does not schedule or un-schedule periodic events.
+ **/
+void
+set_network_participation(bool participation)
+{
+  participating_on_network = participation;
+}
+
+/**
+ * Return true iff we are currently participating on the network.
+ **/
+bool
+is_participating_on_network(void)
+{
+  return participating_on_network;
+}
+
+/**
+ * Update 'state' with the last time at which we were active on the network.
+ **/
+void
+netstatus_flush_to_state(or_state_t *state, time_t now)
+{
+  state->Dormant = ! participating_on_network;
+  if (participating_on_network) {
+    time_t sec_since_activity = MAX(0, now - last_user_activity_seen);
+    state->MinutesSinceUserActivity = (int)(sec_since_activity / 60);
+  } else {
+    state->MinutesSinceUserActivity = 0;
+  }
+}
+
+/**
+ * Update our current view of network participation from an or_state_t object.
+ **/
+void
+netstatus_load_from_state(const or_state_t *state, time_t now)
+{
+  time_t last_activity;
+  if (state->Dormant == -1) { // Initial setup.
+    if (get_options()->DormantOnFirstStartup) {
+      last_activity = 0;
+      participating_on_network = false;
+    } else {
+      // Start up as active, treat activity as happening now.
+      last_activity = now;
+      participating_on_network = true;
+    }
+  } else if (state->Dormant) {
+    last_activity = 0;
+    participating_on_network = false;
+  } else {
+    last_activity = now - 60 * state->MinutesSinceUserActivity;
+    participating_on_network = true;
+  }
+  if (get_options()->DormantCanceledByStartup) {
+    last_activity = now;
+    participating_on_network = true;
+  }
+  reset_user_activity(last_activity);
+}
+
+/**
+ * Adjust the time at which the user was last active by <b>seconds_diff</b>
+ * in response to a clock jump.
+ */
+void
+netstatus_note_clock_jumped(time_t seconds_diff)
+{
+  time_t last_active = get_last_user_activity_time();
+  if (last_active)
+    reset_user_activity(last_active + seconds_diff);
+}
diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h
index ae8547b30d..e8469ff558 100644
--- a/src/core/mainloop/netstatus.h
+++ b/src/core/mainloop/netstatus.h
@@ -10,4 +10,15 @@
 int net_is_disabled(void);
 int net_is_completely_disabled(void);
 
-#endif
+void note_user_activity(time_t now);
+void reset_user_activity(time_t now);
+time_t get_last_user_activity_time(void);
+
+void set_network_participation(bool participation);
+bool is_participating_on_network(void);
+
+void netstatus_flush_to_state(or_state_t *state, time_t now);
+void netstatus_load_from_state(const or_state_t *state, time_t now);
+void netstatus_note_clock_jumped(time_t seconds_diff);
+
+#endif /* !defined(TOR_NETSTATUS_H) */
diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c
index 2651bbbc89..5c2f6f2b36 100644
--- a/src/core/mainloop/periodic.c
+++ b/src/core/mainloop/periodic.c
@@ -6,9 +6,22 @@
  *
  * \brief Generic backend for handling periodic events.
  *
- * The events in this module are used by main.c to track items that need
+ * The events in this module are used to track items that need
  * to fire once every N seconds, possibly picking a new interval each time
- * that they fire.  See periodic_events[] in main.c for examples.
+ * that they fire.  See periodic_events[] in mainloop.c for examples.
+ *
+ * This module manages a global list of periodic_event_item_t objects,
+ * each corresponding to a single event.  To register an event, pass it to
+ * periodic_events_register() when initializing your subsystem.
+ *
+ * Registering an event makes the periodic event subsystem know about it, but
+ * doesn't cause the event to get created immediately.  Before the event can
+ * be started, periodic_event_connect_all() must be called by mainloop.c to
+ * connect all the events to Libevent.
+ *
+ * We expect that periodic_event_item_t objects will be statically allocated;
+ * we set them up and tear them down here, but we don't take ownership of
+ * them.
  */
 
 #include "core/or/or.h"
@@ -24,6 +37,12 @@
  */
 static const int MAX_INTERVAL = 10 * 365 * 86400;
 
+/**
+ * Global list of periodic events that have been registered with
+ * <b>periodic_event_register</a>.
+ **/
+static smartlist_t *the_periodic_events = NULL;
+
 /** Set the event <b>event</b> to run in <b>next_interval</b> seconds from
  * now. */
 static void
@@ -45,10 +64,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   periodic_event_item_t *event = data;
   tor_assert(ev == event->ev);
 
-  if (BUG(!periodic_event_is_enabled(event))) {
-    return;
-  }
-
   time_t now = time(NULL);
   update_current_time(now);
   const or_options_t *options = get_options();
@@ -57,7 +72,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   int next_interval = 0;
 
   if (!periodic_event_is_enabled(event)) {
-    /* The event got disabled from inside its callback; no need to
+    /* The event got disabled from inside its callback, or before: no need to
      * reschedule. */
     return;
   }
@@ -91,15 +106,16 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
 void
 periodic_event_reschedule(periodic_event_item_t *event)
 {
-  /* Don't reschedule a disabled event. */
-  if (periodic_event_is_enabled(event)) {
+  /* Don't reschedule a disabled or uninitialized event. */
+  if (event->ev && periodic_event_is_enabled(event)) {
     periodic_event_set_interval(event, 1);
   }
 }
 
-/** Initializes the libevent backend for a periodic event. */
+/** Connects a periodic event to the Libevent backend.  Does not launch the
+ * event immediately. */
 void
-periodic_event_setup(periodic_event_item_t *event)
+periodic_event_connect(periodic_event_item_t *event)
 {
   if (event->ev) { /* Already setup? This is a bug */
     log_err(LD_BUG, "Initial dispatch should only be done once.");
@@ -117,7 +133,7 @@ void
 periodic_event_launch(periodic_event_item_t *event)
 {
   if (! event->ev) { /* Not setup? This is a bug */
-    log_err(LD_BUG, "periodic_event_launch without periodic_event_setup");
+    log_err(LD_BUG, "periodic_event_launch without periodic_event_connect");
     tor_assert(0);
   }
   /* Event already enabled? This is a bug */
@@ -131,9 +147,9 @@ periodic_event_launch(periodic_event_item_t *event)
   periodic_event_dispatch(event->ev, event);
 }
 
-/** Release all storage associated with <b>event</b> */
-void
-periodic_event_destroy(periodic_event_item_t *event)
+/** Disconnect and unregister the periodic event in <b>event</b> */
+static void
+periodic_event_disconnect(periodic_event_item_t *event)
 {
   if (!event)
     return;
@@ -177,3 +193,177 @@ periodic_event_disable(periodic_event_item_t *event)
   mainloop_event_cancel(event->ev);
   event->enabled = 0;
 }
+
+/**
+ * Disable an event, then schedule it to run once.
+ * Do nothing if the event was already disabled.
+ */
+void
+periodic_event_schedule_and_disable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  if (!periodic_event_is_enabled(event))
+    return;
+
+  periodic_event_disable(event);
+
+  mainloop_event_activate(event->ev);
+}
+
+/**
+ * Add <b>item</b> to the list of periodic events.
+ *
+ * Note that <b>item</b> should be statically allocated: we do not
+ * take ownership of it.
+ **/
+void
+periodic_events_register(periodic_event_item_t *item)
+{
+  if (!the_periodic_events)
+    the_periodic_events = smartlist_new();
+
+  if (BUG(smartlist_contains(the_periodic_events, item)))
+    return;
+
+  smartlist_add(the_periodic_events, item);
+}
+
+/**
+ * Make all registered periodic events connect to the libevent backend.
+ */
+void
+periodic_events_connect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (item->ev)
+      continue;
+    periodic_event_connect(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Reset all the registered periodic events so we'll do all our actions again
+ * as if we just started up.
+ *
+ * Useful if our clock just moved back a long time from the future,
+ * so we don't wait until that future arrives again before acting.
+ */
+void
+periodic_events_reset_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    periodic_event_reschedule(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Return the registered periodic event whose name is <b>name</b>.
+ * Return NULL if no such event is found.
+ */
+periodic_event_item_t *
+periodic_events_find(const char *name)
+{
+  if (! the_periodic_events)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (strcmp(name, item->name) == 0)
+      return item;
+  } SMARTLIST_FOREACH_END(item);
+  return NULL;
+}
+
+/**
+ * Start or stop registered periodic events, depending on our current set of
+ * roles.
+ *
+ * Invoked when our list of roles, or the net_disabled flag has changed.
+ **/
+void
+periodic_events_rescan_by_roles(int roles, bool net_disabled)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    int enable = !!(item->roles & roles);
+
+    /* Handle the event flags. */
+    if (net_disabled &&
+        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
+      enable = 0;
+    }
+
+    /* Enable the event if needed. It is safe to enable an event that was
+     * already enabled. Same goes for disabling it. */
+    if (enable) {
+      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
+      periodic_event_enable(item);
+    } else {
+      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
+      if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) {
+        periodic_event_schedule_and_disable(item);
+      } else {
+        periodic_event_disable(item);
+      }
+    }
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Invoked at shutdown: disconnect and unregister all periodic events.
+ *
+ * Does not free the periodic_event_item_t object themselves, because we do
+ * not own them.
+ */
+void
+periodic_events_disconnect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    periodic_event_disconnect(item);
+  } SMARTLIST_FOREACH_END(item);
+
+  smartlist_free(the_periodic_events);
+}
+
+#define LONGEST_TIMER_PERIOD (30 * 86400)
+/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
+ * clipped to the range [1 second, LONGEST_TIMER_PERIOD].
+ *
+ * We use this to answer the question, "how many seconds is it from now until
+ * next" in periodic timer callbacks.  Don't use it for other purposes
+ **/
+int
+safe_timer_diff(time_t now, time_t next)
+{
+  if (next > now) {
+    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
+     * and nothing that could run Tor. It's a bug if 'next' is around then.
+     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
+     * Bang. We cannot extrapolate past a singularity, but there was probably
+     * nothing that could run Tor then, either.
+     **/
+    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
+
+    if (next - LONGEST_TIMER_PERIOD > now)
+      return LONGEST_TIMER_PERIOD;
+    return (int)(next - now);
+  } else {
+    return 1;
+  }
+}
diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h
index e49fafd174..a9aa461969 100644
--- a/src/core/mainloop/periodic.h
+++ b/src/core/mainloop/periodic.h
@@ -15,6 +15,10 @@
 #define PERIODIC_EVENT_ROLE_BRIDGEAUTH  (1U << 4)
 #define PERIODIC_EVENT_ROLE_HS_SERVICE  (1U << 5)
 #define PERIODIC_EVENT_ROLE_DIRSERVER   (1U << 6)
+#define PERIODIC_EVENT_ROLE_CONTROLEV   (1U << 7)
+
+#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8)
+#define PERIODIC_EVENT_ROLE_ALL         (1U << 9)
 
 /* Helper macro to make it a bit less annoying to defined groups of roles that
  * are often used. */
@@ -25,10 +29,6 @@
 /* Authorities that is both bridge and directory. */
 #define PERIODIC_EVENT_ROLE_AUTHORITIES \
   (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH)
-/* All roles. */
-#define PERIODIC_EVENT_ROLE_ALL \
-  (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \
-   PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER)
 
 /*
  * Event flags which can change the behavior of an event.
@@ -39,6 +39,11 @@
  * the net_is_disabled() check. */
 #define PERIODIC_EVENT_FLAG_NEED_NET  (1U << 0)
 
+/* Indicate that if the event is enabled, it needs to be run once before
+ * it becomes disabled.
+ */
+#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE  (1U << 1)
+
 /** Callback function for a periodic event to take action.  The return value
 * influences the next time the function will get called.  Return
 * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled
@@ -78,11 +83,20 @@ periodic_event_is_enabled(const periodic_event_item_t *item)
 }
 
 void periodic_event_launch(periodic_event_item_t *event);
-void periodic_event_setup(periodic_event_item_t *event);
-void periodic_event_destroy(periodic_event_item_t *event);
+void periodic_event_connect(periodic_event_item_t *event);
+//void periodic_event_disconnect(periodic_event_item_t *event);
 void periodic_event_reschedule(periodic_event_item_t *event);
 void periodic_event_enable(periodic_event_item_t *event);
 void periodic_event_disable(periodic_event_item_t *event);
+void periodic_event_schedule_and_disable(periodic_event_item_t *event);
 
-#endif /* !defined(TOR_PERIODIC_H) */
+void periodic_events_register(periodic_event_item_t *item);
+void periodic_events_connect_all(void);
+void periodic_events_reset_all(void);
+periodic_event_item_t *periodic_events_find(const char *name);
+void periodic_events_rescan_by_roles(int roles, bool net_disabled);
+void periodic_events_disconnect_all(void);
+
+int safe_timer_diff(time_t now, time_t next);
 
+#endif /* !defined(TOR_PERIODIC_H) */
diff --git a/src/core/or/.may_include b/src/core/or/.may_include
new file mode 100644
index 0000000000..5173e8a2b6
--- /dev/null
+++ b/src/core/or/.may_include
@@ -0,0 +1,38 @@
+!advisory
+
+orconfig.h
+
+lib/arch/*.h
+lib/buf/*.h
+lib/cc/*.h
+lib/compress/*.h
+lib/container/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/defs/*.h
+lib/encoding/*.h
+lib/err/*.h
+lib/evloop/*.h
+lib/fs/*.h
+lib/geoip/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/math/*.h
+lib/net/*.h
+lib/pubsub/*.h
+lib/string/*.h
+lib/subsys/*.h
+lib/test/*.h
+lib/testsupport/*.h
+lib/thread/*.h
+lib/time/*.h
+lib/tls/*.h
+lib/wallclock/*.h
+
+trunnel/*.h
+
+core/mainloop/*.h
+core/proto/*.h
+core/crypto/*.h
+core/or/*.h
\ No newline at end of file
diff --git a/src/core/or/addr_policy_st.h b/src/core/or/addr_policy_st.h
index a75f1a731d..11442d29b4 100644
--- a/src/core/or/addr_policy_st.h
+++ b/src/core/or/addr_policy_st.h
@@ -43,4 +43,4 @@ struct addr_policy_t {
   uint16_t prt_max; /**< Highest port number to accept/reject. */
 };
 
-#endif
+#endif /* !defined(TOR_ADDR_POLICY_ST_H) */
diff --git a/src/core/or/address_set.h b/src/core/or/address_set.h
index 7a9e71628e..95608a9a53 100644
--- a/src/core/or/address_set.h
+++ b/src/core/or/address_set.h
@@ -28,4 +28,4 @@ void address_set_add_ipv4h(address_set_t *set, uint32_t addr);
 int address_set_probably_contains(const address_set_t *set,
                                   const struct tor_addr_t *addr);
 
-#endif
+#endif /* !defined(TOR_ADDRESS_SET_H) */
diff --git a/src/core/or/cell_queue_st.h b/src/core/or/cell_queue_st.h
index 130b95a011..7ba339b965 100644
--- a/src/core/or/cell_queue_st.h
+++ b/src/core/or/cell_queue_st.h
@@ -26,4 +26,4 @@ struct cell_queue_t {
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
+#endif /* !defined(PACKED_CELL_ST_H) */
diff --git a/src/core/or/cell_st.h b/src/core/or/cell_st.h
index 7ab7eceb50..c4eec4f4b5 100644
--- a/src/core/or/cell_st.h
+++ b/src/core/or/cell_st.h
@@ -16,5 +16,5 @@ struct cell_t {
   uint8_t payload[CELL_PAYLOAD_SIZE]; /**< Cell body. */
 };
 
-#endif
+#endif /* !defined(CELL_ST_H) */
 
diff --git a/src/core/or/channel.c b/src/core/or/channel.c
index 3886906875..596932b008 100644
--- a/src/core/or/channel.c
+++ b/src/core/or/channel.c
@@ -1418,6 +1418,7 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
   int ret = -1;
   size_t cell_bytes;
+  uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids);
 
   tor_assert(chan);
   tor_assert(cell);
@@ -1452,6 +1453,16 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
   /* Successfully sent the cell. */
   ret = 0;
 
+  /* Update padding statistics for the packed codepath.. */
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
+  if (command == CELL_PADDING)
+    rep_hist_padding_count_write(PADDING_TYPE_CELL);
+  if (chan->padding_enabled) {
+    rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
+    if (command == CELL_PADDING)
+      rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
+  }
+
  done:
   return ret;
 }
diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c
index f874e39946..d6c9e21631 100644
--- a/src/core/or/channeltls.c
+++ b/src/core/or/channeltls.c
@@ -59,6 +59,7 @@
 #include "feature/nodelist/torcert.h"
 #include "feature/nodelist/networkstatus.h"
 #include "trunnel/channelpadding_negotiation.h"
+#include "trunnel/netinfo.h"
 #include "core/or/channelpadding.h"
 
 #include "core/or/cell_st.h"
@@ -949,7 +950,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
 void
 channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                           or_connection_t *conn,
-                                          uint8_t old_state,
                                           uint8_t state)
 {
   channel_t *base_chan;
@@ -958,8 +958,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
   tor_assert(conn);
   tor_assert(conn->chan == chan);
   tor_assert(chan->conn == conn);
-  /* Shut the compiler up without triggering -Wtautological-compare */
-  (void)old_state;
 
   base_chan = TLS_CHAN_TO_BASE(chan);
 
@@ -1029,6 +1027,16 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
 }
 #endif /* defined(KEEP_TIMING_STATS) */
 
+#ifdef KEEP_TIMING_STATS
+#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
+    ++num ## tp;                                                \
+    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
+                             channel_tls_process_ ## tp ## _cell);  \
+    } STMT_END
+#else /* !defined(KEEP_TIMING_STATS) */
+#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
+#endif /* defined(KEEP_TIMING_STATS) */
+
 /**
  * Handle an incoming cell on a channel_tls_t.
  *
@@ -1048,16 +1056,6 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   channel_tls_t *chan;
   int handshaking;
 
-#ifdef KEEP_TIMING_STATS
-#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
-    ++num ## tp;                                                \
-    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
-                             channel_tls_process_ ## tp ## _cell);  \
-    } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
-#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
-#endif /* defined(KEEP_TIMING_STATS) */
-
   tor_assert(cell);
   tor_assert(conn);
 
@@ -1075,7 +1073,8 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
     return;
 
   /* Reject all but VERSIONS and NETINFO when handshaking. */
-  /* (VERSIONS should actually be impossible; it's variable-length.) */
+  /* (VERSIONS actually indicates a protocol warning: it's variable-length,
+   * so if it reaches this function, we're on a v1 connection.) */
   if (handshaking && cell->command != CELL_VERSIONS &&
       cell->command != CELL_NETINFO) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
@@ -1096,13 +1095,13 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   entry_guards_note_internet_connectivity(get_guard_selection_info());
   rep_hist_padding_count_read(PADDING_TYPE_TOTAL);
 
-  if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+  if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
     rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL);
 
   switch (cell->command) {
     case CELL_PADDING:
       rep_hist_padding_count_read(PADDING_TYPE_CELL);
-      if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+      if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
         rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL);
       ++stats_n_padding_cells_processed;
       /* do nothing */
@@ -1330,6 +1329,8 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
   }
 }
 
+#undef PROCESS_CELL
+
 /**
  * Update channel marks after connection_or.c has changed an address.
  *
@@ -1645,6 +1646,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan)
 }
 
 /**
+ * Convert <b>netinfo_addr</b> into corresponding <b>tor_addr</b>.
+ * Return 0 on success; on failure, return -1 and log a warning.
+ */
+static int
+tor_addr_from_netinfo_addr(tor_addr_t *tor_addr,
+                           const netinfo_addr_t *netinfo_addr) {
+  tor_assert(tor_addr);
+  tor_assert(netinfo_addr);
+
+  uint8_t type = netinfo_addr_get_addr_type(netinfo_addr);
+  uint8_t len = netinfo_addr_get_len(netinfo_addr);
+
+  if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4)  {
+    uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr);
+    tor_addr_from_ipv4h(tor_addr, ipv4);
+  } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) {
+    const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6(
+                                  netinfo_addr);
+    tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes);
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO "
+                                     "- wrong type/length.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/**
  * Helper: compute the absolute value of a time_t.
  *
  * (we need this because labs() doesn't always work for time_t, since
@@ -1668,8 +1698,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   time_t timestamp;
   uint8_t my_addr_type;
   uint8_t my_addr_len;
-  const uint8_t *my_addr_ptr;
-  const uint8_t *cp, *end;
   uint8_t n_other_addrs;
   time_t now = time(NULL);
   const routerinfo_t *me = router_get_my_routerinfo();
@@ -1717,7 +1745,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
         tor_assert(tor_digest_is_zero(
                   (const char*)(chan->conn->handshake_state->
                       authenticated_rsa_peer_id)));
-        tor_assert(tor_mem_is_zero(
+        tor_assert(fast_mem_is_zero(
                   (const char*)(chan->conn->handshake_state->
                                 authenticated_ed25519_peer_id.pubkey), 32));
         /* If the client never authenticated, it's a tor client or bridge
@@ -1740,38 +1768,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   }
 
   /* Decode the cell. */
-  timestamp = ntohl(get_uint32(cell->payload));
-  const time_t sent_versions_at =
-    chan->conn->handshake_state->sent_versions_at;
-  if (now > sent_versions_at && (now - sent_versions_at) < 180) {
-    /* If we have gotten the NETINFO cell reasonably soon after having
-     * sent our VERSIONS cell, maybe we can learn skew information from it. */
-    apparent_skew = now - timestamp;
+  netinfo_cell_t *netinfo_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload,
+                                      CELL_PAYLOAD_SIZE);
+
+  if (parsed < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+           "Failed to parse NETINFO cell - closing connection.");
+    connection_or_close_for_error(chan->conn, 0);
+    return;
   }
 
-  my_addr_type = (uint8_t) cell->payload[4];
-  my_addr_len = (uint8_t) cell->payload[5];
-  my_addr_ptr = (uint8_t*) cell->payload + 6;
-  end = cell->payload + CELL_PAYLOAD_SIZE;
-  cp = cell->payload + 6 + my_addr_len;
+  timestamp = netinfo_cell_get_timestamp(netinfo_cell);
+
+  const netinfo_addr_t *my_addr =
+    netinfo_cell_getconst_other_addr(netinfo_cell);
 
+  my_addr_type = netinfo_addr_get_addr_type(my_addr);
+  my_addr_len = netinfo_addr_get_len(my_addr);
+
+  if ((now - chan->conn->handshake_state->sent_versions_at) < 180) {
+    apparent_skew = now - timestamp;
+  }
   /* We used to check:
    *    if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) {
    *
    * This is actually never going to happen, since my_addr_len is at most 255,
    * and CELL_PAYLOAD_LEN - 6 is 503.  So we know that cp is < end. */
 
-  if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) {
-    tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr));
+  if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) {
+    connection_or_close_for_error(chan->conn, 0);
+    netinfo_cell_free(netinfo_cell);
+    return;
+  }
 
+  if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) {
     if (!get_options()->BridgeRelay && me &&
-        get_uint32(my_addr_ptr) == htonl(me->addr)) {
+        tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) {
       TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
     }
-
-  } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) {
-    tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr);
-
+  } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 &&
+             my_addr_len == 16) {
     if (!get_options()->BridgeRelay && me &&
         !tor_addr_is_null(&me->ipv6_addr) &&
         tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) {
@@ -1779,18 +1817,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
     }
   }
 
-  n_other_addrs = (uint8_t) *cp++;
-  while (n_other_addrs && cp < end-2) {
+  n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell);
+  for (uint8_t i = 0; i < n_other_addrs; i++) {
     /* Consider all the other addresses; if any matches, this connection is
      * "canonical." */
+
+    const netinfo_addr_t *netinfo_addr =
+      netinfo_cell_getconst_my_addrs(netinfo_cell, i);
+
     tor_addr_t addr;
-    const uint8_t *next =
-      decode_address_from_payload(&addr, cp, (int)(end-cp));
-    if (next == NULL) {
+
+    if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) {
       log_fn(LOG_PROTOCOL_WARN,  LD_OR,
-             "Bad address in netinfo cell; closing connection.");
-      connection_or_close_for_error(chan->conn, 0);
-      return;
+             "Bad address in netinfo cell; Skipping.");
+      continue;
     }
     /* A relay can connect from anywhere and be canonical, so
      * long as it tells you from where it came. This may sound a bit
@@ -1803,10 +1843,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       connection_or_set_canonical(chan->conn, 1);
       break;
     }
-    cp = next;
-    --n_other_addrs;
   }
 
+  netinfo_cell_free(netinfo_cell);
+
   if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer &&
       channel_is_canonical(TLS_CHAN_TO_BASE(chan))) {
     const char *descr =
diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h
index a2648ff537..634a2a00e9 100644
--- a/src/core/or/channeltls.h
+++ b/src/core/or/channeltls.h
@@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan);
 void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn);
 void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                                or_connection_t *conn,
-                                               uint8_t old_state,
                                                uint8_t state);
 void channel_tls_handle_var_cell(var_cell_t *var_cell,
                                  or_connection_t *conn);
diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h
index d4339ff50d..eae3c908d5 100644
--- a/src/core/or/circuit_st.h
+++ b/src/core/or/circuit_st.h
@@ -12,6 +12,11 @@
 #include "core/or/cell_queue_st.h"
 
 struct hs_token_t;
+struct circpad_machine_spec_t;
+struct circpad_machine_runtime_t;
+
+/** Number of padding state machines on a circuit. */
+#define CIRCPAD_MAX_MACHINES (2)
 
 /** "magic" value for an origin_circuit_t */
 #define ORIGIN_CIRCUIT_MAGIC 0x35315243u
@@ -61,12 +66,6 @@ struct circuit_t {
    */
   circid_t n_circ_id;
 
-  /**
-   * Circuit mux associated with n_chan to which this circuit is attached;
-   * NULL if we have no n_chan.
-   */
-  circuitmux_t *n_mux;
-
   /** Queue of cells waiting to be transmitted on n_chan */
   cell_queue_t n_chan_cells;
 
@@ -93,6 +92,10 @@ struct circuit_t {
   /** True iff this circuit has received a DESTROY cell in either direction */
   unsigned int received_destroy : 1;
 
+  /** True iff we have sent a sufficiently random data cell since last
+   * we reset send_randomness_after_n_cells. */
+  unsigned int have_sent_sufficiently_random_cell : 1;
+
   uint8_t state; /**< Current status of this circuit. */
   uint8_t purpose; /**< Why are we creating this circuit? */
 
@@ -105,6 +108,32 @@ struct circuit_t {
    * circuit-level sendme cells to indicate that we're willing to accept
    * more. */
   int deliver_window;
+  /**
+   * How many cells do we have until we need to send one that contains
+   * sufficient randomness?  Used to ensure that authenticated SENDME cells
+   * will reflect some unpredictable information.
+   **/
+  uint16_t send_randomness_after_n_cells;
+
+  /** FIFO containing the digest of the cells that are just before a SENDME is
+   * sent by the client. It is done at the last cell before our package_window
+   * goes down to 0 which is when we expect a SENDME.
+   *
+   * Our current circuit package window is capped to 1000
+   * (CIRCWINDOW_START_MAX) which is also the start value. The increment is
+   * set to 100 (CIRCWINDOW_INCREMENT) which means we don't allow more than
+   * 1000/100 = 10 outstanding SENDME cells worth of data. Meaning that this
+   * list can not contain more than 10 digests of DIGEST_LEN bytes (20).
+   *
+   * At position i in the list, the digest corresponds to the
+   * (CIRCWINDOW_INCREMENT * i)-nth cell received since we expect a SENDME to
+   * be received containing that cell digest.
+   *
+   * For example, position 2 (starting at 0) means that we've received 300
+   * cells so the 300th cell digest is kept at index 2.
+   *
+   * At maximum, this list contains 200 bytes plus the smartlist overhead. */
+  smartlist_t *sendme_last_digests;
 
   /** Temporary field used during circuits_handle_oom. */
   uint32_t age_tmp;
@@ -177,6 +206,27 @@ struct circuit_t {
   /** Hashtable node: used to look up the circuit by its HS token using the HS
       circuitmap. */
   HT_ENTRY(circuit_t) hs_circuitmap_node;
+
+  /** Adaptive Padding state machines: these are immutable. The state machines
+   *  that come from the consensus are saved to a global structure, to avoid
+   *  per-circuit allocations. This merely points to the global copy in
+   *  origin_padding_machines or relay_padding_machines that should never
+   *  change or get deallocated.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES];
+
+  /** Adaptive Padding machine runtime info for above machines. This is
+   *  the per-circuit mutable information, such as the current state and
+   *  histogram token counts. Some of it is optional (aka NULL).
+   *  If a machine is being shut down, these indexes can be NULL
+   *  without the corresponding padding_machine being NULL, while we
+   *  wait for the other end to respond to our shutdown request.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  struct circpad_machine_runtime_t *padding_info[CIRCPAD_MAX_MACHINES];
 };
 
-#endif
+#endif /* !defined(CIRCUIT_ST_H) */
diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c
index f3a5791d6c..16bb7e7d79 100644
--- a/src/core/or/circuitbuild.c
+++ b/src/core/or/circuitbuild.c
@@ -26,10 +26,11 @@
  **/
 
 #define CIRCUITBUILD_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/crypto/hs_ntor.h"
 #include "core/crypto/onion_crypto.h"
 #include "core/crypto/onion_fast.h"
@@ -42,17 +43,20 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/command.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/onion.h"
+#include "core/or/ocirc_event.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
@@ -87,8 +91,6 @@ static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
 static int circuit_deliver_create_cell(circuit_t *circ,
                                        const create_cell_t *create_cell,
                                        int relayed);
-static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
-STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 static int circuit_send_first_onion_skin(origin_circuit_t *circ);
 static int circuit_build_no_more_hops(origin_circuit_t *circ);
 static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
@@ -492,7 +494,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
     return NULL;
   }
 
-  control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
+  circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
 
   if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
     circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
@@ -508,6 +510,27 @@ origin_circuit_get_guard_state(origin_circuit_t *circ)
   return circ->guard_state;
 }
 
+/**
+ * Helper function to publish a channel association message
+ *
+ * circuit_handle_first_hop() calls this to notify subscribers about a
+ * channel launch event, which associates a circuit with a channel.
+ * This doesn't always correspond to an assignment of the circuit's
+ * n_chan field, because that seems to be only for fully-open
+ * channels.
+ **/
+static void
+circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = circ->global_identifier;
+  msg->chan = chan->global_identifier;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_chan_publish(msg);
+}
+
 /** Start establishing the first hop of our circuit. Figure out what
  * OR we should connect to, and if necessary start the connection to
  * it. If we're already connected, then send the 'create' cell.
@@ -522,7 +545,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   int should_launch = 0;
   const or_options_t *options = get_options();
 
-  firsthop = onion_next_hop_in_cpath(circ->cpath);
+  firsthop = cpath_get_next_non_open_hop(circ->cpath);
   tor_assert(firsthop);
   tor_assert(firsthop->extend_info);
 
@@ -559,8 +582,6 @@ circuit_handle_first_hop(origin_circuit_t *circ)
     circ->base_.n_hop = extend_info_dup(firsthop->extend_info);
 
     if (should_launch) {
-      if (circ->build_state->onehop_tunnel)
-        control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
       n_chan = channel_connect_for_circuit(
           &firsthop->extend_info->addr,
           firsthop->extend_info->port,
@@ -570,6 +591,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
       }
+      circuit_chan_publish(circ, n_chan);
     }
 
     log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
@@ -581,6 +603,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   } else { /* it's already open. use it. */
     tor_assert(!circ->base_.n_hop);
     circ->base_.n_chan = n_chan;
+    circuit_chan_publish(circ, n_chan);
     log_debug(LD_CIRC,"Conn open. Delivering first onion skin.");
     if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
       log_info(LD_CIRC,"circuit_send_next_onion_skin failed.");
@@ -923,15 +946,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
   tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
   tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
 
-  crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath);
+  crypt_path_t *hop = cpath_get_next_non_open_hop(circ->cpath);
   circuit_build_times_handle_completed_hop(circ);
 
+  circpad_machine_event_circ_added_hop(circ);
+
   if (hop) {
     /* Case two: we're on a hop after the first. */
     return circuit_send_intermediate_onion_skin(circ, hop);
   }
 
   /* Case three: the circuit is finished. Do housekeeping tasks on it. */
+  circpad_machine_event_circ_built(circ);
   return circuit_build_no_more_hops(circ);
 }
 
@@ -1332,34 +1358,6 @@ circuit_extend(cell_t *cell, circuit_t *circ)
   return 0;
 }
 
-/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
- *
- * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
- * service circuits and <b>key_data</b> must be at least
- * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
- *
- * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
- * bytes, which are used as follows:
- *   - 20 to initialize f_digest
- *   - 20 to initialize b_digest
- *   - 16 to key f_crypto
- *   - 16 to key b_crypto
- *
- * (If 'reverse' is true, then f_XX and b_XX are swapped.)
- *
- * Return 0 if init was successful, else -1 if it failed.
- */
-int
-circuit_init_cpath_crypto(crypt_path_t *cpath,
-                          const char *key_data, size_t key_data_len,
-                          int reverse, int is_hs_v3)
-{
-
-  tor_assert(cpath);
-  return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse,
-                           is_hs_v3);
-}
-
 /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>.
  * (The body of <b>reply</b> varies depending on what sort of handshake
  * this is.)
@@ -1385,7 +1383,7 @@ circuit_finish_handshake(origin_circuit_t *circ,
   if (circ->cpath->state == CPATH_STATE_AWAITING_KEYS) {
     hop = circ->cpath;
   } else {
-    hop = onion_next_hop_in_cpath(circ->cpath);
+    hop = cpath_get_next_non_open_hop(circ->cpath);
     if (!hop) { /* got an extended when we're all done? */
       log_warn(LD_PROTOCOL,"got extended when circ already built? Closing.");
       return - END_CIRC_REASON_TORPROTOCOL;
@@ -1409,14 +1407,14 @@ circuit_finish_handshake(origin_circuit_t *circ,
 
   onion_handshake_state_release(&hop->handshake_state);
 
-  if (circuit_init_cpath_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
+  if (cpath_init_circuit_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
     return -END_CIRC_REASON_TORPROTOCOL;
   }
 
   hop->state = CPATH_STATE_OPEN;
   log_info(LD_CIRC,"Finished building circuit hop:");
   circuit_log_path(LOG_INFO,LD_CIRC,circ);
-  control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0);
+  circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0);
 
   return 0;
 }
@@ -1461,7 +1459,7 @@ circuit_truncated(origin_circuit_t *circ, int reason)
     }
 
     layer->next = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
   log_info(LD_CIRC, "finished");
@@ -1655,24 +1653,28 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei)
  * to handle the desired path length, return -1.
  */
 STATIC int
-new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
+new_route_len(uint8_t purpose, extend_info_t *exit_ei,
+              const smartlist_t *nodes)
 {
-  int num_acceptable_routers;
   int routelen;
 
   tor_assert(nodes);
 
   routelen = route_len_for_purpose(purpose, exit_ei);
 
-  num_acceptable_routers = count_acceptable_nodes(nodes);
+  int num_acceptable_direct = count_acceptable_nodes(nodes, 1);
+  int num_acceptable_indirect = count_acceptable_nodes(nodes, 0);
 
-  log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).",
-            routelen, num_acceptable_routers, smartlist_len(nodes));
+  log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect "
+             "routers suitable).", routelen, num_acceptable_direct,
+             num_acceptable_indirect);
 
-  if (num_acceptable_routers < routelen) {
+  if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) {
     log_info(LD_CIRC,
-             "Not enough acceptable routers (%d/%d). Discarding this circuit.",
-             num_acceptable_routers, routelen);
+             "Not enough acceptable routers (%d/%d direct and %d/%d "
+             "indirect routers suitable). Discarding this circuit.",
+             num_acceptable_direct, routelen,
+             num_acceptable_indirect, routelen);
     return -1;
   }
 
@@ -2276,7 +2278,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
   state->chosen_exit = extend_info_dup(exit_ei);
 
   ++circ->build_state->desired_path_len;
-  onion_append_hop(&circ->cpath, exit_ei);
+  cpath_append_hop(&circ->cpath, exit_ei);
   return 0;
 }
 
@@ -2314,7 +2316,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
  * particular router. See bug #25885.)
  */
 MOCK_IMPL(STATIC int,
-count_acceptable_nodes, (smartlist_t *nodes))
+count_acceptable_nodes, (const smartlist_t *nodes, int direct))
 {
   int num=0;
 
@@ -2328,7 +2330,7 @@ count_acceptable_nodes, (smartlist_t *nodes))
     if (! node->is_valid)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i);
       continue;
-    if (! node_has_any_descriptor(node))
+    if (! node_has_preferred_descriptor(node, direct))
       continue;
     /* The node has a descriptor, so we can just check the ntor key directly */
     if (!node_has_curve25519_onion_key(node))
@@ -2341,47 +2343,6 @@ count_acceptable_nodes, (smartlist_t *nodes))
   return num;
 }
 
-/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
- * This function is used to extend cpath by another hop.
- */
-void
-onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop)
-{
-  if (*head_ptr) {
-    new_hop->next = (*head_ptr);
-    new_hop->prev = (*head_ptr)->prev;
-    (*head_ptr)->prev->next = new_hop;
-    (*head_ptr)->prev = new_hop;
-  } else {
-    *head_ptr = new_hop;
-    new_hop->prev = new_hop->next = new_hop;
-  }
-}
-
-#ifdef TOR_UNIT_TESTS
-
-/** Unittest helper function: Count number of hops in cpath linked list. */
-unsigned int
-cpath_get_n_hops(crypt_path_t **head_ptr)
-{
-  unsigned int n_hops = 0;
-  crypt_path_t *tmp;
-
-  if (!*head_ptr) {
-    return 0;
-  }
-
-  tmp = *head_ptr;
-  do {
-    n_hops++;
-    tmp = tmp->next;
-  } while (tmp != *head_ptr);
-
-  return n_hops;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 /**
  * Build the exclude list for vanguard circuits.
  *
@@ -2579,7 +2540,24 @@ choose_good_middle_server(uint8_t purpose,
     return choice;
   }
 
-  choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  if (options->MiddleNodes) {
+    smartlist_t *sl = smartlist_new();
+    routerset_get_all_nodes(sl, options->MiddleNodes,
+                            options->ExcludeNodes, 1);
+
+    smartlist_subtract(sl, excluded);
+
+    choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID);
+    smartlist_free(sl);
+    if (choice) {
+      log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s",
+          hex_str(choice->identity, DIGEST_LEN));
+    } else {
+      log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available");
+    }
+  } else {
+    choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  }
   smartlist_free(excluded);
   return choice;
 }
@@ -2639,20 +2617,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
   return choice;
 }
 
-/** Return the first non-open hop in cpath, or return NULL if all
- * hops are open. */
-static crypt_path_t *
-onion_next_hop_in_cpath(crypt_path_t *cpath)
-{
-  crypt_path_t *hop = cpath;
-  do {
-    if (hop->state != CPATH_STATE_OPEN)
-      return hop;
-    hop = hop->next;
-  } while (hop != cpath);
-  return NULL;
-}
-
 /** Choose a suitable next hop for the circuit <b>circ</b>.
  * Append the hop info to circ->cpath.
  *
@@ -2709,33 +2673,11 @@ onion_extend_cpath(origin_circuit_t *circ)
             extend_info_describe(info),
             cur_len+1, build_state_get_exit_nickname(state));
 
-  onion_append_hop(&circ->cpath, info);
+  cpath_append_hop(&circ->cpath, info);
   extend_info_free(info);
   return 0;
 }
 
-/** Create a new hop, annotate it with information about its
- * corresponding router <b>choice</b>, and append it to the
- * end of the cpath <b>head_ptr</b>. */
-STATIC int
-onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
-{
-  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
-
-  /* link hop into the cpath, at the end. */
-  onion_append_to_cpath(head_ptr, hop);
-
-  hop->magic = CRYPT_PATH_MAGIC;
-  hop->state = CPATH_STATE_CLOSED;
-
-  hop->extend_info = extend_info_dup(choice);
-
-  hop->package_window = circuit_initial_package_window();
-  hop->deliver_window = CIRCWINDOW_START;
-
-  return 0;
-}
-
 /** Allocate a new extend_info object based on the various arguments. */
 extend_info_t *
 extend_info_new(const char *nickname,
@@ -2940,7 +2882,7 @@ extend_info_supports_ntor(const extend_info_t* ei)
 {
   tor_assert(ei);
   /* Valid ntor keys have at least one non-zero byte */
-  return !tor_mem_is_zero(
+  return !fast_mem_is_zero(
                           (const char*)ei->curve25519_onion_key.public_key,
                           CURVE25519_PUBKEY_LEN);
 }
diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h
index 969f5b5cc3..ad7d032cd4 100644
--- a/src/core/or/circuitbuild.h
+++ b/src/core/or/circuitbuild.h
@@ -34,9 +34,6 @@ int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);
 void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle);
 int circuit_extend(cell_t *cell, circuit_t *circ);
-int circuit_init_cpath_crypto(crypt_path_t *cpath,
-                              const char *key_data, size_t key_data_len,
-                              int reverse, int is_hs_v3);
 struct created_cell_t;
 int circuit_finish_handshake(origin_circuit_t *circ,
                              const struct created_cell_t *created_cell);
@@ -51,7 +48,6 @@ MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now,
 
 int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info);
 int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info);
-void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop);
 extend_info_t *extend_info_new(const char *nickname,
                                const char *rsa_id_digest,
                                const struct ed25519_public_key_t *ed_id,
@@ -83,8 +79,9 @@ void circuit_upgrade_circuits_from_guard_wait(void);
 #ifdef CIRCUITBUILD_PRIVATE
 STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
 STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
-                         smartlist_t *nodes);
-MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
+                         const smartlist_t *nodes);
+MOCK_DECL(STATIC int, count_acceptable_nodes, (const smartlist_t *nodes,
+                                               int direct));
 
 STATIC int onion_extend_cpath(origin_circuit_t *circ);
 
@@ -92,11 +89,6 @@ STATIC int
 onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
                       int is_hs_v3_rp_circuit);
 
-#if defined(TOR_UNIT_TESTS)
-unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 #endif /* defined(CIRCUITBUILD_PRIVATE) */
 
 #endif /* !defined(TOR_CIRCUITBUILD_H) */
diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c
index 03d495cdfd..4102cf3d4c 100644
--- a/src/core/or/circuitlist.c
+++ b/src/core/or/circuitlist.c
@@ -51,6 +51,7 @@
  * logic, which was originally circuit-focused.
  **/
 #define CIRCUITLIST_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 #include "lib/cc/torint.h"  /* TOR_PRIuSZ */
 
 #include "core/or/or.h"
@@ -61,11 +62,13 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/circuitstats.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/crypt_path.h"
 #include "core/mainloop/connection.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/crypt_ops/crypto_dh.h"
@@ -94,7 +97,10 @@
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
+
+#define OCIRC_EVENT_PRIVATE
+#include "core/or/ocirc_event.h"
 
 #include "ht.h"
 
@@ -127,7 +133,6 @@ static smartlist_t *circuits_pending_other_guards = NULL;
  * circuit_mark_for_close and which are waiting for circuit_about_to_free. */
 static smartlist_t *circuits_pending_close = NULL;
 
-static void circuit_free_cpath_node(crypt_path_t *victim);
 static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
 static void circuit_about_to_free_atexit(circuit_t *circ);
 static void circuit_about_to_free(circuit_t *circ);
@@ -481,6 +486,54 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
   }
 }
 
+/**
+ * Helper function to publish a message about events on an origin circuit
+ *
+ * Publishes a message to subscribers of origin circuit events, and
+ * sends the control event.
+ **/
+int
+circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                     int reason_code)
+{
+  ocirc_cevent_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  tor_assert(circ);
+
+  msg->gid = circ->global_identifier;
+  msg->evtype = tp;
+  msg->reason = reason_code;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_cevent_publish(msg);
+  return control_event_circuit_status(circ, tp, reason_code);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * circuit_set_state() calls this to notify subscribers about a change
+ * of the state of an origin circuit.  @a circ must be an origin
+ * circuit.
+ **/
+static void
+circuit_state_publish(const circuit_t *circ)
+{
+  ocirc_state_msg_t *msg = tor_malloc(sizeof(*msg));
+  const origin_circuit_t *ocirc;
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  ocirc = CONST_TO_ORIGIN_CIRCUIT(circ);
+  /* Only inbound OR circuits can be in this state, not origin circuits. */
+  tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING);
+
+  msg->gid = ocirc->global_identifier;
+  msg->state = circ->state;
+  msg->onehop = ocirc->build_state->onehop_tunnel;
+
+  ocirc_state_publish(msg);
+}
+
 /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing
  * it from lists as appropriate. */
 void
@@ -510,6 +563,8 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN)
     tor_assert(!circ->n_chan_create_cell);
   circ->state = state;
+  if (CIRCUIT_IS_ORIGIN(circ))
+    circuit_state_publish(circ);
 }
 
 /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for
@@ -767,6 +822,8 @@ circuit_purpose_to_controller_string(uint8_t purpose)
       return "PATH_BIAS_TESTING";
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "HS_VANGUARDS";
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "CIRCUIT_PADDING";
 
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
@@ -796,6 +853,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_CONTROLLER:
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
       return NULL;
 
     case CIRCUIT_PURPOSE_INTRO_POINT:
@@ -896,6 +954,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "Hidden service: Pre-built vanguard circuit";
 
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "Circuit kept open for padding";
+
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
       return buf;
@@ -931,6 +992,7 @@ init_circuit_base(circuit_t *circ)
 
   circ->package_window = circuit_initial_package_window();
   circ->deliver_window = CIRCWINDOW_START;
+  circuit_reset_sendme_randomness(circ);
   cell_queue_init(&circ->n_chan_cells);
 
   smartlist_add(circuit_get_global_list(), circ);
@@ -1092,7 +1154,7 @@ circuit_free_(circuit_t *circ)
 
     if (ocirc->build_state) {
         extend_info_free(ocirc->build_state->chosen_exit);
-        circuit_free_cpath_node(ocirc->build_state->pending_final_cpath);
+        cpath_free(ocirc->build_state->pending_final_cpath);
         cpath_ref_decref(ocirc->build_state->service_pending_final_cpath_ref);
     }
     tor_free(ocirc->build_state);
@@ -1171,11 +1233,20 @@ circuit_free_(circuit_t *circ)
    * "active" checks will be violated. */
   cell_queue_clear(&circ->n_chan_cells);
 
+  /* Cleanup possible SENDME state. */
+  if (circ->sendme_last_digests) {
+    SMARTLIST_FOREACH(circ->sendme_last_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(circ->sendme_last_digests);
+  }
+
   log_info(LD_CIRC, "Circuit %u (id: %" PRIu32 ") has been freed.",
            n_circ_id,
            CIRCUIT_IS_ORIGIN(circ) ?
               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
 
+  /* Free any circuit padding structures */
+  circpad_circuit_free_all_machineinfos(circ);
+
   if (should_free) {
     memwipe(mem, 0xAA, memlen); /* poison memory */
     tor_free(mem);
@@ -1207,10 +1278,10 @@ circuit_clear_cpath(origin_circuit_t *circ)
   while (cpath->next && cpath->next != head) {
     victim = cpath;
     cpath = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
-  circuit_free_cpath_node(cpath);
+  cpath_free(cpath);
 
   circ->cpath = NULL;
 }
@@ -1267,29 +1338,13 @@ circuit_free_all(void)
   HT_CLEAR(chan_circid_map, &chan_circid_map);
 }
 
-/** Deallocate space associated with the cpath node <b>victim</b>. */
-static void
-circuit_free_cpath_node(crypt_path_t *victim)
-{
-  if (!victim)
-    return;
-
-  relay_crypto_clear(&victim->crypto);
-  onion_handshake_state_release(&victim->handshake_state);
-  crypto_dh_free(victim->rend_dh_handshake_state);
-  extend_info_free(victim->extend_info);
-
-  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
-  tor_free(victim);
-}
-
 /** Release a crypt_path_reference_t*, which may be NULL. */
 static void
 cpath_ref_decref(crypt_path_reference_t *cpath_ref)
 {
   if (cpath_ref != NULL) {
     if (--(cpath_ref->refcount) == 0) {
-      circuit_free_cpath_node(cpath_ref->cpath);
+      cpath_free(cpath_ref->cpath);
       tor_free(cpath_ref);
     }
   }
@@ -2140,6 +2195,11 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
   tor_assert(line);
   tor_assert(file);
 
+  /* Check whether the circuitpadding subsystem wants to block this close */
+  if (circpad_marked_circuit_for_padding(circ, reason)) {
+    return;
+  }
+
   if (circ->marked_for_close) {
     log_warn(LD_BUG,
         "Duplicate call to circuit_mark_for_close at %s:%d"
@@ -2270,7 +2330,7 @@ circuit_about_to_free(circuit_t *circ)
     smartlist_remove(circuits_pending_other_guards, circ);
   }
   if (CIRCUIT_IS_ORIGIN(circ)) {
-    control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
+    circuit_event_status(TO_ORIGIN_CIRCUIT(circ),
      (circ->state == CIRCUIT_STATE_OPEN ||
       circ->state == CIRCUIT_STATE_GUARD_WAIT) ?
                                  CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
@@ -2374,13 +2434,9 @@ marked_circuit_free_cells(circuit_t *circ)
     return;
   }
   cell_queue_clear(&circ->n_chan_cells);
-  if (circ->n_mux)
-    circuitmux_clear_num_cells(circ->n_mux, circ);
   if (! CIRCUIT_IS_ORIGIN(circ)) {
     or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
     cell_queue_clear(&orcirc->p_chan_cells);
-    if (orcirc->p_mux)
-      circuitmux_clear_num_cells(orcirc->p_mux, circ);
   }
 }
 
@@ -2724,59 +2780,6 @@ circuits_handle_oom(size_t current_allocation)
              n_dirconns_killed);
 }
 
-/** Verify that cpath layer <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-void
-assert_cpath_layer_ok(const crypt_path_t *cp)
-{
-//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
-//  tor_assert(cp->port);
-  tor_assert(cp);
-  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
-  switch (cp->state)
-    {
-    case CPATH_STATE_OPEN:
-      relay_crypto_assert_ok(&cp->crypto);
-      FALLTHROUGH;
-    case CPATH_STATE_CLOSED:
-      /*XXXX Assert that there's no handshake_state either. */
-      tor_assert(!cp->rend_dh_handshake_state);
-      break;
-    case CPATH_STATE_AWAITING_KEYS:
-      /* tor_assert(cp->dh_handshake_state); */
-      break;
-    default:
-      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
-      tor_assert(0);
-    }
-  tor_assert(cp->package_window >= 0);
-  tor_assert(cp->deliver_window >= 0);
-}
-
-/** Verify that cpath <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-static void
-assert_cpath_ok(const crypt_path_t *cp)
-{
-  const crypt_path_t *start = cp;
-
-  do {
-    assert_cpath_layer_ok(cp);
-    /* layers must be in sequence of: "open* awaiting? closed*" */
-    if (cp != start) {
-      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      } else if (cp->state == CPATH_STATE_OPEN) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      }
-    }
-    cp = cp->next;
-    tor_assert(cp);
-  } while (cp != start);
-}
-
 /** Verify that circuit <b>c</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */
@@ -2838,7 +2841,7 @@ assert_circuit_ok,(const circuit_t *c))
                !smartlist_contains(circuits_pending_chans, c));
   }
   if (origin_circ && origin_circ->cpath) {
-    assert_cpath_ok(origin_circ->cpath);
+    cpath_assert_ok(origin_circ->cpath);
   }
   if (c->purpose == CIRCUIT_PURPOSE_REND_ESTABLISHED) {
     tor_assert(or_circ);
diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h
index b87c6a3667..80c1f7ac4e 100644
--- a/src/core/or/circuitlist.h
+++ b/src/core/or/circuitlist.h
@@ -14,6 +14,7 @@
 
 #include "lib/testsupport/testsupport.h"
 #include "feature/hs/hs_ident.h"
+#include "core/or/ocirc_event.h"
 
 /** Circuit state: I'm the origin, still haven't done all my handshakes. */
 #define CIRCUIT_STATE_BUILDING 0
@@ -91,31 +92,33 @@
 #define CIRCUIT_PURPOSE_C_HS_MAX_ 13
 /** This circuit is used for build time measurement only */
 #define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 14
-#define CIRCUIT_PURPOSE_C_MAX_ 14
+/** This circuit is being held open by circuit padding */
+#define CIRCUIT_PURPOSE_C_CIRCUIT_PADDING 15
+#define CIRCUIT_PURPOSE_C_MAX_ 15
 
-#define CIRCUIT_PURPOSE_S_HS_MIN_ 15
+#define CIRCUIT_PURPOSE_S_HS_MIN_ 16
 /** Hidden-service-side circuit purpose: at the service, waiting for
  * introductions. */
-#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 15
+#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 16
 /** Hidden-service-side circuit purpose: at the service, successfully
  * established intro. */
-#define CIRCUIT_PURPOSE_S_INTRO 16
+#define CIRCUIT_PURPOSE_S_INTRO 17
 /** Hidden-service-side circuit purpose: at the service, connecting to rend
  * point. */
-#define CIRCUIT_PURPOSE_S_CONNECT_REND 17
+#define CIRCUIT_PURPOSE_S_CONNECT_REND 18
 /** Hidden-service-side circuit purpose: at the service, rendezvous
  * established. */
-#define CIRCUIT_PURPOSE_S_REND_JOINED 18
+#define CIRCUIT_PURPOSE_S_REND_JOINED 19
 /** This circuit is used for uploading hsdirs */
-#define CIRCUIT_PURPOSE_S_HSDIR_POST 19
-#define CIRCUIT_PURPOSE_S_HS_MAX_ 19
+#define CIRCUIT_PURPOSE_S_HSDIR_POST 20
+#define CIRCUIT_PURPOSE_S_HS_MAX_ 20
 
 /** A testing circuit; not meant to be used for actual traffic. */
-#define CIRCUIT_PURPOSE_TESTING 20
+#define CIRCUIT_PURPOSE_TESTING 21
 /** A controller made this circuit and Tor should not use it. */
-#define CIRCUIT_PURPOSE_CONTROLLER 21
+#define CIRCUIT_PURPOSE_CONTROLLER 22
 /** This circuit is used for path bias probing only */
-#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 22
+#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 23
 
 /** This circuit is used for vanguards/restricted paths.
  *
@@ -123,9 +126,9 @@
  *  on-demand. When an HS operation needs to take place (e.g. connect to an
  *  intro point), these circuits are then cannibalized and repurposed to the
  *  actual needed HS purpose. */
-#define CIRCUIT_PURPOSE_HS_VANGUARDS 23
+#define CIRCUIT_PURPOSE_HS_VANGUARDS 24
 
-#define CIRCUIT_PURPOSE_MAX_ 23
+#define CIRCUIT_PURPOSE_MAX_ 24
 /** A catch-all for unrecognized purposes. Currently we don't expect
  * to make or see any circuits with this purpose. */
 #define CIRCUIT_PURPOSE_UNKNOWN 255
@@ -184,6 +187,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id);
 void channel_mark_circid_usable(channel_t *chan, circid_t id);
 time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id,
                                                   channel_t *chan);
+int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                         int reason_code);
 void circuit_set_state(circuit_t *circ, uint8_t state);
 void circuit_close_all_marked(void);
 int32_t circuit_initial_package_window(void);
@@ -213,7 +218,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void);
 void circuit_synchronize_written_or_bandwidth(const circuit_t *c,
                                               circuit_channel_direction_t dir);
 MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason,
-                                          int line, const char *file));
+                                          int line, const char *cfile));
 int circuit_get_cpath_len(origin_circuit_t *circ);
 int circuit_get_cpath_opened_len(const origin_circuit_t *);
 void circuit_clear_cpath(origin_circuit_t *circ);
@@ -225,7 +230,6 @@ int circuit_count_pending_on_channel(channel_t *chan);
 #define circuit_mark_for_close(c, reason)                               \
   circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__)
 
-void assert_cpath_layer_ok(const crypt_path_t *cp);
 MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c));
 void circuit_free_all(void);
 void circuits_handle_oom(size_t current_allocation);
diff --git a/src/core/or/circuitmux.c b/src/core/or/circuitmux.c
index 88f9ac7923..4b19a12e3c 100644
--- a/src/core/or/circuitmux.c
+++ b/src/core/or/circuitmux.c
@@ -79,6 +79,8 @@
 #include "core/or/destroy_cell_queue_st.h"
 #include "core/or/or_circuit_st.h"
 
+#include "lib/crypt_ops/crypto_util.h"
+
 /*
  * Private typedefs for circuitmux.c
  */
@@ -294,9 +296,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /* Clear n_mux */
-            circ->n_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else if (circ->magic == OR_CIRCUIT_MAGIC) {
@@ -309,12 +308,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /*
-             * It has a sensible p_chan and direction == CELL_DIRECTION_IN,
-             * so clear p_mux.
-             */
-            TO_OR_CIRCUIT(circ)->p_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else {
@@ -836,18 +829,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      */
     log_info(LD_CIRC,
              "Circuit %u on channel %"PRIu64 " was already attached to "
-             "cmux %p (trying to attach to %p)",
+             "(trying to attach to %p)",
              (unsigned)circ_id, (channel_id),
-             ((direction == CELL_DIRECTION_OUT) ?
-                circ->n_mux : TO_OR_CIRCUIT(circ)->p_mux),
              cmux);
 
     /*
      * The mux pointer on this circuit and the direction in result should
      * match; otherwise assert.
      */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == cmux);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
     tor_assert(hashent->muxinfo.direction == direction);
 
     /*
@@ -872,13 +861,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
              "Attaching circuit %u on channel %"PRIu64 " to cmux %p",
               (unsigned)circ_id, (channel_id), cmux);
 
-    /*
-     * Assert that the circuit doesn't already have a mux for this
-     * direction.
-     */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == NULL);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == NULL);
-
     /* Insert it in the map */
     hashent = tor_malloc_zero(sizeof(*hashent));
     hashent->chan_id = channel_id;
@@ -902,10 +884,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     HT_INSERT(chanid_circid_muxinfo_map, cmux->chanid_circid_map,
               hashent);
 
-    /* Set the circuit's mux for this direction */
-    if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux;
-    else TO_OR_CIRCUIT(circ)->p_mux = cmux;
-
     /* Update counters */
     ++(cmux->n_circuits);
     if (cell_count > 0) {
@@ -993,14 +971,14 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
 
     /* Consistency check: the direction must match the direction searched */
     tor_assert(last_searched_direction == hashent->muxinfo.direction);
-    /* Clear the circuit's mux for this direction */
-    if (last_searched_direction == CELL_DIRECTION_OUT) circ->n_mux = NULL;
-    else TO_OR_CIRCUIT(circ)->p_mux = NULL;
 
     /* Now remove it from the map */
     HT_REMOVE(chanid_circid_muxinfo_map, cmux->chanid_circid_map, hashent);
 
-    /* Free the hash entry */
+    /* Wipe and free the hash entry */
+    // This isn't sensitive, but we want to be sure to know if we're accessing
+    // this accidentally.
+    memwipe(hashent, 0xef, sizeof(*hashent));
     tor_free(hashent);
   }
 }
@@ -1361,4 +1339,3 @@ circuitmux_compare_muxes, (circuitmux_t *cmux_1, circuitmux_t *cmux_2))
     return 0;
   }
 }
-
diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c
index 3f83c3fd5a..07f83af0f1 100644
--- a/src/core/or/circuitmux_ewma.c
+++ b/src/core/or/circuitmux_ewma.c
@@ -38,6 +38,7 @@
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
 #include "app/config/or_options_st.h"
 
@@ -147,7 +148,9 @@ TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol)
 {
   if (!pol) return NULL;
   else {
-    tor_assert(pol->magic == EWMA_POL_DATA_MAGIC);
+    tor_assertf(pol->magic == EWMA_POL_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_DATA_MAGIC);
     return DOWNCAST(ewma_policy_data_t, pol);
   }
 }
@@ -162,7 +165,9 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
 {
   if (!pol) return NULL;
   else {
-    tor_assert(pol->magic == EWMA_POL_CIRC_DATA_MAGIC);
+    tor_assertf(pol->magic == EWMA_POL_CIRC_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_CIRC_DATA_MAGIC);
     return DOWNCAST(ewma_policy_circ_data_t, pol);
   }
 }
@@ -295,6 +300,7 @@ ewma_free_cmux_data(circuitmux_t *cmux,
   pol = TO_EWMA_POL_DATA(pol_data);
 
   smartlist_free(pol->active_circuit_pqueue);
+  memwipe(pol, 0xda, sizeof(ewma_policy_data_t));
   tor_free(pol);
 }
 
@@ -361,7 +367,7 @@ ewma_free_circ_data(circuitmux_t *cmux,
   if (!pol_circ_data) return;
 
   cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
-
+  memwipe(cdata, 0xdc, sizeof(ewma_policy_circ_data_t));
   tor_free(cdata);
 }
 
diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c
new file mode 100644
index 0000000000..7f761fed6b
--- /dev/null
+++ b/src/core/or/circuitpadding.c
@@ -0,0 +1,3099 @@
+/* Copyright (c) 2017 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.c
+ * \brief Circuit-level padding implementation
+ *
+ * \details
+ *
+ * This file implements Tor proposal 254 "Padding Negotiation" which is heavily
+ * inspired by the paper "Toward an Efficient Website Fingerprinting Defense"
+ * by M. Juarez, M. Imani, M. Perry, C. Diaz, M. Wright.
+ *
+ * In particular the code in this file describes mechanisms for clients to
+ * negotiate various types of circuit-level padding from relays.
+ *
+ * Each padding type is described by a state machine (circpad_machine_spec_t),
+ * which is also referred as a "padding machine" in this file.  Currently,
+ * these state machines are hardcoded in the source code (e.g. see
+ * circpad_machines_init()), but in the future we will be able to
+ * serialize them in the torrc or the consensus.
+ *
+ * As specified by prop#254, clients can negotiate padding with relays by using
+ * PADDING_NEGOTIATE cells. After successful padding negotiation, padding
+ * machines are assigned to the circuit in their mutable form as a
+ * circpad_machine_runtime_t.
+ *
+ * Each state of a padding state machine can be either:
+ * - A histogram that specifies inter-arrival padding delays.
+ * - Or a parametrized probability distribution that specifies inter-arrival
+ *   delays (see circpad_distribution_type_t).
+ *
+ * Padding machines start from the START state and finish with the END
+ * state. They can transition between states using the events in
+ * circpad_event_t.
+ *
+ * When a padding machine reaches the END state, it gets wiped from the circuit
+ * so that other padding machines can take over if needed (see
+ * circpad_machine_spec_transitioned_to_end()).
+ *
+ ****************************
+ * General notes:
+ *
+ * All used machines should be heap allocated and placed into
+ * origin_padding_machines/relay_padding_machines so that they get correctly
+ * cleaned up by the circpad_free_all() function.
+ **/
+
+#define CIRCUITPADDING_PRIVATE
+
+#include <math.h>
+#include "lib/math/fp.h"
+#include "lib/math/prob_distr.h"
+#include "core/or/or.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/relay.h"
+#include "feature/stats/rephist.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/channel.h"
+
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/crypto/relay_crypto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "app/config/config.h"
+
+static inline circpad_circuit_state_t circpad_circuit_state(
+                                        origin_circuit_t *circ);
+static void circpad_setup_machine_on_circ(circuit_t *on_circ,
+                                        const circpad_machine_spec_t *machine);
+static double circpad_distribution_sample(circpad_distribution_t dist);
+
+static inline void circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi);
+
+/** Cached consensus params */
+static uint8_t circpad_padding_disabled;
+static uint8_t circpad_padding_reduced;
+static uint8_t circpad_global_max_padding_percent;
+static uint16_t circpad_global_allowed_cells;
+static uint16_t circpad_max_circ_queued_cells;
+
+/** Global cell counts, for rate limiting */
+static uint64_t circpad_global_padding_sent;
+static uint64_t circpad_global_nonpadding_sent;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 1 (ie: are for client side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *origin_padding_machines = NULL;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 0 (ie: are for relay side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *relay_padding_machines = NULL;
+
+/** Loop over the current padding state machines using <b>loop_var</b> as the
+ *  loop variable. */
+#define FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                         \
+  STMT_BEGIN                                                             \
+  for (int loop_var = 0; loop_var < CIRCPAD_MAX_MACHINES; loop_var++) {
+#define FOR_EACH_CIRCUIT_MACHINE_END } STMT_END ;
+
+/** Loop over the current active padding state machines using <b>loop_var</b>
+ *  as the loop variable. If a machine is not active, skip it. */
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(loop_var, circ)            \
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                               \
+  if (!(circ)->padding_info[loop_var])                           \
+    continue;
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ;
+
+/**
+ * Free the machineinfo at an index
+ */
+static void
+circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx)
+{
+  if (circ->padding_info[idx]) {
+    log_fn(LOG_INFO,LD_CIRC, "Freeing padding info idx %d on circuit %u (%d)",
+           idx, CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+           circ->purpose);
+
+    tor_free(circ->padding_info[idx]->histogram);
+    timer_free(circ->padding_info[idx]->padding_timer);
+    tor_free(circ->padding_info[idx]);
+  }
+}
+
+/**
+ * Return true if circpad has decided to hold the circuit open for additional
+ * padding. This function is used to take and retain ownership of certain
+ * types of circuits that have padding machines on them, that have been passed
+ * to circuit_mark_for_close().
+ *
+ * circuit_mark_for_close() calls this function to ask circpad if any padding
+ * machines want to keep the circuit open longer to pad.
+ *
+ * Any non-measurement circuit that was closed for a normal, non-error reason
+ * code may be held open for up to CIRCPAD_DELAY_INFINITE microseconds between
+ * network-driven cell events.
+ *
+ * After CIRCPAD_DELAY_INFINITE microseconds of silence on a circuit, this
+ * function will no longer hold it open (it will return 0 regardless of
+ * what the machines ask for, and thus circuit_expire_old_circuits_clientside()
+ * will close the circuit after roughly 1.25hr of idle time, maximum,
+ * regardless of the padding machine state.
+ */
+int
+circpad_marked_circuit_for_padding(circuit_t *circ, int reason)
+{
+  /* If the circuit purpose is measurement or path bias, don't
+   * hold it open */
+  if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING ||
+      circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+    return 0;
+  }
+
+  /* If the circuit is closed for any reason other than these three valid,
+   * client-side close reasons, do not try to keep it open. It is probably
+   * damaged or unusable. Note this is OK with vanguards because
+   * controller-closed circuits have REASON=REQUESTED, so vanguards-closed
+   * circuits will not be held open (we want them to close ASAP). */
+  if (!(reason == END_CIRC_REASON_NONE ||
+        reason == END_CIRC_REASON_FINISHED ||
+        reason == END_CIRC_REASON_IP_NOW_REDUNDANT)) {
+    return 0;
+  }
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    circpad_machine_runtime_t *mi = circ->padding_info[i];
+    if (!mi) {
+      continue; // No padding runtime info; check next machine
+    }
+
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+
+    /* If we're in END state (NULL here), then check next machine */
+    if (!state) {
+      continue; // check next machine
+    }
+
+    /* If the machine does not want to control the circuit close itself, then
+     * check the next machine */
+    if (!circ->padding_machine[i]->manage_circ_lifetime) {
+      continue; // check next machine
+    }
+
+    /* If the machine has reached the END state, we can close. Check next
+     * machine. */
+    if (mi->current_state == CIRCPAD_STATE_END) {
+      continue; // check next machine
+    }
+
+    log_info(LD_CIRC, "Circuit %d is not marked for close because of a "
+             "pending padding machine in index %d.",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0, i);
+
+    /* If the machine has had no network events at all within the
+     * last circpad_delay_t timespan, it's in some deadlock state.
+     * Tell circuit_mark_for_close() that we don't own it anymore.
+     * This will allow circuit_expire_old_circuits_clientside() to
+     * close it.
+     */
+    if (circ->padding_info[i]->last_cell_time_sec +
+        (time_t)CIRCPAD_DELAY_MAX_SECS < approx_time()) {
+      log_notice(LD_BUG, "Circuit %d was not marked for close because of a "
+               "pending padding machine in index %d for over an hour. "
+               "Circuit is a %s",
+               CIRCUIT_IS_ORIGIN(circ) ?
+               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+               i, circuit_purpose_to_string(circ->purpose));
+
+      return 0; // abort timer reached; mark the circuit for close now
+    }
+
+    /* If we weren't marked dirty yet, let's pretend we're dirty now.
+     * ("Dirty" means that a circuit has been used for application traffic
+     * by Tor.. Dirty circuits have different expiry times, and are not
+     * considered in counts of built circuits, etc. By claiming that we're
+     * dirty, the rest of Tor will make decisions as if we were actually
+     * used by application data.
+     *
+     * This is most important for circuit_expire_old_circuits_clientside(),
+     * where we want that function to expire us after the padding machine
+     * has shut down, but using the MaxCircuitDirtiness timer instead of
+     * the idle circuit timer (again, we want this because we're not
+     * supposed to look idle to Guard nodes that can see our lifespan). */
+    if (!circ->timestamp_dirty)
+      circ->timestamp_dirty = approx_time();
+
+    /* Take ownership of the circuit */
+    circuit_change_purpose(circ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+    return 1;
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+
+  return 0; // No machine wanted to keep the circuit open; mark for close
+}
+
+/**
+ * Free all the machineinfos in <b>circ</b> that match <b>machine_num</b>.
+ *
+ * Returns true if any machineinfos with that number were freed.
+ * False otherwise. */
+static int
+free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num)
+{
+  int found = 0;
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    if (circ->padding_machine[i] &&
+        circ->padding_machine[i]->machine_num == machine_num) {
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      circ->padding_machine[i] = NULL;
+      found = 1;
+    }
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return found;
+}
+
+/**
+ * Free all padding machines and mutable info associated with circuit
+ */
+void
+circpad_circuit_free_all_machineinfos(circuit_t *circ)
+{
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    circpad_circuit_machineinfo_free_idx(circ, i);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Allocate a new mutable machineinfo structure.
+ */
+STATIC circpad_machine_runtime_t *
+circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index)
+{
+  circpad_machine_runtime_t *mi =
+    tor_malloc_zero(sizeof(circpad_machine_runtime_t));
+  mi->machine_index = machine_index;
+  mi->on_circ = on_circ;
+  mi->last_cell_time_sec = approx_time();
+
+  return mi;
+}
+
+/**
+ * Return the circpad_state_t for the current state based on the
+ * mutable info.
+ *
+ * This function returns NULL when the machine is in the end state or in an
+ * invalid state.
+ */
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    return NULL;
+  } else if (BUG(mi->current_state >= machine->num_states)) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Invalid circuit padding state %d",
+           mi->current_state);
+
+    return NULL;
+  }
+
+  return &machine->states[mi->current_state];
+}
+
+/**
+ * Get the lower bound of a histogram bin.
+ *
+ * You can obtain the upper bound using histogram_get_bin_upper_bound().
+ *
+ * This function can also be called with 'bin' set to a value equal or greater
+ * than histogram_len in which case the infinity bin is chosen and
+ * CIRCPAD_DELAY_INFINITE is returned.
+ */
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* The infinity bin has an upper bound of infinity, so make sure we return
+   * that if they ask for it. */
+  if (bin > CIRCPAD_INFINITY_BIN(state)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  return state->histogram_edges[bin] + rtt_add_usec;
+}
+
+/**
+ * Like circpad_histogram_bin_to_usec() but return the upper bound of bin.
+ * (The upper bound is included in the bin.)
+ */
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  return circpad_histogram_bin_to_usec(mi, bin+1) - 1;
+}
+
+/** Return the midpoint of the histogram bin <b>bin_index</b>. */
+static circpad_delay_t
+circpad_get_histogram_bin_midpoint(const circpad_machine_runtime_t *mi,
+                           int bin_index)
+{
+  circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index);
+  circpad_delay_t right_bound = histogram_get_bin_upper_bound(mi, bin_index);
+
+  return left_bound + (right_bound - left_bound)/2;
+}
+
+/**
+ * Return the bin that contains the usec argument.
+ * "Contains" is defined as us in [lower, upper).
+ *
+ * This function will never return the infinity bin (histogram_len-1), in order
+ * to simplify the rest of the code, so if a usec is provided that falls above
+ * the highest non-infinity bin, that bin index will be returned.
+ */
+STATIC circpad_hist_index_t
+circpad_histogram_usec_to_bin(const circpad_machine_runtime_t *mi,
+                              circpad_delay_t usec)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+  circpad_hist_index_t bin;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return 0;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  /* Walk through the bins and check the upper bound of each bin, if 'usec' is
+   * less-or-equal to that, return that bin. If rtt_estimate is enabled then
+   * add that to the upper bound of each bin.
+   *
+   * We don't want to return the infinity bin here, so don't go there. */
+  for (bin = 0 ; bin < CIRCPAD_INFINITY_BIN(state) ; bin++) {
+    if (usec <= histogram_get_bin_upper_bound(mi, bin) + rtt_add_usec) {
+      return bin;
+    }
+  }
+
+  /* We don't want to return the infinity bin here, so if we still didn't find
+   * the right bin, return the highest non-infinity bin */
+  return CIRCPAD_INFINITY_BIN(state)-1;
+}
+
+/**
+ * Return true if the machine supports token removal.
+ *
+ * Token removal is equivalent to having a mutable histogram in the
+ * circpad_machine_runtime_t mutable info. So while we're at it,
+ * let's assert that everything is consistent between the mutable
+ * runtime and the readonly machine spec.
+ */
+static inline int
+circpad_is_token_removal_supported(circpad_machine_runtime_t *mi)
+{
+  /* No runtime histogram == no token removal */
+  if (mi->histogram == NULL) {
+    /* Machines that don't want token removal are trying to avoid
+     * potentially expensive mallocs, extra memory accesses, and/or
+     * potentially expensive monotime calls. Let's minimize checks
+     * and keep this path fast. */
+    tor_assert_nonfatal(mi->histogram_len == 0);
+    return 0;
+  } else {
+    /* Machines that do want token removal are less sensitive to performance.
+     * Let's spend some time to check that our state is consistent and sane */
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return 1;
+    }
+    tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE);
+    tor_assert_nonfatal(state->histogram_len == mi->histogram_len);
+    tor_assert_nonfatal(mi->histogram_len != 0);
+    return 1;
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * This function frees any token bins allocated from a previous state
+ *
+ * Called after a state transition, or if the bins are empty.
+ */
+STATIC void
+circpad_machine_setup_tokens(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+
+  /* If this state doesn't exist, or doesn't have token removal,
+   * free any previous state's runtime histogram, and bail.
+   *
+   * If we don't have a token removal strategy, we also don't need a runtime
+   * histogram and we rely on the immutable one in machine_spec_t. */
+  if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) {
+    if (mi->histogram) {
+      tor_free(mi->histogram);
+      mi->histogram = NULL;
+      mi->histogram_len = 0;
+    }
+    return;
+  }
+
+  /* Try to avoid re-mallocing if we don't really need to */
+  if (!mi->histogram || (mi->histogram
+          && mi->histogram_len != state->histogram_len)) {
+    tor_free(mi->histogram); // null ok
+    mi->histogram = tor_malloc_zero(sizeof(circpad_hist_token_t)
+                                    *state->histogram_len);
+  }
+  mi->histogram_len = state->histogram_len;
+
+  memcpy(mi->histogram, state->histogram,
+         sizeof(circpad_hist_token_t)*state->histogram_len);
+}
+
+/**
+ * Choose a length for this state (in cells), if specified.
+ */
+static void
+circpad_choose_state_length(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  double length;
+
+  if (!state || state->length_dist.type == CIRCPAD_DIST_NONE) {
+    mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+    return;
+  }
+
+  length = circpad_distribution_sample(state->length_dist);
+  length = MAX(0, length);
+  length += state->start_length;
+
+  if (state->max_length) {
+    length = MIN(length, state->max_length);
+  }
+
+  mi->state_length = clamp_double_to_int64(length);
+
+  log_info(LD_CIRC, "State length sampled to %"PRIu64" for circuit %u",
+      mi->state_length, CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+}
+
+/**
+ * Sample a value from our iat_dist, and clamp it safely
+ * to circpad_delay_t.
+ *
+ * Before returning, add <b>delay_shift</b> (can be zero) to the sampled value.
+ */
+static circpad_delay_t
+circpad_distribution_sample_iat_delay(const circpad_state_t *state,
+                                      circpad_delay_t delay_shift)
+{
+  double val = circpad_distribution_sample(state->iat_dist);
+  /* These comparisons are safe, because the output is in the range
+   * [0, 2**32), and double has a precision of 53 bits. */
+  /* We want a positive sample value */
+  val = MAX(0, val);
+  /* Respect the maximum sample setting */
+  val = MIN(val, state->dist_max_sample_usec);
+
+  /* Now apply the shift:
+   * This addition is exact: val is at most 2**32-1, delay_shift is at most
+   * 2**32-1, and doubles have a precision of 53 bits. */
+  val += delay_shift;
+
+  /* Clamp the distribution at infinite delay val */
+  return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE);
+}
+
+/**
+ * Sample an expected time-until-next-packet delay from the histogram or
+ * probability distribution.
+ *
+ * A bin of the histogram is chosen with probability proportional to the number
+ * of tokens in each bin, and then a time value is chosen uniformly from that
+ * bin's [start,end) time range.
+ */
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  const circpad_hist_token_t *histogram = NULL;
+  circpad_hist_index_t curr_bin = 0;
+  circpad_delay_t bin_start, bin_end;
+  /* These three must all be larger than circpad_hist_token_t, because
+   * we sum several circpad_hist_token_t values across the histogram */
+  uint64_t curr_weight = 0;
+  uint64_t histogram_total_tokens = 0;
+  uint64_t bin_choice;
+
+  tor_assert(state);
+
+  if (state->iat_dist.type != CIRCPAD_DIST_NONE) {
+    /* Sample from a fixed IAT distribution and return */
+    circpad_delay_t iat_delay_shift = state->use_rtt_estimate ?
+      mi->rtt_estimate_usec + state->dist_added_shift_usec :
+      state->dist_added_shift_usec;
+    return circpad_distribution_sample_iat_delay(state, iat_delay_shift);
+  } else if (circpad_is_token_removal_supported(mi)) {
+    histogram = mi->histogram;
+    for (circpad_hist_index_t b = 0; b < state->histogram_len; b++)
+      histogram_total_tokens += histogram[b];
+  } else {
+    /* We have a histogram, but it's immutable */
+    histogram = state->histogram;
+    histogram_total_tokens = state->histogram_total_tokens;
+  }
+
+  /* If we are out of tokens, don't schedule padding. */
+  if (!histogram_total_tokens) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  bin_choice = crypto_fast_rng_get_uint64(get_thread_fast_rng(),
+                                          histogram_total_tokens);
+
+  /* Skip all the initial zero bins */
+  while (!histogram[curr_bin]) {
+    curr_bin++;
+  }
+  curr_weight = histogram[curr_bin];
+
+  // TODO: This is not constant-time. Pretty sure we don't
+  // really need it to be, though.
+  while (curr_weight < bin_choice) {
+    curr_bin++;
+    /* It should be impossible to run past the end of the histogram */
+    if (BUG(curr_bin >= state->histogram_len)) {
+      return CIRCPAD_DELAY_INFINITE;
+    }
+    curr_weight += histogram[curr_bin];
+  }
+
+  /* Do some basic checking of the current bin we are in */
+  if (BUG(curr_bin >= state->histogram_len) ||
+      BUG(histogram[curr_bin] == 0)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  // Store this index to remove the token upon callback.
+  if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) {
+    mi->chosen_bin = curr_bin;
+  }
+
+  if (curr_bin >= CIRCPAD_INFINITY_BIN(state)) {
+    if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE &&
+        mi->histogram[curr_bin] > 0) {
+      mi->histogram[curr_bin]--;
+    }
+
+    // Infinity: Don't send a padding packet. Wait for a real packet
+    // and then see if our bins are empty or what else we should do.
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state));
+
+  bin_start = circpad_histogram_bin_to_usec(mi, curr_bin);
+  /* We don't need to reduct 1 from the upper bound because the random range
+   * function below samples from [bin_start, bin_end) */
+  bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1);
+
+  /* Bin edges are monotonically increasing so this is a bug. Handle it. */
+  if (BUG(bin_start >= bin_end)) {
+    return bin_start;
+  }
+
+  return (circpad_delay_t)crypto_fast_rng_uint64_range(get_thread_fast_rng(),
+                                                       bin_start, bin_end);
+}
+
+/**
+ * Sample a value from the specified probability distribution.
+ *
+ * Uses functions from src/lib/math/prob_distr.c .
+ */
+static double
+circpad_distribution_sample(circpad_distribution_t dist)
+{
+  log_fn(LOG_DEBUG,LD_CIRC, "Sampling delay with distribution %d",
+         dist.type);
+
+  switch (dist.type) {
+    case CIRCPAD_DIST_NONE:
+      {
+        /* We should not get in here like this */
+        tor_assert_nonfatal_unreached();
+        return 0;
+      }
+    case CIRCPAD_DIST_UNIFORM:
+      {
+        // param2 is upper bound, param1 is lower
+        const struct uniform my_uniform = {
+          .base = UNIFORM(my_uniform),
+          .a = dist.param1,
+          .b = dist.param2,
+        };
+        return dist_sample(&my_uniform.base);
+      }
+    case CIRCPAD_DIST_LOGISTIC:
+      {
+      /* param1 is Mu, param2 is sigma. */
+        const struct logistic my_logistic = {
+          .base = LOGISTIC(my_logistic),
+          .mu = dist.param1,
+          .sigma = dist.param2,
+        };
+        return dist_sample(&my_logistic.base);
+      }
+    case CIRCPAD_DIST_LOG_LOGISTIC:
+      {
+        /* param1 is Alpha, param2 is 1.0/Beta */
+        const struct log_logistic my_log_logistic = {
+          .base = LOG_LOGISTIC(my_log_logistic),
+          .alpha = dist.param1,
+          .beta = dist.param2,
+        };
+        return dist_sample(&my_log_logistic.base);
+      }
+    case CIRCPAD_DIST_GEOMETRIC:
+      {
+        /* param1 is 'p' (success probability) */
+        const struct geometric my_geometric = {
+          .base = GEOMETRIC(my_geometric),
+          .p = dist.param1,
+        };
+        return dist_sample(&my_geometric.base);
+      }
+    case CIRCPAD_DIST_WEIBULL:
+      {
+        /* param1 is k, param2 is Lambda */
+        const struct weibull my_weibull = {
+          .base = WEIBULL(my_weibull),
+          .k = dist.param1,
+          .lambda = dist.param2,
+        };
+        return dist_sample(&my_weibull.base);
+      }
+    case CIRCPAD_DIST_PARETO:
+      {
+        /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */
+        const struct genpareto my_genpareto = {
+          .base = GENPARETO(my_genpareto),
+          .mu = 0,
+          .sigma = dist.param1,
+          .xi = dist.param2,
+        };
+        return dist_sample(&my_genpareto.base);
+      }
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * Find the index of the first bin whose upper bound is
+ * greater than the target, and that has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_higher_index(const circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  /* Don't remove from the infinity bin */
+  for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) {
+    if (mi->histogram[bin] &&
+        histogram_get_bin_upper_bound(mi, bin) >= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return mi->histogram_len;
+}
+
+/**
+ * Find the index of the first bin whose lower bound is lower or equal to
+ * <b>target_bin_usec</b>, and that still has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_lower_index(const circpad_machine_runtime_t *mi,
+                                  circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  for (; bin >= 0; bin--) {
+    if (mi->histogram[bin] &&
+        circpad_histogram_bin_to_usec(mi, bin) <= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return -1;
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * greater than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                    circpad_delay_t target_bin_usec)
+{
+  /* We need to remove the token from the first bin
+   * whose upper bound is greater than the target, and that
+   * has tokens remaining. */
+  circpad_hist_index_t bin = circpad_machine_first_higher_index(mi,
+                                                     target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * lower than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_machine_first_lower_index(mi,
+          target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/* Helper macro: Ensure that the bin has tokens available, and BUG out of the
+ * function if it's not the case. */
+#define ENSURE_BIN_CAPACITY(bin_index) \
+  if (BUG(mi->histogram[bin_index] == 0)) {                   \
+    return;                                                   \
+  }
+
+/**
+ * Remove a token from the closest non-empty bin to the target.
+ *
+ * If use_usec is true, measure "closest" in terms of the next closest bin
+ * midpoint.
+ *
+ * If it is false, use bin index distance only.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                     circpad_delay_t target_bin_usec,
+                                     bool use_usec)
+{
+  circpad_hist_index_t lower, higher, current;
+  circpad_hist_index_t bin_to_remove = -1;
+
+  lower = circpad_machine_first_lower_index(mi, target_bin_usec);
+  higher = circpad_machine_first_higher_index(mi, target_bin_usec);
+  current = circpad_histogram_usec_to_bin(mi, target_bin_usec);
+
+  /* Sanity check the results */
+  if (BUG(lower > current) || BUG(higher < current)) {
+    return;
+  }
+
+  /* Take care of edge cases first */
+  if (higher == mi->histogram_len && lower == -1) {
+    /* All bins are empty */
+    return;
+  } else if (higher == mi->histogram_len) {
+    /* All higher bins are empty */
+    ENSURE_BIN_CAPACITY(lower);
+    mi->histogram[lower]--;
+    return;
+  } else if (lower == -1) {
+    /* All lower bins are empty */
+    ENSURE_BIN_CAPACITY(higher);
+    mi->histogram[higher]--;
+    return;
+  }
+
+  /* Now handle the intermediate cases */
+  if (use_usec) {
+    /* Find the closest bin midpoint to the target */
+    circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower);
+    circpad_delay_t higher_usec =
+      circpad_get_histogram_bin_midpoint(mi, higher);
+
+    if (target_bin_usec < lower_usec) {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    } else if (target_bin_usec > higher_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else if (target_bin_usec-lower_usec > higher_usec-target_bin_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    }
+    mi->histogram[bin_to_remove]--;
+    log_debug(LD_CIRC, "Removing token from bin %d", bin_to_remove);
+    return;
+  } else {
+    if (current - lower > higher - current) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      mi->histogram[higher]--;
+      return;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      mi->histogram[lower]--;
+      return;
+    }
+  }
+}
+
+#undef ENSURE_BIN_CAPACITY
+
+/**
+ * Remove a token from the exact bin corresponding to the target.
+ *
+ * If it is empty, do nothing.
+ *
+ * Used for histograms with token removal.
+ */
+static void
+circpad_machine_remove_exact(circpad_machine_runtime_t *mi,
+                             circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+          target_bin_usec);
+
+  if (mi->histogram[bin] > 0)
+    mi->histogram[bin]--;
+}
+
+/**
+ * Check our state's cell limit count and tokens.
+ *
+ * Returns 1 if either limits are hit and we decide to change states,
+ * otherwise returns 0.
+ */
+static circpad_decision_t
+check_machine_token_supply(circpad_machine_runtime_t *mi)
+{
+  uint32_t histogram_total_tokens = 0;
+
+  /* Check if bins empty. This requires summing up the current mutable
+   * machineinfo histogram token total and checking if it is zero.
+   * Machineinfo does not keep a running token count. We're assuming the
+   * extra space is not worth this short loop iteration.
+   *
+   * We also do not count infinity bin in histogram totals.
+   */
+  if (circpad_is_token_removal_supported(mi)) {
+    for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++)
+      histogram_total_tokens += mi->histogram[b];
+
+    /* If we change state, we're done */
+    if (histogram_total_tokens == 0) {
+      if (circpad_internal_event_bins_empty(mi) == CIRCPAD_STATE_CHANGED)
+        return CIRCPAD_STATE_CHANGED;
+    }
+  }
+
+  if (mi->state_length == 0) {
+    return circpad_internal_event_state_length_up(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Count that a padding packet was sent.
+ *
+ * This updates our state length count, our machine rate limit counts,
+ * and if token removal is used, decrements the histogram.
+ */
+static inline void
+circpad_machine_count_padding_sent(circpad_machine_runtime_t *mi)
+{
+  /* If we have a valid state length bound, consider it */
+  if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE &&
+      !BUG(mi->state_length <= 0)) {
+    mi->state_length--;
+  }
+
+  /*
+   * Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct.
+   */
+  mi->padding_sent++;
+  if (mi->padding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  circpad_global_padding_sent++;
+
+  /* If we have a mutable histogram, reduce the token count from
+   * the chosen padding bin (this assumes we always send padding
+   * when we intended to). */
+  if (circpad_is_token_removal_supported(mi)) {
+    /* Check array bounds and token count before removing */
+    if (!BUG(mi->chosen_bin >= mi->histogram_len) &&
+        !BUG(mi->histogram[mi->chosen_bin] == 0)) {
+      mi->histogram[mi->chosen_bin]--;
+    }
+  }
+}
+
+/**
+ * Count a nonpadding packet as being sent.
+ *
+ * This function updates our overhead accounting variables, as well
+ * as decrements the state limit packet counter, if the latter was
+ * flagged as applying to non-padding as well.
+ */
+static inline void
+circpad_machine_count_nonpadding_sent(circpad_machine_runtime_t *mi)
+{
+  /* Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct. */
+  mi->nonpadding_sent++;
+  if (mi->nonpadding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  /* Update any state packet length limits that apply */
+  circpad_machine_update_state_length_for_nonpadding(mi);
+
+  /* Remove a token from the histogram, if applicable */
+  circpad_machine_remove_token(mi);
+}
+
+/**
+ * Decrement the state length counter for a non-padding packet.
+ *
+ * Only updates the state length if we're using that feature, we
+ * have a state, and the machine wants to count non-padding packets
+ * towards the state length.
+ */
+static inline void
+circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+
+  if (mi->state_length == CIRCPAD_STATE_LENGTH_INFINITE)
+    return;
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+
+  /* If we're enforcing a state length on non-padding packets,
+   * decrement it */
+  if (state->length_includes_nonpadding &&
+      mi->state_length > 0) {
+    mi->state_length--;
+  }
+}
+
+/**
+ * When a non-padding packet arrives, remove a token from the bin
+ * corresponding to the delta since last sent packet. If that bin
+ * is empty, choose a token based on the specified removal strategy
+ * in the state machine.
+ */
+STATIC void
+circpad_machine_remove_token(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+  circpad_time_t current_time;
+  circpad_delay_t target_bin_usec;
+
+  /* Dont remove any tokens if there was no padding scheduled */
+  if (!mi->padding_scheduled_at_usec) {
+    return;
+  }
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+  /* Don't remove any tokens if we're not doing token removal */
+  if (state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE)
+    return;
+
+  current_time = monotime_absolute_usec();
+
+  /* If we have scheduled padding some time in the future, we want to see what
+     bin we are in at the current time */
+  target_bin_usec = (circpad_delay_t)
+                  MIN((current_time - mi->padding_scheduled_at_usec),
+                      CIRCPAD_DELAY_INFINITE-1);
+
+  /* We are treating this non-padding cell as a padding cell, so we cancel
+     padding timer, if present. */
+  mi->padding_scheduled_at_usec = 0;
+  if (mi->is_padding_timer_scheduled) {
+    mi->is_padding_timer_scheduled = 0;
+    timer_disable(mi->padding_timer);
+  }
+
+  /* Perform the specified token removal strategy */
+  switch (state->token_removal) {
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 1);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 0);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_LOWER:
+      circpad_machine_remove_lower_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_HIGHER:
+      circpad_machine_remove_higher_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_EXACT:
+      circpad_machine_remove_exact(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_NONE:
+    default:
+      tor_assert_nonfatal_unreached();
+      log_warn(LD_BUG, "Circpad: Unknown token removal strategy %d",
+               state->token_removal);
+      break;
+  }
+}
+
+/**
+ * Send a relay command with a relay cell payload on a circuit to
+ * the particular hopnum.
+ *
+ * Hopnum starts at 1 (1=guard, 2=middle, 3=exit, etc).
+ *
+ * Payload may be null.
+ *
+ * Returns negative on error, 0 on success.
+ */
+MOCK_IMPL(STATIC signed_error_t,
+circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len))
+{
+  crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum);
+  signed_error_t ret;
+
+  /* Check that the cpath has the target hop */
+  if (!target_hop) {
+    log_fn(LOG_WARN, LD_BUG, "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier, circuit_get_cpath_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Check that the target hop is opened */
+  if (target_hop->state != CPATH_STATE_OPEN) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier,
+           circuit_get_cpath_opened_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Send the drop command to the second hop */
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ), relay_command,
+                                     (const char*)payload, payload_len,
+                                     target_hop);
+  return ret;
+}
+
+/**
+ * Callback helper to send a padding cell.
+ *
+ * This helper is called after our histogram-sampled delay period passes
+ * without another packet being sent first. If a packet is sent before this
+ * callback happens, it is canceled. So when we're called here, send padding
+ * right away.
+ *
+ * If sending this padding cell forced us to transition states return
+ * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED.
+ */
+circpad_decision_t
+circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi)
+{
+  circuit_t *circ = mi->on_circ;
+  int machine_idx = mi->machine_index;
+  mi->padding_scheduled_at_usec = 0;
+  circpad_statenum_t state = mi->current_state;
+
+  /* Make sure circuit didn't close on us */
+  if (mi->on_circ->marked_for_close) {
+    log_fn(LOG_INFO,LD_CIRC,
+           "Padding callback on circuit marked for close (%u). Ignoring.",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+         TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_CHANGED;
+  }
+
+  circpad_machine_count_padding_sent(mi);
+
+  if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+    circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ),
+                                CIRCPAD_GET_MACHINE(mi)->target_hopnum,
+                                RELAY_COMMAND_DROP, NULL, 0);
+    log_info(LD_CIRC, "Callback: Sending padding to origin circuit %u"
+             " (%d) [length: %"PRIu64"]",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier,
+             mi->on_circ->purpose, mi->state_length);
+  } else {
+    // If we're a non-origin circ, we can just send from here as if we're the
+    // edge.
+    if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) {
+      log_info(LD_CIRC, "Callback: Sending padding to circuit (%d)"
+               " [length: %"PRIu64"]", mi->on_circ->purpose, mi->state_length);
+      relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL,
+                                   0, NULL);
+      rep_hist_padding_count_write(PADDING_TYPE_DROP);
+    } else {
+      static ratelim_t cell_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC,
+                     "Too many cells (%d) in circ queue to send padding.",
+                      TO_OR_CIRCUIT(circ)->p_chan_cells.n);
+    }
+  }
+
+  /* This is a padding cell sent from the client or from the middle node,
+   * (because it's invoked from circuitpadding.c) */
+  circpad_cell_event_padding_sent(circ);
+
+  /* The circpad_cell_event_padding_sent() could cause us to transition.
+   * Check that we still have a padding machineinfo, and then check our token
+   * supply. */
+  if (circ->padding_info[machine_idx] != NULL) {
+    if (state != circ->padding_info[machine_idx]->current_state)
+      return CIRCPAD_STATE_CHANGED;
+    else
+      return check_machine_token_supply(circ->padding_info[machine_idx]);
+  } else {
+    return CIRCPAD_STATE_CHANGED;
+  }
+}
+
+/**
+ * Tor-timer compatible callback that tells us to send a padding cell.
+ *
+ * Timers are associated with circpad_machine_runtime_t's. When the machineinfo
+ * is freed on a circuit, the timers are cancelled. Since the lifetime
+ * of machineinfo is always longer than the timers, handles are not
+ * needed.
+ */
+static void
+circpad_send_padding_callback(tor_timer_t *timer, void *args,
+                              const struct monotime_t *time)
+{
+  circpad_machine_runtime_t *mi = ((circpad_machine_runtime_t*)args);
+  (void)timer; (void)time;
+
+  if (mi && mi->on_circ) {
+    assert_circuit_ok(mi->on_circ);
+    circpad_send_padding_cell_for_callback(mi);
+  } else {
+    // This shouldn't happen (represents a timer leak)
+    log_fn(LOG_WARN,LD_CIRC,
+            "Circuit closed while waiting for padding timer.");
+    tor_fragile_assert();
+  }
+
+  // TODO-MP-AP: Unify this counter with channelpadding for rephist stats
+  //total_timers_pending--;
+}
+
+/**
+ * Cache our consensus parameters upon consensus update.
+ */
+void
+circpad_new_consensus_params(const networkstatus_t *ns)
+{
+  circpad_padding_disabled =
+      networkstatus_get_param(ns, "circpad_padding_disabled",
+         0, 0, 1);
+
+  circpad_padding_reduced =
+      networkstatus_get_param(ns, "circpad_padding_reduced",
+         0, 0, 1);
+
+  circpad_global_allowed_cells =
+      networkstatus_get_param(ns, "circpad_global_allowed_cells",
+         0, 0, UINT16_MAX-1);
+
+  circpad_global_max_padding_percent =
+      networkstatus_get_param(ns, "circpad_global_max_padding_pct",
+         0, 0, 100);
+
+  circpad_max_circ_queued_cells =
+      networkstatus_get_param(ns, "circpad_max_circ_queued_cells",
+         CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX);
+}
+
+/**
+ * Return true if padding is allowed by torrc and consensus.
+ */
+static bool
+circpad_is_padding_allowed(void)
+{
+  /* If padding has been disabled in the consensus, don't send any more
+   * padding. Technically the machine should be shut down when the next
+   * machine condition check happens, but machine checks only happen on
+   * certain circuit events, and if padding is disabled due to some
+   * network overload or DoS condition, we really want to stop ASAP. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * Check this machine against its padding limits, as well as global
+ * consensus limits.
+ *
+ * We have two limits: a percent and a cell count. The cell count
+ * limit must be reached before the percent is enforced (this is to
+ * optionally allow very light padding of things like circuit setup
+ * while there is no other traffic on the circuit).
+ *
+ * TODO: Don't apply limits to machines form torrc.
+ *
+ * Returns 1 if limits are set and we've hit them. Otherwise returns 0.
+ */
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  /* If machine_padding_pct is non-zero, and we've sent more
+   * than the allowed count of padding cells, then check our
+   * percent limits for this machine. */
+  if (machine->max_padding_percent &&
+      mi->padding_sent >= machine->allowed_padding_count) {
+    uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*(uint32_t)mi->padding_sent) / total_cells >
+        machine->max_padding_percent) {
+      return 1; // limit is reached. Stop.
+    }
+  }
+
+  /* If circpad_max_global_padding_pct is non-zero, and we've
+   * sent more than the global padding cell limit, then check our
+   * global tor process percentage limit on padding. */
+  if (circpad_global_max_padding_percent &&
+      circpad_global_padding_sent >= circpad_global_allowed_cells) {
+    uint64_t total_cells = circpad_global_padding_sent +
+              circpad_global_nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*circpad_global_padding_sent) / total_cells >
+        circpad_global_max_padding_percent) {
+      return 1; // global limit reached. Stop.
+    }
+  }
+
+  return 0; // All good!
+}
+
+/**
+ * Schedule the next padding time according to the machineinfo on a
+ * circuit.
+ *
+ * The histograms represent inter-packet-delay. Whenever you get an packet
+ * event you should be scheduling your next timer (after cancelling any old
+ * ones and updating tokens accordingly).
+ *
+ * Returns 1 if we decide to transition states (due to infinity bin),
+ * 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi))
+{
+  circpad_delay_t in_usec = 0;
+  struct timeval timeout;
+  tor_assert(mi);
+
+  /* Don't schedule padding if it is disabled */
+  if (!circpad_is_padding_allowed()) {
+    static ratelim_t padding_lim = RATELIM_INIT(600);
+    log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+         "Padding has been disabled, but machine still on circuit %"PRIu64
+         ", %d",
+         mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+         mi->on_circ->n_circ_id);
+
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Don't schedule padding if we are currently in dormant mode. */
+  if (!is_participating_on_network()) {
+    log_info(LD_CIRC, "Not scheduling padding because we are dormant.");
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  // Don't pad in end (but  also don't cancel any previously
+  // scheduled padding either).
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    log_fn(LOG_INFO, LD_CIRC, "Padding end state on circuit %u",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check our padding limits */
+  if (circpad_machine_reached_padding_limit(mi)) {
+   if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Padding machine has reached padding limit on circuit %u",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier);
+    } else {
+      static ratelim_t padding_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+           "Padding machine has reached padding limit on circuit %"PRIu64
+           ", %d",
+           mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+           mi->on_circ->n_circ_id);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (mi->is_padding_timer_scheduled) {
+    /* Cancel current timer (if any) */
+    timer_disable(mi->padding_timer);
+    mi->is_padding_timer_scheduled = 0;
+  }
+
+  /* in_usec = in microseconds */
+  in_usec = circpad_machine_sample_delay(mi);
+  /* If we're using token removal, we need to know when the padding
+   * was scheduled at, so we can remove the appropriate token if
+   * a non-padding cell is sent before the padding timer expires.
+   *
+   * However, since monotime is unpredictably expensive, let's avoid
+   * using it for machines that don't need token removal. */
+  if (circpad_is_token_removal_supported(mi)) {
+    mi->padding_scheduled_at_usec = monotime_absolute_usec();
+  } else {
+    mi->padding_scheduled_at_usec = 1;
+  }
+  log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec on circuit %u", in_usec,
+       CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+
+  // Don't schedule if we have infinite delay.
+  if (in_usec == CIRCPAD_DELAY_INFINITE) {
+    return circpad_internal_event_infinity(mi);
+  }
+
+  if (mi->state_length == 0) {
+    /* If we're at length 0, that means we hit 0 after sending
+     * a cell earlier, and emitted an event for it, but
+     * for whatever reason we did not decide to change states then.
+     * So maybe the machine is waiting for bins empty, or for an
+     * infinity event later? That would be a strange machine,
+     * but there's no reason to make it impossible. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (in_usec <= 0) {
+    return circpad_send_padding_cell_for_callback(mi);
+  }
+
+  timeout.tv_sec = in_usec/TOR_USEC_PER_SEC;
+  timeout.tv_usec = (in_usec%TOR_USEC_PER_SEC);
+
+  log_fn(LOG_INFO, LD_CIRC, "\tPadding circuit %u in %u sec, %u usec",
+     CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+          (unsigned)timeout.tv_sec, (unsigned)timeout.tv_usec);
+
+  if (mi->padding_timer) {
+    timer_set_cb(mi->padding_timer, circpad_send_padding_callback, mi);
+  } else {
+    mi->padding_timer =
+        timer_new(circpad_send_padding_callback, mi);
+  }
+  timer_schedule(mi->padding_timer, &timeout);
+  mi->is_padding_timer_scheduled = 1;
+
+  // TODO-MP-AP: Unify with channelpadding counter
+  //rep_hist_padding_count_timers(++total_timers_pending);
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * If the machine transitioned to the END state, we need
+ * to check to see if it wants us to shut it down immediately.
+ * If it does, then we need to send the appropiate negotiation commands
+ * depending on which side it is.
+ *
+ * After this function is called, mi may point to freed memory. Do
+ * not access it.
+ */
+static void
+circpad_machine_spec_transitioned_to_end(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+  circuit_t *on_circ = mi->on_circ;
+
+  log_fn(LOG_INFO,LD_CIRC, "Padding machine in end state on circuit %u (%d)",
+         CIRCUIT_IS_ORIGIN(on_circ) ?
+         TO_ORIGIN_CIRCUIT(on_circ)->global_identifier : 0,
+         on_circ->purpose);
+
+  /*
+   * We allow machines to shut down and delete themselves as opposed
+   * to just going back to START or waiting forever in END so that
+   * we can handle the case where this machine started while it was
+   * the only machine that matched conditions, but *since* then more
+   * "higher ranking" machines now match the conditions, and would
+   * be given a chance to take precedence over this one in
+   * circpad_add_matching_machines().
+   *
+   * Returning to START or waiting forever in END would not give those
+   * other machines a chance to be launched, where as shutting down
+   * here does.
+   */
+  if (machine->should_negotiate_end) {
+    if (machine->is_origin_side) {
+      /* We free the machine info here so that we can be replaced
+       * by a different machine. But we must leave the padding_machine
+       * in place to wait for the negotiated response */
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(on_circ),
+                                machine->machine_num,
+                                machine->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    } else {
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_padding_negotiated(on_circ,
+                                machine->machine_num,
+                                CIRCPAD_COMMAND_STOP,
+                                CIRCPAD_RESPONSE_OK);
+      on_circ->padding_machine[machine->machine_index] = NULL;
+    }
+  }
+}
+
+/**
+ * Generic state transition function for padding state machines.
+ *
+ * Given an event and our mutable machine info, decide if/how to
+ * transition to a different state, and perform actions accordingly.
+ *
+ * Returns 1 if we transition states, 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_spec_transition,(circpad_machine_runtime_t *mi,
+                            circpad_event_t event))
+{
+  const circpad_state_t *state =
+      circpad_machine_current_state(mi);
+
+  /* If state is null we are in the end state. */
+  if (!state) {
+    /* If we in end state we don't pad no matter what. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check if this event is ignored or causes a cancel */
+  if (state->next_state[event] == CIRCPAD_STATE_IGNORE) {
+    return CIRCPAD_STATE_UNCHANGED;
+  } else if (state->next_state[event] == CIRCPAD_STATE_CANCEL) {
+    /* Check cancel events and cancel any pending padding */
+    mi->padding_scheduled_at_usec = 0;
+    if (mi->is_padding_timer_scheduled) {
+      mi->is_padding_timer_scheduled = 0;
+      /* Cancel current timer (if any) */
+      timer_disable(mi->padding_timer);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  } else {
+    circpad_statenum_t s = state->next_state[event];
+    /* See if we need to transition to any other states based on this event.
+     * Whenever a transition happens, even to our own state, we schedule
+     * padding.
+     *
+     * So if a state only wants to schedule padding for an event, it specifies
+     * a transition to itself. All non-specified events are ignored.
+     */
+    log_fn(LOG_INFO, LD_CIRC,
+           "Circuit %u circpad machine %d transitioning from %u to %u",
+             CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+           mi->machine_index, mi->current_state, s);
+
+    /* If this is not the same state, switch and init tokens,
+     * otherwise just reschedule padding. */
+    if (mi->current_state != s) {
+      mi->current_state = s;
+      circpad_machine_setup_tokens(mi);
+      circpad_choose_state_length(mi);
+
+      /* If we transition to the end state, check to see
+       * if this machine wants to be shut down at end */
+      if (s == CIRCPAD_STATE_END) {
+        circpad_machine_spec_transitioned_to_end(mi);
+        /* We transitioned but we don't pad in end. Also, mi
+         * may be freed. Returning STATE_CHANGED prevents us
+         * from accessing it in any callers of this function. */
+        return CIRCPAD_STATE_CHANGED;
+      }
+
+      /* We transitioned to a new state, schedule padding */
+      circpad_machine_schedule_padding(mi);
+      return CIRCPAD_STATE_CHANGED;
+    }
+
+    /* We transitioned back to the same state. Schedule padding,
+     * and inform if that causes a state transition. */
+    return circpad_machine_schedule_padding(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Estimate the circuit RTT from the current middle hop out to the
+ * end of the circuit.
+ *
+ * We estimate RTT by calculating the time between "receive" and
+ * "send" at a middle hop. This is because we "receive" a cell
+ * from the origin, and then relay it towards the exit before a
+ * response comes back. It is that response time from the exit side
+ * that we want to measure, so that we can make use of it for synthetic
+ * response delays.
+ */
+static void
+circpad_estimate_circ_rtt_on_received(circuit_t *circ,
+                                      circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update)
+    return;
+
+  /* If we already have a last received packet time, that means we
+   * did not get a response before this packet. The RTT estimate
+   * only makes sense if we do not have multiple packets on the
+   * wire, so stop estimating if this is the second packet
+   * back to back. However, for the first set of back-to-back
+   * packets, we can wait until the very first response comes back
+   * to us, to measure that RTT (for the response to optimistic
+   * data, for example). Hence stop_rtt_update is only checked
+   * in this received side function, and not in send side below.
+   */
+  if (mi->last_received_time_usec) {
+    /* We also allow multiple back-to-back packets if the circuit is not
+     * opened, to handle var cells.
+     * XXX: Will this work with out var cell plans? Maybe not,
+     * since we're opened at the middle hop as soon as we process
+     * one var extend2 :/ */
+    if (circ->state == CIRCUIT_STATE_OPEN) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Stopping padding RTT estimation on circuit (%"PRIu64
+           ", %d) after two back to back packets. Current RTT: %d",
+           circ->n_chan ?  circ->n_chan->global_identifier : 0,
+           circ->n_circ_id, mi->rtt_estimate_usec);
+      mi->stop_rtt_update = 1;
+
+      if (!mi->rtt_estimate_usec) {
+        static ratelim_t rtt_lim = RATELIM_INIT(600);
+        log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+          "Circuit got two cells back to back before estimating RTT.");
+      }
+    }
+  } else {
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return;
+    }
+
+    /* Since monotime is unpredictably expensive, only update this field
+     * if rtt estimates are needed. Otherwise, stop the rtt update. */
+    if (state->use_rtt_estimate) {
+      mi->last_received_time_usec = monotime_absolute_usec();
+    } else {
+      /* Let's fast-path future decisions not to update rtt if the
+       * feature is not in use. */
+      mi->stop_rtt_update = 1;
+    }
+  }
+}
+
+/**
+ * Handles the "send" side of RTT calculation at middle nodes.
+ *
+ * This function calculates the RTT from the middle to the end
+ * of the circuit by subtracting the last received cell timestamp
+ * from the current time. It allows back-to-back cells until
+ * the circuit is opened, to allow for var cell handshakes.
+ * XXX: Check our var cell plans to make sure this will work.
+ */
+static void
+circpad_estimate_circ_rtt_on_send(circuit_t *circ,
+                                  circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ))
+    return;
+
+  /* If last_received_time_usec is non-zero, we are waiting for a response
+   * from the exit side. Calculate the time delta and use it as RTT. */
+  if (mi->last_received_time_usec) {
+    circpad_time_t rtt_time = monotime_absolute_usec() -
+        mi->last_received_time_usec;
+
+    /* Reset the last RTT packet time, so we can tell if two cells
+     * arrive back to back */
+    mi->last_received_time_usec = 0;
+
+    /* Use INT32_MAX to ensure the addition doesn't overflow */
+    if (rtt_time >= INT32_MAX) {
+      log_fn(LOG_WARN,LD_CIRC,
+             "Circuit padding RTT estimate overflowed: %"PRIu64
+             " vs %"PRIu64, monotime_absolute_usec(),
+               mi->last_received_time_usec);
+      return;
+    }
+
+    /* If the old RTT estimate is lower than this one, use this one, because
+     * the circuit is getting longer. If this estimate is somehow
+     * faster than the previous, then maybe that was network jitter, or a
+     * bad monotonic clock source (so our ratchet returned a zero delta).
+     * In that case, average them. */
+    if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) {
+      mi->rtt_estimate_usec = (circpad_delay_t)rtt_time;
+    } else {
+      mi->rtt_estimate_usec += (circpad_delay_t)rtt_time;
+      mi->rtt_estimate_usec /= 2;
+    }
+  } else if (circ->state == CIRCUIT_STATE_OPEN) {
+    /* If last_received_time_usec is zero, then we have gotten two cells back
+     * to back. Stop estimating RTT in this case. Note that we only
+     * stop RTT update if the circuit is opened, to allow for RTT estimates
+     * of var cells during circ setup. */
+    if (!mi->rtt_estimate_usec && !mi->stop_rtt_update) {
+      static ratelim_t rtt_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+        "Circuit sent two cells back to back before estimating RTT.");
+    }
+    mi->stop_rtt_update = 1;
+  }
+}
+
+/**
+ * A "non-padding" cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_sent(circuit_t *on_circ)
+{
+  /* Update global cell count */
+  circpad_global_nonpadding_sent++;
+
+  /* If there are no machines then this loop should not iterate */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]);
+
+    /* Then, do accounting */
+    circpad_machine_count_nonpadding_sent(on_circ->padding_info[i]);
+
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                                 CIRCPAD_EVENT_NONPADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/** Check if this cell or circuit are related to circuit padding and handle
+ *  them if so.  Return 0 if the cell was handled in this subsystem and does
+ *  not need any other consideration, otherwise return 1.
+ */
+int
+circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                            crypt_path_t *layer_hint,
+                            const relay_header_t *rh)
+{
+  /* First handle the padding commands, since we want to ignore any other
+   * commands if this circuit is padding-specific. */
+  switch (rh->command) {
+    case RELAY_COMMAND_DROP:
+      /* Already examined in circpad_deliver_recognized_relay_cell_events */
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATE:
+      circpad_handle_padding_negotiate(circ, cell);
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATED:
+      if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0)
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
+      return 0;
+  }
+
+  /* If this is a padding circuit we don't need to parse any other commands
+   * than the padding ones. Just drop them to the floor.
+   *
+   * Note: we deliberately do not call circuit_read_valid_data() here. The
+   * vanguards addon (specifically the 'bandguards' component's dropped cell
+   * detection) will thus close this circuit, as it would for any other
+   * unexpected cell. However, default tor will *not* close the circuit.
+   *
+   * This is intentional. We are not yet certain that is it optimal to keep
+   * padding circuits open in cases like these, rather than closing them.
+   * We suspect that continuing to pad is optimal against a passive classifier,
+   * but as soon as the adversary is active (even as a client adversary) this
+   * might change.
+   *
+   * So as a way forward, we log the cell command and circuit number, to
+   * help us enumerate the most common instances of this in testing with
+   * vanguards, to see which are common enough to verify and handle
+   * properly.
+   * - Mike
+   */
+  if (circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Ignored cell (%d) that arrived in padding circuit "
+                      " %u.", rh->command, CIRCUIT_IS_ORIGIN(circ) ?
+                           TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * A "non-padding" cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_received(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]);
+
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                               CIRCPAD_EVENT_NONPADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_padding_sent(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+
+      on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                             CIRCPAD_EVENT_PADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_padding_received(circuit_t *on_circ)
+{
+  /* identical to padding sent */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                              CIRCPAD_EVENT_PADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * An "infinite" delay has ben chosen from one of our histograms.
+ *
+ * "Infinite" delays mean don't send padding -- but they can also
+ * mean transition to another state depending on the state machine
+ * definitions. Check the rules and react accordingly.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY);
+}
+
+/**
+ * All of the bins of our current state's histogram's are empty.
+ *
+ * Check to see if this means transition to another state, and if
+ * not, refill the tokens.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *mi)
+{
+  if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY)
+      == CIRCPAD_STATE_CHANGED) {
+    return CIRCPAD_STATE_CHANGED;
+  } else {
+    /* If we dont transition, then we refill the tokens */
+    circpad_machine_setup_tokens(mi);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+}
+
+/**
+ * This state has used up its cell count. Emit the event and
+ * see if we transition.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_state_length_up(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT);
+}
+
+/**
+ * Returns true if the circuit matches the conditions.
+ */
+static inline bool
+circpad_machine_conditions_met(origin_circuit_t *circ,
+                               const circpad_machine_spec_t *machine)
+{
+  /* If padding is disabled, no machines should match/apply. This has
+   * the effect of shutting down all machines, and not adding any more. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding)
+    return 0;
+
+  /* If the consensus or our torrc has selected reduced connection padding,
+   * then only allow this machine if it is flagged as acceptable under
+   * reduced padding conditions */
+  if (circpad_padding_reduced || get_options()->ReducedCircuitPadding) {
+    if (!machine->conditions.reduced_padding_ok)
+      return 0;
+  }
+
+  if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose)
+      & machine->conditions.purpose_mask))
+    return 0;
+
+  if (machine->conditions.requires_vanguards) {
+    const or_options_t *options = get_options();
+
+    /* Pinned middles are effectively vanguards */
+    if (!(options->HSLayer2Nodes || options->HSLayer3Nodes))
+      return 0;
+  }
+
+  /* We check for any bits set in the circuit state mask so that machines
+   * can say any of the following through their state bitmask:
+   * "I want to apply to circuits with either streams or no streams"; OR
+   * "I only want to apply to circuits with streams"; OR
+   * "I only want to apply to circuits without streams". */
+  if (!(circpad_circuit_state(circ) & machine->conditions.state_mask))
+    return 0;
+
+  if (circuit_get_cpath_opened_len(circ) < machine->conditions.min_hops)
+    return 0;
+
+  return 1;
+}
+
+/**
+ * Returns a minimized representation of the circuit state.
+ *
+ * The padding code only cares if the circuit is building,
+ * opened, used for streams, and/or still has relay early cells.
+ * This returns a bitmask of all state properities that apply to
+ * this circuit.
+ */
+static inline
+circpad_circuit_state_t
+circpad_circuit_state(origin_circuit_t *circ)
+{
+  circpad_circuit_state_t retmask = 0;
+
+  if (circ->p_streams)
+    retmask |= CIRCPAD_CIRC_STREAMS;
+  else
+    retmask |= CIRCPAD_CIRC_NO_STREAMS;
+
+  /* We use has_opened to prevent cannibialized circs from flapping. */
+  if (circ->has_opened)
+    retmask |= CIRCPAD_CIRC_OPENED;
+  else
+    retmask |= CIRCPAD_CIRC_BUILDING;
+
+  if (circ->remaining_relay_early_cells > 0)
+    retmask |= CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  else
+    retmask |= CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+
+  return retmask;
+}
+
+/**
+ * Convert a normal circuit purpose into a bitmask that we can
+ * use for determining matching circuits.
+ */
+circpad_purpose_mask_t
+circpad_circ_purpose_to_mask(uint8_t circ_purpose)
+{
+  /* Treat OR circ purposes as ignored. They should not be passed here*/
+  if (BUG(circ_purpose <= CIRCUIT_PURPOSE_OR_MAX_)) {
+    return 0;
+  }
+
+  /* Treat new client circuit purposes as "OMG ITS EVERYTHING".
+   * This also should not happen */
+  if (BUG(circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1 > 32)) {
+    return CIRCPAD_PURPOSE_ALL;
+  }
+
+  /* Convert the purpose to a bit position */
+  return 1 << (circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1);
+}
+
+/**
+ * Shut down any machines whose conditions no longer match
+ * the current circuit.
+ */
+static void
+circpad_shutdown_old_machines(origin_circuit_t *on_circ)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    if (!circpad_machine_conditions_met(on_circ,
+                                        circ->padding_machine[i])) {
+      // Clear machineinfo (frees timers)
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      // Send padding negotiate stop
+      circpad_negotiate_padding(on_circ,
+                                circ->padding_machine[i]->machine_num,
+                                circ->padding_machine[i]->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Negotiate new machines that would apply to this circuit, given the machines
+ * inside <b>machines_sl</b>.
+ *
+ * This function checks to see if we have any free machine indexes,
+ * and for each free machine index, it initializes the most recently
+ * added origin-side padding machine that matches the target machine
+ * index and circuit conditions, and negotiates it with the appropriate
+ * middle relay.
+ */
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+#ifdef TOR_UNIT_TESTS
+  /* Tests don't have to init our padding machines */
+  if (!machines_sl)
+    return;
+#endif
+
+  /* If padding negotiation failed before, do not try again */
+  if (on_circ->padding_negotiation_failed)
+    return;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* If there is a padding machine info, this index is occupied.
+     * No need to check conditions for this index. */
+    if (circ->padding_info[i])
+      continue;
+
+    /* We have a free machine index. Check the origin padding
+     * machines in reverse order, so that more recently added
+     * machines take priority over older ones. */
+    SMARTLIST_FOREACH_REVERSE_BEGIN(machines_sl,
+                                    circpad_machine_spec_t *,
+                                    machine) {
+      /* Machine definitions have a specific target machine index.
+       * This is so event ordering is deterministic with respect
+       * to which machine gets events first when there are two
+       * machines installed on a circuit. Make sure we only
+       * add this machine if its target machine index is free. */
+      if (machine->machine_index == i &&
+          circpad_machine_conditions_met(on_circ, machine)) {
+
+        // We can only replace this machine if the target hopnum
+        // is the same, otherwise we'll get invalid data
+        if (circ->padding_machine[i]) {
+          if (circ->padding_machine[i]->target_hopnum !=
+              machine->target_hopnum)
+            continue;
+          /* Replace it. (Don't free - is global). */
+          circ->padding_machine[i] = NULL;
+        }
+
+        /* Set up the machine immediately so that the slot is occupied.
+         * We will tear it down on error return, or if there is an error
+         * response from the relay. */
+        circpad_setup_machine_on_circ(circ, machine);
+        if (circpad_negotiate_padding(on_circ, machine->machine_num,
+                                  machine->target_hopnum,
+                                  CIRCPAD_COMMAND_START) < 0) {
+          log_info(LD_CIRC,
+                   "Padding not negotiated. Cleaning machine from circuit %u",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+          circpad_circuit_machineinfo_free_idx(circ, i);
+          circ->padding_machine[i] = NULL;
+          on_circ->padding_negotiation_failed = 1;
+        } else {
+          /* Success. Don't try any more machines */
+          return;
+        }
+      }
+    } SMARTLIST_FOREACH_END(machine);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Event that tells us we added a hop to an origin circuit.
+ *
+ * This event is used to decide if we should create a padding machine
+ * on a circuit.
+ */
+void
+circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ)
+{
+  /* Since our padding conditions do not specify a max_hops,
+   * all we can do is add machines here */
+  circpad_add_matching_machines(on_circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is now built.
+ *
+ * Shut down any machines that only applied to un-built circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_built(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Circpad purpose changed event.
+ *
+ * Shut down any machines that don't apply to our circ purpose.
+ * Activate any new ones that do.
+ */
+void
+circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is out of RELAY_EARLY
+ * cells.
+ *
+ * Shut down any machines that only applied to RELAY_EARLY circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams attached event.
+ *
+ * Called from link_apconn_to_circ() and handle_hs_exit_conn()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams detached event.
+ *
+ * Called from circuit_detach_stream()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Verify that padding is coming from the expected hop.
+ *
+ * Returns true if from_hop matches the target hop from
+ * one of our padding machines.
+ *
+ * Returns false if we're not an origin circuit, or if from_hop
+ * does not match one of the padding machines.
+ */
+bool
+circpad_padding_is_from_expected_hop(circuit_t *circ,
+                                     crypt_path_t *from_hop)
+{
+  crypt_path_t *target_hop = NULL;
+  if (!CIRCUIT_IS_ORIGIN(circ))
+    return 0;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* We have to check padding_machine and not padding_info/active
+     * machines here because padding may arrive after we shut down a
+     * machine. The info is gone, but the padding_machine waits
+     * for the padding_negotiated response to come back. */
+    if (!circ->padding_machine[i])
+      continue;
+
+    target_hop = circuit_get_cpath_hop(TO_ORIGIN_CIRCUIT(circ),
+                    circ->padding_machine[i]->target_hopnum);
+
+    if (target_hop == from_hop)
+      return 1;
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return 0;
+}
+
+/**
+ * Deliver circpad events for an "unrecognized cell".
+ *
+ * Unrecognized cells are sent to relays and are forwarded
+ * onto the next hop of their circuits. Unrecognized cells
+ * are by definition not padding. We need to tell relay-side
+ * state machines that a non-padding cell was sent or received,
+ * depending on the direction, so they can update their histograms
+ * and decide to pad or not.
+ */
+void
+circpad_deliver_unrecognized_cell_events(circuit_t *circ,
+                                         cell_direction_t dir)
+{
+  // We should never see unrecognized cells at origin.
+  // Our caller emits a warn when this happens.
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    return;
+  }
+
+  if (dir == CELL_DIRECTION_OUT) {
+    /* When direction is out (away from origin), then we received non-padding
+       cell coming from the origin to us. */
+    circpad_cell_event_nonpadding_received(circ);
+  } else if (dir == CELL_DIRECTION_IN) {
+    /* It's in and not origin, so the cell is going away from us.
+     * So we are relaying a non-padding cell towards the origin. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for "recognized" relay cells.
+ *
+ * Recognized cells are destined for this hop, either client or middle.
+ * Check if this is a padding cell or not, and send the appropiate
+ * received event.
+ */
+void
+circpad_deliver_recognized_relay_cell_events(circuit_t *circ,
+                                             uint8_t relay_command,
+                                             crypt_path_t *layer_hint)
+{
+  if (relay_command == RELAY_COMMAND_DROP) {
+    rep_hist_padding_count_read(PADDING_TYPE_DROP);
+
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      if (circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), 0);
+      } else {
+        /* This is unexpected padding. Ignore it for now. */
+        return;
+      }
+    }
+
+    /* The cell should be recognized by now, which means that we are on the
+       destination, which means that we received a padding cell. We might be
+       the client or the Middle node, still, because leaky-pipe. */
+    circpad_cell_event_padding_received(circ);
+    log_fn(LOG_INFO, LD_CIRC, "Got padding cell on %s circuit %u.",
+           CIRCUIT_IS_ORIGIN(circ) ? "origin" : "non-origin",
+           CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+  } else {
+    /* We received a non-padding cell on the edge */
+    circpad_cell_event_nonpadding_received(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for relay cells sent from us.
+ *
+ * If this is a padding cell, update our padding stats
+ * and deliver the event. Otherwise just deliver the event.
+ */
+void
+circpad_deliver_sent_relay_cell_events(circuit_t *circ,
+                                       uint8_t relay_command)
+{
+  /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in
+   * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed
+   * or processed here).
+   *
+   * We do generate events for PADDING_NEGOTIATE and PADDING_NEGOTIATED cells.
+   */
+  if (relay_command == RELAY_COMMAND_DROP) {
+    /* Optimization: The event for RELAY_COMMAND_DROP is sent directly
+     * from circpad_send_padding_cell_for_callback(). This is to avoid
+     * putting a cell_t and a relay_header_t on the stack repeatedly
+     * if we decide to send a long train of padding cells back-to-back
+     * with 0 delay. So we do nothing here. */
+    return;
+  } else {
+    /* This is a non-padding cell sent from the client or from
+     * this node. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Initialize the states array for a circpad machine.
+ */
+void
+circpad_machine_states_init(circpad_machine_spec_t *machine,
+                            circpad_statenum_t num_states)
+{
+  if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) {
+    num_states = CIRCPAD_MAX_MACHINE_STATES;
+  }
+
+  machine->num_states = num_states;
+  machine->states = tor_malloc_zero(sizeof(circpad_state_t)*num_states);
+
+  /* Initialize the default next state for all events to
+   * "ignore" -- if events aren't specified, they are ignored. */
+  for (circpad_statenum_t s = 0; s < num_states; s++) {
+    for (int e = 0; e < CIRCPAD_NUM_EVENTS; e++) {
+      machine->states[s].next_state[e] = CIRCPAD_STATE_IGNORE;
+    }
+  }
+}
+
+static void
+circpad_setup_machine_on_circ(circuit_t *on_circ,
+                              const circpad_machine_spec_t *machine)
+{
+  if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up non-origin machine on origin circuit!");
+    return;
+  }
+
+  if (!CIRCUIT_IS_ORIGIN(on_circ) && machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up origin machine on non-origin circuit!");
+    return;
+  }
+
+  IF_BUG_ONCE(on_circ->padding_machine[machine->machine_index] != NULL) {
+    return;
+  }
+  IF_BUG_ONCE(on_circ->padding_info[machine->machine_index] != NULL) {
+    return;
+  }
+
+  /* Log message */
+  if (CIRCUIT_IS_ORIGIN(on_circ)) {
+    log_info(LD_CIRC, "Registering machine %s to origin circ %u (%d)",
+             machine->name,
+             TO_ORIGIN_CIRCUIT(on_circ)->global_identifier, on_circ->purpose);
+  } else {
+    log_info(LD_CIRC, "Registering machine %s to non-origin circ (%d)",
+             machine->name, on_circ->purpose);
+  }
+
+  on_circ->padding_info[machine->machine_index] =
+      circpad_circuit_machineinfo_new(on_circ, machine->machine_index);
+  on_circ->padding_machine[machine->machine_index] = machine;
+}
+
+/** Validate a single state of a padding machine */
+static bool
+padding_machine_state_is_valid(const circpad_state_t *state)
+{
+  int b;
+  uint32_t tokens_count = 0;
+  circpad_delay_t prev_bin_edge = 0;
+
+  /* We only validate histograms */
+  if (!state->histogram_len) {
+    return true;
+  }
+
+  /* We need at least two bins in a histogram */
+  if (state->histogram_len < 2) {
+    log_warn(LD_CIRC, "You can't have a histogram with less than 2 bins");
+    return false;
+  }
+
+  /* For each machine state, if it's a histogram, make sure all the
+   * histogram edges are well defined (i.e. are strictly monotonic). */
+  for (b = 0 ; b < state->histogram_len ; b++) {
+    /* Check that histogram edges are strictly increasing. Ignore the first
+     * edge since it can be zero. */
+    if (prev_bin_edge >= state->histogram_edges[b] && b > 0) {
+      log_warn(LD_CIRC, "Histogram edges are not increasing [%u/%u]",
+               prev_bin_edge, state->histogram_edges[b]);
+      return false;
+    }
+
+    prev_bin_edge = state->histogram_edges[b];
+
+    /* Also count the number of tokens as we go through the histogram states */
+    tokens_count += state->histogram[b];
+  }
+  /* Verify that the total number of tokens is correct */
+  if (tokens_count != state->histogram_total_tokens) {
+    log_warn(LD_CIRC, "Histogram token count is wrong [%u/%u]",
+             tokens_count, state->histogram_total_tokens);
+    return false;
+  }
+
+  return true;
+}
+
+/** Basic validation of padding machine */
+static bool
+padding_machine_is_valid(const circpad_machine_spec_t *machine)
+{
+  int i;
+
+  /* Validate the histograms of the padding machine */
+  for (i = 0 ; i < machine->num_states ; i++) {
+    if (!padding_machine_state_is_valid(&machine->states[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/* Validate and register <b>machine</b> into <b>machine_list</b>. If
+ * <b>machine_list</b> is NULL, then just validate. */
+void
+circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                 smartlist_t *machine_list)
+{
+  if (!padding_machine_is_valid(machine)) {
+    log_warn(LD_CIRC, "Machine #%u is invalid. Ignoring.",
+             machine->machine_num);
+    return;
+  }
+
+  if (machine_list) {
+    smartlist_add(machine_list, machine);
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+/* These padding machines are only used for tests pending #28634. */
+static void
+circpad_circ_client_machine_init(void)
+{
+  circpad_machine_spec_t *circ_client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  circ_client_machine->conditions.min_hops = 2;
+  circ_client_machine->conditions.state_mask =
+      CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circ_client_machine->conditions.reduced_padding_ok = 1;
+
+  circ_client_machine->target_hopnum = 2;
+  circ_client_machine->is_origin_side = 1;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_client_machine, 3);
+
+  circ_client_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* If we are in burst state, and we send a non-padding cell, then we cancel
+     the timer for the next padding cell:
+     We dont want to send fake extends when actual extends are going on */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1]= 1000000;
+
+  /* We have 5 tokens in the histogram, which means that all circuits will look
+   * like they have 7 hops (since we start this machine after the second hop,
+   * and tokens are decremented for any valid hops, and fake extends are
+   * used after that -- 2+5==7). */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5;
+
+  circ_client_machine->machine_num = smartlist_len(origin_padding_machines);
+  circpad_register_padding_machine(circ_client_machine,
+                                   origin_padding_machines);
+}
+
+static void
+circpad_circ_responder_machine_init(void)
+{
+  circpad_machine_spec_t *circ_responder_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Shut down the machine after we've sent enough packets */
+  circ_responder_machine->should_negotiate_end = 1;
+
+  /* The relay-side doesn't care what hopnum it is, but for consistency,
+   * let's match the client */
+  circ_responder_machine->target_hopnum = 2;
+  circ_responder_machine->is_origin_side = 0;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_responder_machine, 3);
+
+  /* This is the settings of the state machine. In the future we are gonna
+     serialize this into the consensus or the torrc */
+
+  /* We transition to the burst state on padding receive and on non-padding
+   * recieve */
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we _stay_ in the burst state when a non-padding
+   * is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we transition to the gap state when we receive a
+   * padding cell */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* These describe the padding charasteristics when in burst state */
+
+  /* use_rtt_estimate tries to estimate how long padding cells take to go from
+     C->M, and uses that as what as the base of the histogram */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+  /* The histogram is 2 bins: an empty one, and infinity */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1] =
+    1000000;
+  /* During burst state we wait forever for padding to arrive.
+
+     We are waiting for a padding cell from the client to come in, so that we
+     respond, and we immitate how extend looks like */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  // Only infinity bin:
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      histogram_total_tokens = 1;
+
+  /* From the gap state, we _stay_ in the gap state, when we receive padding
+   * or non padding */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* And from the gap state, we go to the end, when the bins are empty or a
+   * non-padding cell is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_END;
+
+  // FIXME: Tune this histogram
+
+  /* The gap state is the delay you wait after you receive a padding cell
+     before you send a padding response */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6;
+  /* Specify histogram bins */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[1]= 1000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[2]= 2000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[3]= 4000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[4]= 8000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[5]= 16000;
+  /* Specify histogram tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[3] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1;
+  /* Total number of tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6;
+
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC;
+
+  circ_responder_machine->machine_num = smartlist_len(relay_padding_machines);
+  circpad_register_padding_machine(circ_responder_machine,
+                                   relay_padding_machines);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize all of our padding machines.
+ *
+ * This is called at startup. It sets up some global machines, and then
+ * loads some from torrc, and from the tor consensus.
+ */
+void
+circpad_machines_init(void)
+{
+  tor_assert_nonfatal(origin_padding_machines == NULL);
+  tor_assert_nonfatal(relay_padding_machines == NULL);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  /* Register machines for hiding client-side intro circuits */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+
+  /* Register machines for hiding client-side rendezvous circuits */
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  // TODO: Parse machines from consensus and torrc
+#ifdef TOR_UNIT_TESTS
+  circpad_circ_client_machine_init();
+  circpad_circ_responder_machine_init();
+#endif
+}
+
+/**
+ * Free our padding machines
+ */
+void
+circpad_machines_free(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH(origin_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(origin_padding_machines);
+  }
+
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH(relay_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/**
+ * Check the Protover info to see if a node supports padding.
+ */
+static bool
+circpad_node_supports_padding(const node_t *node)
+{
+  if (node->rs) {
+    log_fn(LOG_INFO, LD_CIRC, "Checking padding: %s",
+           node->rs->pv.supports_hs_setup_padding ?
+              "supported" : "unsupported");
+    return node->rs->pv.supports_hs_setup_padding;
+  }
+
+  log_fn(LOG_INFO, LD_CIRC, "Empty routerstatus in padding check");
+  return 0;
+}
+
+/**
+ * Get a node_t for the nth hop in our circuit, starting from 1.
+ *
+ * Returns node_t from the consensus for that hop, if it is opened.
+ * Otherwise returns NULL.
+ */
+MOCK_IMPL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop))
+{
+  crypt_path_t *iter = circuit_get_cpath_hop(circ, hop);
+
+  if (!iter || iter->state != CPATH_STATE_OPEN)
+    return NULL;
+
+  return node_get_by_id(iter->extend_info->identity_digest);
+}
+
+/**
+ * Return true if a particular circuit supports padding
+ * at the desired hop.
+ */
+static bool
+circpad_circuit_supports_padding(origin_circuit_t *circ,
+                                 int target_hopnum)
+{
+  const node_t *hop;
+
+  if (!(hop = circuit_get_nth_node(circ, target_hopnum))) {
+    return 0;
+  }
+
+  return circpad_node_supports_padding(hop);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_negotiate_padding(origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command)
+{
+  circpad_negotiate_t type;
+  cell_t cell;
+  ssize_t len;
+
+  /* Check that the target hop lists support for padding in
+   * its ProtoVer fields */
+  if (!circpad_circuit_supports_padding(circ, target_hopnum)) {
+    return -1;
+  }
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiate_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiate_set_command(&type, command);
+  circpad_negotiate_set_version(&type, 0);
+  circpad_negotiate_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return -1;
+
+  log_fn(LOG_INFO,LD_CIRC,
+         "Negotiating padding on circuit %u (%d), command %d",
+         circ->global_identifier, TO_CIRCUIT(circ)->purpose, command);
+
+  return circpad_send_command_to_hop(circ, target_hopnum,
+                                     RELAY_COMMAND_PADDING_NEGOTIATE,
+                                     cell.payload, len);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns 1 if successful (or already set up), 0 otherwise.
+ */
+bool
+circpad_padding_negotiated(circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response)
+{
+  circpad_negotiated_t type;
+  cell_t cell;
+  ssize_t len;
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiated_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiated_set_command(&type, command);
+  circpad_negotiated_set_response(&type, response);
+  circpad_negotiated_set_version(&type, 0);
+  circpad_negotiated_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiated_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return 0;
+
+  /* Use relay_send because we're from the middle to the origin. We don't
+   * need to specify a target hop or layer_hint. */
+  return relay_send_command_from_edge(0, circ,
+                                      RELAY_COMMAND_PADDING_NEGOTIATED,
+                                      (void*)cell.payload,
+                                      (size_t)len, NULL) == 0;
+}
+
+/**
+ * Parse and react to a padding_negotiate cell.
+ *
+ * This is called at the middle node upon receipt of the client's choice of
+ * state machine, so that it can use the requested state machine index, if
+ * it is available.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell)
+{
+  int retval = 0;
+  circpad_negotiate_t *negotiate;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiate cell unsupported at origin (circuit %u)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATE cell; dropping.");
+    return -1;
+  }
+
+  if (negotiate->command == CIRCPAD_COMMAND_STOP) {
+    /* Free the machine corresponding to this machine type */
+    if (free_circ_machineinfos_with_machine_num(circ,
+                negotiate->machine_type)) {
+      log_info(LD_CIRC, "Received STOP command for machine %u",
+               negotiate->machine_type);
+      goto done;
+    }
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received circuit padding stop command for unknown machine.");
+    goto err;
+  } else if (negotiate->command == CIRCPAD_COMMAND_START) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            const circpad_machine_spec_t *, m) {
+      if (m->machine_num == negotiate->machine_type) {
+        circpad_setup_machine_on_circ(circ, m);
+        circpad_cell_event_nonpadding_received(circ);
+        goto done;
+      }
+    } SMARTLIST_FOREACH_END(m);
+  }
+
+  err:
+    retval = -1;
+
+  done:
+    circpad_padding_negotiated(circ, negotiate->machine_type,
+                   negotiate->command,
+                   (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR);
+    circpad_negotiate_free(negotiate);
+
+    return retval;
+}
+
+/**
+ * Parse and react to a padding_negotiated cell.
+ *
+ * This is called at the origin upon receipt of the middle's response
+ * to our choice of state machine.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell,
+                                  crypt_path_t *layer_hint)
+{
+  circpad_negotiated_t *negotiated;
+
+  if (!CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell unsupported at non-origin.");
+    return -1;
+  }
+
+  /* Verify this came from the expected hop */
+  if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell from wrong hop on circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATED cell on circuit %u; "
+          "dropping.", TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (negotiated->command == CIRCPAD_COMMAND_STOP) {
+    log_info(LD_CIRC,
+             "Received STOP command on PADDING_NEGOTIATED for circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    /* There may not be a padding_info here if we shut down the
+     * machine in circpad_shutdown_old_machines(). Or, if
+     * circpad_add_matching_matchines() added a new machine,
+     * there may be a padding_machine for a different machine num
+     * than this response. */
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+  } else if (negotiated->command == CIRCPAD_COMMAND_START &&
+             negotiated->response == CIRCPAD_RESPONSE_ERR) {
+    // This can happen due to consensus drift.. free the machines
+    // and be sad
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+    TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1;
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Middle node did not accept our padding request on circuit %u (%d)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier,
+           circ->purpose);
+  }
+
+  circpad_negotiated_free(negotiated);
+  return 0;
+}
+
+/** Free memory allocated by this machine spec. */
+STATIC void
+machine_spec_free_(circpad_machine_spec_t *m)
+{
+  if (!m) return;
+
+  tor_free(m->states);
+  tor_free(m);
+}
+
+/** Free all memory allocated by the circuitpadding subsystem. */
+void
+circpad_free_all(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(origin_padding_machines);
+  }
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/* Serialization */
+// TODO: Should we use keyword=value here? Are there helpers for that?
+#if 0
+static void
+circpad_state_serialize(const circpad_state_t *state,
+                        smartlist_t *chunks)
+{
+  smartlist_add_asprintf(chunks, " %u", state->histogram[0]);
+  for (int i = 1; i < state->histogram_len; i++) {
+    smartlist_add_asprintf(chunks, ",%u",
+                           state->histogram[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " 0x%x",
+                         state->transition_cancel_events);
+
+  for (int i = 0; i < CIRCPAD_NUM_STATES; i++) {
+    smartlist_add_asprintf(chunks, ",0x%x",
+                           state->transition_events[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " %u %u",
+                         state->use_rtt_estimate,
+                         state->token_removal);
+}
+
+char *
+circpad_machine_spec_to_string(const circpad_machine_spec_t *machine)
+{
+  smartlist_t *chunks = smartlist_new();
+  char *out;
+  (void)machine;
+
+  circpad_state_serialize(&machine->start, chunks);
+  circpad_state_serialize(&machine->gap, chunks);
+  circpad_state_serialize(&machine->burst, chunks);
+
+  out = smartlist_join_strings(chunks, "", 0, NULL);
+
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+  return out;
+}
+
+// XXX: Writeme
+const circpad_machine_spec_t *
+circpad_string_to_machine(const char *str)
+{
+  (void)str;
+  return NULL;
+}
+
+#endif /* 0 */
diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h
new file mode 100644
index 0000000000..e9eb32c618
--- /dev/null
+++ b/src/core/or/circuitpadding.h
@@ -0,0 +1,813 @@
+/*
+ * Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.h
+ * \brief Header file for circuitpadding.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_H
+#define TOR_CIRCUITPADDING_H
+
+#include "trunnel/circpad_negotiation.h"
+#include "lib/evloop/timers.h"
+
+struct circuit_t;
+struct origin_circuit_t;
+struct cell_t;
+
+/**
+ * Signed error return with the specific property that negative
+ * values mean error codes of various semantics, 0 means success,
+ * and positive values are unused.
+ *
+ * XXX: Tor uses this concept a lot but just calls it int. Should we move
+ * this somewhere centralized? Where?
+ */
+typedef int signed_error_t;
+
+/**
+ * These constants specify the types of events that can cause
+ * transitions between state machine states.
+ *
+ * Note that SENT and RECV are relative to this endpoint. For
+ * relays, SENT means packets destined towards the client and
+ * RECV means packets destined towards the relay. On the client,
+ * SENT means packets destined towards the relay, where as RECV
+ * means packets destined towards the client.
+ */
+typedef enum {
+  /* A non-padding cell was received. */
+  CIRCPAD_EVENT_NONPADDING_RECV = 0,
+  /* A non-padding cell was sent. */
+  CIRCPAD_EVENT_NONPADDING_SENT = 1,
+  /* A padding cell (RELAY_COMMAND_DROP) was sent. */
+  CIRCPAD_EVENT_PADDING_SENT = 2,
+  /* A padding cell was received. */
+  CIRCPAD_EVENT_PADDING_RECV = 3,
+  /* We tried to schedule padding but we ended up picking the infinity bin
+   * which means that padding was delayed infinitely */
+  CIRCPAD_EVENT_INFINITY = 4,
+  /* All histogram bins are empty (we are out of tokens) */
+  CIRCPAD_EVENT_BINS_EMPTY = 5,
+  /* This state has used up its cell count */
+  CIRCPAD_EVENT_LENGTH_COUNT = 6
+} circpad_event_t;
+#define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1)
+
+/** Boolean type that says if we decided to transition states or not */
+typedef enum {
+  CIRCPAD_STATE_UNCHANGED = 0,
+  CIRCPAD_STATE_CHANGED = 1
+} circpad_decision_t;
+
+/** The type for the things in histogram bins (aka tokens) */
+typedef uint32_t circpad_hist_token_t;
+
+/** The type for histogram indexes (needs to be negative for errors) */
+typedef int8_t circpad_hist_index_t;
+
+/** The type for absolute time, from monotime_absolute_usec() */
+typedef uint64_t circpad_time_t;
+
+/** The type for timer delays, in microseconds */
+typedef uint32_t circpad_delay_t;
+#define CIRCPAD_DELAY_UNITS_PER_SECOND  (1000*1000)
+
+/**
+ * An infinite padding cell delay means don't schedule any padding --
+ * simply wait until a different event triggers a transition.
+ *
+ * This means that the maximum delay we can schedule is UINT32_MAX-1
+ * microseconds, or about 4300 seconds (1.25 hours).
+ * XXX: Is this enough if we want to simulate light, intermittent
+ * activity on an onion service?
+ */
+#define CIRCPAD_DELAY_INFINITE  (UINT32_MAX)
+
+/**
+ * This is the maximum delay that the circuit padding system can have, in
+ * seconds.
+ */
+#define CIRCPAD_DELAY_MAX_SECS   \
+    ((CIRCPAD_DELAY_INFINITE/CIRCPAD_DELAY_UNITS_PER_SECOND)+1)
+
+/**
+ * Macro to clarify when we're checking the infinity bin.
+ *
+ * Works with either circpad_state_t or circpad_machine_runtime_t
+ */
+#define CIRCPAD_INFINITY_BIN(mi)  ((mi)->histogram_len-1)
+
+/**
+ * These constants form a bitfield that specifies when a state machine
+ * should be applied to a circuit.
+ *
+ * If any of these elements is set, then the circuit will be tested against
+ * that specific condition. If an element is unset, then we don't test it.
+ * (E.g., if neither NO_STREAMS or STREAMS are set, then we will not care
+ * whether a circuit has streams attached when we apply a state machine.)
+ *
+ * The helper function circpad_circuit_state() converts circuit state
+ * flags into this more compact representation.
+ */
+typedef enum {
+  /* Only apply machine if the circuit is still building */
+  CIRCPAD_CIRC_BUILDING = 1<<0,
+  /* Only apply machine if the circuit is open */
+  CIRCPAD_CIRC_OPENED = 1<<1,
+  /* Only apply machine if the circuit has no attached streams */
+  CIRCPAD_CIRC_NO_STREAMS = 1<<2,
+  /* Only apply machine if the circuit has attached streams */
+  CIRCPAD_CIRC_STREAMS = 1<<3,
+  /* Only apply machine if the circuit still allows RELAY_EARLY cells */
+  CIRCPAD_CIRC_HAS_RELAY_EARLY = 1<<4,
+  /* Only apply machine if the circuit has depleted its RELAY_EARLY cells
+   * allowance. */
+  CIRCPAD_CIRC_HAS_NO_RELAY_EARLY = 1<<5
+} circpad_circuit_state_t;
+
+/** Bitmask that says "apply this machine to all states" */
+#define CIRCPAD_STATE_ALL   \
+    (CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED| \
+     CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_NO_STREAMS| \
+     CIRCPAD_CIRC_HAS_RELAY_EARLY|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY)
+
+/**
+ * A compact circuit purpose bitfield mask that allows us to compactly
+ * specify which circuit purposes a machine should apply to.
+ *
+ * The helper function circpad_circ_purpose_to_mask() converts circuit
+ * purposes into bit positions in this bitmask.
+ */
+typedef uint32_t circpad_purpose_mask_t;
+
+/** Bitmask that says "apply this machine to all purposes". */
+#define CIRCPAD_PURPOSE_ALL (0xFFFFFFFF)
+
+/**
+ * This type specifies all of the conditions that must be met before
+ * a client decides to initiate padding on a circuit.
+ *
+ * A circuit must satisfy every sub-field in this type in order
+ * to be considered to match the conditions.
+ */
+typedef struct circpad_machine_conditions_t {
+  /** Only apply the machine *if* the circuit has at least this many hops */
+  unsigned min_hops : 3;
+
+  /** Only apply the machine *if* vanguards are enabled */
+  unsigned requires_vanguards : 1;
+
+  /**
+   * This machine is ok to use if reduced padding is set in consensus
+   * or torrc. This machine will still be applied even if reduced padding
+   * is not set; this flag only acts to exclude machines that don't have
+   * it set when reduced padding is requested. Therefore, reduced padding
+   * machines should appear at the lowest priority in the padding machine
+   * lists (aka first in the list), so that non-reduced padding machines
+   * for the same purpose are given a chance to apply when reduced padding
+   * is not requested. */
+  unsigned reduced_padding_ok : 1;
+
+  /** Only apply the machine *if* the circuit's state matches any of
+   *  the bits set in this bitmask. */
+  circpad_circuit_state_t state_mask;
+
+  /** Only apply a machine *if* the circuit's purpose matches one
+   *  of the bits set in this bitmask */
+  circpad_purpose_mask_t purpose_mask;
+
+} circpad_machine_conditions_t;
+
+/**
+ * Token removal strategy options.
+ *
+ * The WTF-PAD histograms are meant to specify a target distribution to shape
+ * traffic towards. This is accomplished by removing tokens from the histogram
+ * when either padding or non-padding cells are sent.
+ *
+ * When we see a non-padding cell at a particular time since the last cell, you
+ * remove a token from the corresponding delay bin. These flags specify
+ * which bin to choose if that bin is already empty.
+ */
+typedef enum {
+  /** Don't remove any tokens */
+  CIRCPAD_TOKEN_REMOVAL_NONE = 0,
+  /**
+   * Remove from the first non-zero higher bin index when current is zero.
+   * This is the recommended strategy from the Adaptive Padding paper. */
+  CIRCPAD_TOKEN_REMOVAL_HIGHER = 1,
+  /** Remove from the first non-zero lower bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_LOWER = 2,
+  /** Remove from the closest non-zero bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST = 3,
+  /** Remove from the closest bin by time value (since bins are
+   *  exponentially spaced). */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC = 4,
+  /** Only remove from the exact bin corresponding to this delay. If
+   *  the bin is 0, simply do nothing. Don't pick another bin. */
+  CIRCPAD_TOKEN_REMOVAL_EXACT = 5
+} circpad_removal_t;
+
+/**
+ * Distribution types supported by circpad_distribution_sample().
+ *
+ * These can be used instead of histograms for the inter-packet
+ * timing distribution, or to specify a distribution on the number
+ * of cells that can be sent while in a specific state of the state
+ * machine.
+ *
+ * Each distribution takes up to two parameters which are described below. */
+typedef enum {
+  /* No probability distribution is used */
+  CIRCPAD_DIST_NONE = 0,
+  /* Uniform distribution: param1 is lower bound and param2 is upper bound */
+  CIRCPAD_DIST_UNIFORM = 1,
+  /* Logistic distribution: param1 is Mu, param2 is sigma. */
+  CIRCPAD_DIST_LOGISTIC = 2,
+  /* Log-logistic distribution: param1 is Alpha, param2 is 1.0/Beta */
+  CIRCPAD_DIST_LOG_LOGISTIC = 3,
+  /* Geometric distribution: param1 is 'p' (success probability) */
+  CIRCPAD_DIST_GEOMETRIC = 4,
+  /* Weibull distribution: param1 is k, param2 is Lambda */
+  CIRCPAD_DIST_WEIBULL = 5,
+  /* Generalized Pareto distribution: param1 is sigma, param2 is xi */
+  CIRCPAD_DIST_PARETO = 6
+} circpad_distribution_type_t;
+
+/**
+ * Distribution information.
+ *
+ * This type specifies a specific distribution above, as well as
+ * up to two parameters for that distribution. The specific
+ * per-distribution meaning of these parameters is specified
+ * in circpad_distribution_sample().
+ */
+typedef struct circpad_distribution_t {
+  circpad_distribution_type_t type;
+  double param1;
+  double param2;
+} circpad_distribution_t;
+
+/** State number type. Represents current state of state machine. */
+typedef uint16_t circpad_statenum_t;
+#define  CIRCPAD_STATENUM_MAX   (UINT16_MAX)
+
+/** A histogram can be used to sample padding delays given a machine state.
+ * This constant defines the maximum histogram width (i.e. the max number of
+ * bins).
+ *
+ * The current limit is arbitrary and could be raised if there is a need,
+ * however too many bins will be hard to serialize in the future.
+ *
+ * Memory concerns are not so great here since the corresponding histogram and
+ * histogram_edges arrays are global and not per-circuit.
+ *
+ * If we ever upgrade this to a value that can't be represented by 8-bits we
+ * also need to upgrade circpad_hist_index_t.
+ */
+#define CIRCPAD_MAX_HISTOGRAM_LEN (100)
+
+/**
+ * A state of a padding state machine. The information here are immutable and
+ * represent the initial form of the state; it does not get updated as things
+ * happen. The mutable information that gets updated in runtime are carried in
+ * a circpad_machine_runtime_t.
+ *
+ * This struct describes the histograms and/or probability distributions, as
+ * well as parameters of a single state in the adaptive padding machine.
+ * Instances of this struct exist in global circpad machine definitions that
+ * come from torrc or the consensus.
+ */
+typedef struct circpad_state_t {
+  /**
+   * If a histogram is used for this state, this specifies the number of bins
+   * of this histogram. Histograms must have at least 2 bins.
+   *
+   * In particular, the following histogram:
+   *
+   * Tokens
+   *         +
+   *      10 |    +----+
+   *       9 |    |    |           +---------+
+   *       8 |    |    |           |         |
+   *       7 |    |    |     +-----+         |
+   *       6 +----+ Bin+-----+     |         +---------------+
+   *       5 |    | #1 |     |     |         |               |
+   *         | Bin|    | Bin | Bin |  Bin #4 |    Bin #5     |
+   *         | #0 |    | #2  | #3  |         | (infinity bin)|
+   *         |    |    |     |     |         |               |
+   *         |    |    |     |     |         |               |
+   *       0 +----+----+-----+-----+---------+---------------+
+   *         0   100  200   350   500      1000              ∞  microseconds
+   *
+   * would be specified the following way:
+   *    histogram_len = 6;
+   *    histogram[] =        {   6,  10,   6,  7,    9,     6 }
+   *    histogram_edges[] =  { 0, 100, 200, 350, 500, 1000 }
+   *
+   * The final bin is called the "infinity bin" and if it's chosen we don't
+   * schedule any padding. The infinity bin is strange because its lower edge
+   * is the max value of possible non-infinite delay allowed by this histogram,
+   * and its upper edge is CIRCPAD_DELAY_INFINITE. You can tell if the infinity
+   * bin is chosen by inspecting its bin index or inspecting its upper edge.
+   *
+   * If a delay probability distribution is used for this state, this is set
+   * to 0. */
+  circpad_hist_index_t histogram_len;
+  /** The histogram itself: an array of uint16s of tokens, whose
+   *  widths are exponentially spaced, in microseconds.
+   *
+   *  This array must have histogram_len elements that are strictly
+   *  monotonically increasing. */
+  circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN];
+  /* The histogram bin edges in usec.
+   *
+   * Each element of this array specifies the left edge of the corresponding
+   * bin. The rightmost edge is always infinity and is not specified in this
+   * array.
+   *
+   * This array must have histogram_len elements. */
+  circpad_delay_t histogram_edges[CIRCPAD_MAX_HISTOGRAM_LEN+1];
+  /** Total number of tokens in this histogram. This is a constant and is *not*
+   *  decremented every time we spend a token. It's used for initializing and
+   *  refilling the histogram. */
+  uint32_t histogram_total_tokens;
+
+  /**
+   * Represents a delay probability distribution (aka IAT distribution). It's a
+   * parametrized way of encoding inter-packet delay information in
+   * microseconds. It can be used instead of histograms.
+   *
+   * If it is used, token_removal below must be set to
+   * CIRCPAD_TOKEN_REMOVAL_NONE.
+   *
+   * Start_usec, range_sec, and rtt_estimates are still applied to the
+   * results of sampling from this distribution (range_sec is used as a max).
+   */
+  circpad_distribution_t iat_dist;
+  /*  If a delay probability distribution is used, this is used as the max
+   *  value we can sample from the distribution. However, RTT measurements and
+   *  dist_added_shift gets applied on top of this value to derive the final
+   *  padding delay. */
+  circpad_delay_t dist_max_sample_usec;
+  /*  If a delay probability distribution is used and this is set, we will add
+   *  this value on top of the value sampled from the IAT distribution to
+   *  derive the final padding delay (We also add the RTT measurement if it's
+   *  enabled.). */
+  circpad_delay_t dist_added_shift_usec;
+
+  /**
+   * The length dist is a parameterized way of encoding how long this
+   * state machine runs in terms of sent padding cells or all
+   * sent cells. Values are sampled from this distribution, clamped
+   * to max_len, and then start_len is added to that value.
+   *
+   * It may be specified instead of or in addition to
+   * the infinity bins and bins empty conditions. */
+  circpad_distribution_t length_dist;
+  /** A minimum length value, added to the output of length_dist */
+  uint16_t start_length;
+  /** A cap on the length value that can be sampled from the length_dist */
+  uint64_t max_length;
+
+  /** Should we decrement length when we see a nonpadding packet?
+   * XXX: Are there any machines that actually want to set this to 0? There may
+   * not be. OTOH, it's only a bit.. */
+  unsigned length_includes_nonpadding : 1;
+
+  /**
+   * This is an array that specifies the next state to transition to upon
+   * receipt an event matching the indicated array index.
+   *
+   * This aborts our scheduled packet and switches to the state
+   * corresponding to the index of the array. Tokens are filled upon
+   * this transition.
+   *
+   * States are allowed to transition to themselves, which means re-schedule
+   * a new padding timer. They are also allowed to temporarily "transition"
+   * to the "IGNORE" and "CANCEL" pseudo-states. See #defines below
+   * for details on state behavior and meaning.
+   */
+  circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS];
+
+  /**
+   * If true, estimate the RTT from this relay to the exit/website and add that
+   * to start_usec for use as the histogram bin 0 start delay.
+   *
+   * Right now this is only supported for relay-side state machines.
+   */
+  unsigned use_rtt_estimate : 1;
+
+  /** This specifies the token removal strategy to use upon padding and
+   *  non-padding activity. */
+  circpad_removal_t token_removal;
+} circpad_state_t;
+
+/**
+ * The start state for this machine.
+ *
+ * In the original WTF-PAD, this is only used for transition to/from
+ * the burst state. All other fields are not used. But to simplify the
+ * code we've made it a first-class state. This has no performance
+ * consequences, but may make naive serialization of the state machine
+ * large, if we're not careful about how we represent empty fields.
+ */
+#define  CIRCPAD_STATE_START       0
+
+/**
+ * The burst state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD
+ * (https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf and
+ * https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/), the burst
+ * state serves to detect bursts in traffic. This is done by using longer
+ * delays in its histogram, which represent the expected delays between
+ * bursts of packets in the target stream. If this delay expires without a
+ * real packet being sent, the burst state sends a padding packet and then
+ * immediately transitions to the gap state, which is used to generate
+ * a synthetic padding packet train. In this implementation, this transition
+ * needs to be explicitly specified in the burst state's transition events.
+ *
+ * Because of this flexibility, other padding mechanisms can transition
+ * between these two states arbitrarily, to encode other dynamics of
+ * target traffic.
+ */
+#define  CIRCPAD_STATE_BURST       1
+
+/**
+ * The gap state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD, the gap
+ * state serves to simulate an artificial packet train composed of padding
+ * packets. It does this by specifying much lower inter-packet delays than
+ * the burst state, and transitioning back to itself after padding is sent
+ * if these timers expire before real traffic is sent. If real traffic is
+ * sent, it transitions back to the burst state.
+ *
+ * Again, in this implementation, these transitions must be specified
+ * explicitly, and other transitions are also permitted.
+ */
+#define  CIRCPAD_STATE_GAP         2
+
+/**
+ * End is a pseudo-state that causes the machine to go completely
+ * idle, and optionally get torn down (depending on the
+ * value of circpad_machine_spec_t.should_negotiate_end)
+ *
+ * End MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_END         CIRCPAD_STATENUM_MAX
+
+/**
+ * "Ignore" is a pseudo-state that means "do not react to this
+ * event".
+ *
+ * "Ignore" MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_IGNORE         (CIRCPAD_STATENUM_MAX-1)
+
+/**
+ * "Cancel" is a pseudo-state that means "cancel pending timers,
+ * but remain in your current state".
+ *
+ * Cancel MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_CANCEL         (CIRCPAD_STATENUM_MAX-2)
+
+/**
+ * Since we have 3 pseudo-states, the max state array length is
+ * up to one less than cancel's statenum.
+ */
+#define CIRCPAD_MAX_MACHINE_STATES  (CIRCPAD_STATE_CANCEL-1)
+
+/**
+ * Mutable padding machine info.
+ *
+ * This structure contains mutable information about a padding
+ * machine. The mutable information must be kept separate because
+ * it exists per-circuit, where as the machines themselves are global.
+ * This separation is done to conserve space in the circuit structure.
+ *
+ * This is the per-circuit state that changes regarding the global state
+ * machine. Some parts of it are optional (ie NULL).
+ *
+ * XXX: Play with layout to minimize space on x64 Linux (most common relay).
+ */
+typedef struct circpad_machine_runtime_t {
+  /** The callback pointer for the padding callbacks.
+   *
+   *  These timers stick around the machineinfo until the machineinfo's circuit
+   *  is closed, at which point the timer is cancelled. For this reason it's
+   *  safe to assume that the machineinfo exists if this timer gets
+   *  triggered. */
+  tor_timer_t *padding_timer;
+
+  /** The circuit for this machine */
+  struct circuit_t *on_circ;
+
+  /** A mutable copy of the histogram for the current state.
+   *  NULL if remove_tokens is false for that state */
+  circpad_hist_token_t *histogram;
+  /** Length of the above histogram.
+   * XXX: This field *could* be removed at the expense of added
+   * complexity+overhead for reaching back into the immutable machine
+   * state every time we need to inspect the histogram. It's only a byte,
+   * though, so it seemed worth it.
+   */
+  circpad_hist_index_t histogram_len;
+  /** Remove token from this index upon sending padding */
+  circpad_hist_index_t chosen_bin;
+
+  /** Stop padding/transition if this many cells sent */
+  uint64_t state_length;
+#define CIRCPAD_STATE_LENGTH_INFINITE UINT64_MAX
+
+  /** A scaled count of padding packets sent, used to limit padding overhead.
+   * When this reaches UINT16_MAX, we cut it and nonpadding_sent in half. */
+  uint16_t padding_sent;
+  /** A scaled count of non-padding packets sent, used to limit padding
+   *  overhead. When this reaches UINT16_MAX, we cut it and padding_sent in
+   *  half. */
+  uint16_t nonpadding_sent;
+
+  /**
+   * Timestamp of the most recent cell event (sent, received, padding,
+   * non-padding), in seconds from approx_time().
+   *
+   * Used as an emergency break to stop holding padding circuits open.
+   */
+  time_t last_cell_time_sec;
+
+  /**
+   * EWMA estimate of the RTT of the circuit from this hop
+   * to the exit end, in microseconds. */
+  circpad_delay_t rtt_estimate_usec;
+
+  /**
+   * The last time we got an event relevant to estimating
+   * the RTT. Monotonic time in microseconds since system
+   * start.
+   */
+  circpad_time_t last_received_time_usec;
+
+  /**
+   * The time at which we scheduled a non-padding packet,
+   * or selected an infinite delay.
+   *
+   * Monotonic time in microseconds since system start.
+   * This is 0 if we haven't chosen a padding delay.
+   */
+  circpad_time_t padding_scheduled_at_usec;
+
+  /** What state is this machine in? */
+  circpad_statenum_t current_state;
+
+  /**
+   * True if we have scheduled a timer for padding.
+   *
+   * This is 1 if a timer is pending. It is 0 if
+   * no timer is scheduled. (It can be 0 even when
+   * padding_was_scheduled_at_usec is non-zero).
+   */
+  unsigned is_padding_timer_scheduled : 1;
+
+  /**
+   * If this is true, we have seen full duplex behavior.
+   * Stop updating the RTT.
+   */
+  unsigned stop_rtt_update : 1;
+
+/** Max number of padding machines on each circuit. If changed,
+ * also ensure the machine_index bitwith supports the new size. */
+#define CIRCPAD_MAX_MACHINES    (2)
+  /** Which padding machine index was this for.
+   * (make sure changes to the bitwidth can support the
+   * CIRCPAD_MAX_MACHINES define). */
+  unsigned machine_index : 1;
+
+} circpad_machine_runtime_t;
+
+/** Helper macro to get an actual state machine from a machineinfo */
+#define CIRCPAD_GET_MACHINE(machineinfo) \
+    ((machineinfo)->on_circ->padding_machine[(machineinfo)->machine_index])
+
+/**
+ * This specifies a particular padding machine to use after negotiation.
+ *
+ * The constants for machine_num_t are in trunnel.
+ * We want to be able to define extra numbers in the consensus/torrc, though.
+ */
+typedef uint8_t circpad_machine_num_t;
+
+/** Global state machine structure from the consensus */
+typedef struct circpad_machine_spec_t {
+  /* Just a user-friendly machine name for logs */
+  const char *name;
+
+  /** Global machine number */
+  circpad_machine_num_t machine_num;
+
+  /** Which machine index slot should this machine go into in
+   *  the array on the circuit_t */
+  unsigned machine_index : 1;
+
+  /** Send a padding negotiate to shut down machine at end state? */
+  unsigned should_negotiate_end : 1;
+
+  // These next three fields are origin machine-only...
+  /** Origin side or relay side */
+  unsigned is_origin_side : 1;
+
+  /** Which hop in the circuit should we send padding to/from?
+   *  1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */
+  unsigned target_hopnum : 3;
+
+  /** If this flag is enabled, don't close circuits that use this machine even
+   *  if another part of Tor wants to close this circuit.
+   *
+   *  If this flag is set, the circuitpadding subsystem will close circuits the
+   *  moment the machine transitions to the END state, and only if the circuit
+   *  has already been asked to be closed by another part of Tor.
+   *
+   *  Circuits that should have been closed but were kept open by a padding
+   *  machine are re-purposed to CIRCUIT_PURPOSE_C_CIRCUIT_PADDING, hence
+   *  machines should take that purpose into account if they are filtering
+   *  circuits by purpose. */
+  unsigned manage_circ_lifetime : 1;
+
+  /** This machine only kills fascists if the following conditions are met. */
+  circpad_machine_conditions_t conditions;
+
+  /** How many padding cells can be sent before we apply overhead limits?
+   * XXX: Note that we can only allow up to 64k of padding cells on an
+   * otherwise quiet circuit. Is this enough? It's 33MB. */
+  uint16_t allowed_padding_count;
+
+  /** Padding percent cap: Stop padding if we exceed this percent overhead.
+   * 0 means no limit. Overhead is defined as percent of total traffic, so
+   * that we can use 0..100 here. This is the same definition as used in
+   * Prop#265. */
+  uint8_t max_padding_percent;
+
+  /** State array: indexed by circpad_statenum_t */
+  circpad_state_t *states;
+
+  /**
+   * Number of states this machine has (ie: length of the states array).
+   * XXX: This field is not needed other than for safety. */
+  circpad_statenum_t num_states;
+} circpad_machine_spec_t;
+
+void circpad_new_consensus_params(const networkstatus_t *ns);
+
+int circpad_marked_circuit_for_padding(circuit_t *circ, int reason);
+
+/**
+ * The following are event call-in points that are of interest to
+ * the state machines. They are called during cell processing. */
+void circpad_deliver_unrecognized_cell_events(struct circuit_t *circ,
+                                              cell_direction_t dir);
+void circpad_deliver_sent_relay_cell_events(struct circuit_t *circ,
+                                            uint8_t relay_command);
+void circpad_deliver_recognized_relay_cell_events(struct circuit_t *circ,
+                                                  uint8_t relay_command,
+                                                  crypt_path_t *layer_hint);
+
+/** Cell events are delivered by the above delivery functions */
+void circpad_cell_event_nonpadding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_nonpadding_received(struct circuit_t *on_circ);
+void circpad_cell_event_padding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_padding_received(struct circuit_t *on_circ);
+
+/** Internal events are events the machines send to themselves */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi);
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *);
+circpad_decision_t circpad_internal_event_state_length_up(
+                                  circpad_machine_runtime_t *);
+
+/** Machine creation events are events that cause us to set up or
+ *  tear down padding state machines. */
+void circpad_machine_event_circ_added_hop(struct origin_circuit_t *on_circ);
+void circpad_machine_event_circ_built(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_purpose_changed(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_streams(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_no_streams(struct origin_circuit_t *circ);
+void
+circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ);
+
+void circpad_machines_init(void);
+void circpad_machines_free(void);
+void circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                      smartlist_t *machine_list);
+
+void circpad_machine_states_init(circpad_machine_spec_t *machine,
+                                 circpad_statenum_t num_states);
+
+void circpad_circuit_free_all_machineinfos(struct circuit_t *circ);
+
+bool circpad_padding_is_from_expected_hop(struct circuit_t *circ,
+                                         crypt_path_t *from_hop);
+
+/** Serializaton functions for writing to/from torrc and consensus */
+char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine);
+const circpad_machine_spec_t *circpad_string_to_machine(const char *str);
+
+/* Padding negotiation between client and middle */
+signed_error_t circpad_handle_padding_negotiate(struct circuit_t *circ,
+                                      struct cell_t *cell);
+signed_error_t circpad_handle_padding_negotiated(struct circuit_t *circ,
+                                      struct cell_t *cell,
+                                      crypt_path_t *layer_hint);
+signed_error_t circpad_negotiate_padding(struct origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command);
+bool circpad_padding_negotiated(struct circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response);
+
+circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose);
+
+int circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                                crypt_path_t *layer_hint,
+                                const relay_header_t *rh);
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *));
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_spec_transition, (circpad_machine_runtime_t *mi,
+                             circpad_event_t event));
+
+circpad_decision_t circpad_send_padding_cell_for_callback(
+                                 circpad_machine_runtime_t *mi);
+
+void circpad_free_all(void);
+
+#ifdef CIRCUITPADDING_PRIVATE
+STATIC void  machine_spec_free_(circpad_machine_spec_t *m);
+#define machine_spec_free(chan) \
+  FREE_AND_NULL(circpad_machine_spec_t,machine_spec_free_, (m))
+
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi);
+
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi);
+
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi);
+
+STATIC void circpad_machine_remove_token(circpad_machine_runtime_t *mi);
+
+STATIC circpad_hist_index_t circpad_histogram_usec_to_bin(
+                                       const circpad_machine_runtime_t *mi,
+                                       circpad_delay_t us);
+
+STATIC circpad_machine_runtime_t *circpad_circuit_machineinfo_new(
+                                               struct circuit_t *on_circ,
+                                               int machine_index);
+STATIC void circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us,
+                                         bool use_usec);
+STATIC void circpad_machine_setup_tokens(circpad_machine_runtime_t *mi);
+
+MOCK_DECL(STATIC signed_error_t,
+circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len));
+
+MOCK_DECL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop));
+
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl);
+
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *origin_padding_machines;
+extern smartlist_t *relay_padding_machines;
+
+#endif
+
+#endif /* defined(CIRCUITPADDING_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_H) */
diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c
new file mode 100644
index 0000000000..7220d657fc
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.c
@@ -0,0 +1,456 @@
+/* Copyright (c) 2019 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.c
+ * \brief Circuit padding state machines
+ *
+ * \detail
+ *
+ * Introduce circuit padding machines that will be used by Tor circuits, as
+ * specified by proposal 302 "Hiding onion service clients using padding".
+ *
+ * Right now this file introduces two machines that aim to hide the client-side
+ * of onion service circuits against naive classifiers like the ones from the
+ * "Circuit Fingerprinting Attacks: Passive Deanonymization of Tor Hidden
+ * Services" paper from USENIX. By naive classifiers we mean classifiers that
+ * use basic features like "circuit construction circuits" and "incoming and
+ * outgoing cell counts" and "duration of activity".
+ *
+ * In particular, these machines aim to be lightweight and protect against
+ * these basic classifiers. They don't aim to protect against more advanced
+ * attacks that use deep learning or even correlate various circuit
+ * construction events together. Machines that fool such advanced classifiers
+ * are also possible, but they can't be so lightweight and might require more
+ * WTF-PAD features. So for now we opt for the following two machines:
+ *
+ * Client-side introduction circuit hiding machine:
+ *
+ *    This machine hides client-side introduction circuits by making their
+ *    circuit consruction sequence look like normal general circuits that
+ *    download directory information. Furthermore, the circuits are kept open
+ *    until all the padding has been sent, since intro circuits are usually
+ *    very short lived and this act as a distinguisher. For more info see
+ *    circpad_machine_client_hide_intro_circuits() and the sec.
+ *
+ * Client-side rendezvous circuit hiding machine:
+ *
+ *    This machine hides client-side rendezvous circuits by making their
+ *    circuit construction sequence look like normal general circuits. For more
+ *    details see circpad_machine_client_hide_rend_circuits() and the spec.
+ *
+ * TODO: These are simple machines that carefully manipulate the cells of the
+ *   initial circuit setup procedure to make them look like general
+ *   circuits. In the future, more states can be baked into their state machine
+ *   to do more advanced obfuscation.
+ **/
+
+#define CIRCUITPADDING_MACHINES_PRIVATE
+
+#include "core/or/or.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/circuitlist.h"
+
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitpadding.h"
+
+/** Create a client-side padding machine that aims to hide IP circuits. In
+ *  particular, it keeps intro circuits alive until a bunch of fake traffic has
+ *  been pushed through.
+ */
+void
+circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_ip_circ";
+
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad introduction circuits, and we want to start padding
+   * only after the INTRODUCE1 cell has been sent, so set the purposes
+   * appropriately.
+   *
+   * In particular we want introduction circuits to blend as much as possible
+   * with general circuits. Most general circuits have the following initial
+   * relay cell sequence (outgoing cells marked in [brackets]):
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * Whereas normal introduction circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2
+   *   -> [INTRO1] -> INTRODUCE_ACK
+   *
+   * This means that up to the sixth cell (first line of each sequence above),
+   * both general and intro circuits have identical cell sequences. After that
+   * we want to mimic the second line sequence of
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * We achieve this by starting padding INTRODUCE1 has been sent. With padding
+   * negotiation cells, in the common case of the second line looks like:
+   *   -> [INTRO1] -> [PADDING_NEGOTIATE] -> PADDING_NEGOTIATED -> INTRO_ACK
+   *
+   * Then, the middle node will send between INTRO_MACHINE_MINIMUM_PADDING and
+   * INTRO_MACHINE_MAXIMUM_PADDING cells, to match the "...(inbound data cells
+   * continue)" portion of the trace (aka the rest of an HTTPS response body).
+   */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Keep the circuit alive even after the introduction has been finished,
+   * otherwise the short-term lifetime of the circuit will blow our cover */
+  client_machine->manage_circ_lifetime = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* For the origin-side machine, we transition to OBFUSCATE_CIRC_SETUP after
+   * sending PADDING_NEGOTIATE, and we stay there (without sending any padding)
+   * until we receive a STOP from the other side. */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* origin-side machine has no event reactions while in
+   * CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP, so no more state transitions here. */
+
+  /* The client side should never send padding, so it does not need
+   * to specify token removal, or a histogram definition or state lengths.
+   * That is all controlled by the middle node. */
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client intro point hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits. See
+ *  comments on the function above for more details on the workings of the
+ *  machine. */
+void
+circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_ip_circ";
+
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* We want to negotiate END from this side after all our padding is done, so
+   * that the origin-side machine goes into END state, and eventually closes
+   * the circuit. */
+  relay_machine->should_negotiate_end = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* For the relay-side machine, we want to transition
+   * START -> OBFUSCATE_CIRC_SETUP upon first non-padding
+   * cell sent (PADDING_NEGOTIATED in this case).  */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* For the relay-side, we want to transition from OBFUSCATE_CIRC_SETUP to END
+   * state when the length finishes. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Now let's define the OBF -> OBF transitions that maintain our padding
+   * flow:
+   *
+   * For the relay-side machine, we want to keep on sending padding bytes even
+   * when nothing else happens on this circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_PADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+  /* For the relay-side machine, we need this transition so that we re-enter
+     the state, after PADDING_NEGOTIATED is sent. Otherwise, the remove token
+     function will disable the timer, and nothing will restart it since there
+     is no other motion on an intro circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* Token removal strategy for OBFUSCATE_CIRC_SETUP state: Don't
+   * remove any tokens.
+   *
+   * We rely on the state length sampling and not token removal, to avoid
+   * the mallocs required to copy the histograms for token removal,
+   * and to avoid monotime calls needed to determine histogram
+   * bins for token removal. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Figure out the length of the OBFUSCATE_CIRC_SETUP state so that it's
+   * randomized. The relay side will send between INTRO_MACHINE_MINIMUM_PADDING
+   * and INTRO_MACHINE_MAXIMUM_PADDING padding cells towards the client. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = INTRO_MACHINE_MINIMUM_PADDING;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = INTRO_MACHINE_MAXIMUM_PADDING;
+
+  /* Configure histogram */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+     histogram_len = 2;
+
+  /* For the relay-side machine we want to batch padding instantly to pretend
+   * its an incoming directory download. So set the histogram edges tight:
+   * (1, 10ms, infinity). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 1000;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 10000;
+
+  /* We put all our tokens in bin 0, which means we want 100% probability
+   * for choosing a inter-packet delay of between 1000 and 10000 microseconds
+   * (1 to 10ms). Since we only have 1 bin, it doesn't matter how many tokens
+   * there are, 1000 out of 1000 is 100% */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1000;
+
+  /* just one bin, so setup the total tokens */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens =
+      relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].histogram[0];
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay intro circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
+
+/************************** Rendezvous-circuit machine ***********************/
+
+/** Create a client-side padding machine that aims to hide rendezvous
+ *  circuits.*/
+void
+circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad rendezvous circuits, and we want to start padding only
+   * after the rendezvous circuit has been established.
+   *
+   * Following a similar argument as for intro circuits, we are aiming for
+   * padded rendezvous circuits to blend in with the initial cell sequence of
+   * general circuits which usually look like this:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *    -> [DATA] -> [DATA] -> DATA -> DATA...(incoming cells continue)
+   *
+   * Whereas normal rendezvous circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> REND2 -> [BEGIN]
+   *
+   * This means that up to the sixth cell (in the first line), both general and
+   * rend circuits have identical cell sequences.
+   *
+   * After that we want to mimic a [DATA] -> [DATA] -> DATA -> DATA sequence.
+   *
+   * With padding negotiation right after the REND_ESTABLISHED, the sequence
+   * becomes:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP...
+   *
+   * After which normal application DATA cells continue on the circuit.
+   *
+   * Hence this way we make rendezvous circuits look like general circuits up
+   * till the end of the circuit setup. */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_JOINED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = 1;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATE) */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that we send our outgoing [DROP] before the PADDING_NEGOTIATED comes
+   * back from the relay side. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client rendezvous circuit hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits.
+ *
+ *  This is meant to follow the client-side machine.
+ */
+void
+circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+    = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  relay_machine->conditions.min_hops = 2;
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = 1;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATED) */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_END;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that the outgoing DROP cell is sent immediately after the
+   * PADDING_NEGOTIATED. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay rendezvous circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
diff --git a/src/core/or/circuitpadding_machines.h b/src/core/or/circuitpadding_machines.h
new file mode 100644
index 0000000000..3c9798d42d
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.h
+ * \brief Header file for circuitpadding_machines.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_MACHINES_H
+#define TOR_CIRCUITPADDING_MACHINES_H
+
+void circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl);
+
+#ifdef CIRCUITPADDING_MACHINES_PRIVATE
+
+/** State of the padding machines that actually sends padding */
+#define CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP CIRCPAD_STATE_BURST
+
+/** Constants defining the amount of padding that a machine will send to hide
+ *  HS circuits. The actual value is sampled uniformly random between the
+ *  min/max values.
+ */
+
+/** Minimum number of relay-side padding cells to be sent by this machine */
+#define INTRO_MACHINE_MINIMUM_PADDING 7
+/** Maximum number of relay-side padding cells to be sent by this machine.
+ *  The actual value will be sampled between the min and max.*/
+#define INTRO_MACHINE_MAXIMUM_PADDING 10
+
+#endif /* defined(CIRCUITPADDING_MACHINES_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_MACHINES_H) */
diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c
index 2cde21fa1f..7a7f3ca600 100644
--- a/src/core/or/circuitstats.c
+++ b/src/core/or/circuitstats.c
@@ -29,8 +29,8 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
-#include "feature/control/control.h"
+#include "lib/confmgt/confparse.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/networkstatus.h"
@@ -44,6 +44,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/relay/relay_periodic.h"
 
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -639,9 +640,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n)
 void
 circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ,
-                               CIRC_EVENT_FAILED,
-                               END_CIRC_REASON_TIMEOUT);
+  circuit_event_status(circ,
+                       CIRC_EVENT_FAILED,
+                       END_CIRC_REASON_TIMEOUT);
   circuit_change_purpose(TO_CIRCUIT(circ),
                          CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
   /* Record this event to check for too many timeouts
@@ -1420,6 +1421,7 @@ void
 circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 {
   time_t now = approx_time();
+  // XXXX this should use pubsub
   if (cbt->liveness.nonlive_timeouts > 0) {
     time_t time_since_live = now - cbt->liveness.network_last_live;
     log_notice(LD_CIRC,
diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c
index 7273be3510..284e315f9c 100644
--- a/src/core/or/circuituse.c
+++ b/src/core/or/circuituse.c
@@ -35,13 +35,14 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_client.h"
@@ -69,7 +70,7 @@
 #include "core/or/origin_circuit_st.h"
 #include "core/or/socks_request_st.h"
 
-static void circuit_expire_old_circuits_clientside(void);
+STATIC void circuit_expire_old_circuits_clientside(void);
 static void circuit_increment_failure_count(void);
 
 /** Check whether the hidden service destination of the stream at
@@ -177,7 +178,6 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
       purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
       purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
     tor_addr_t addr;
-    const int family = tor_addr_parse(&addr, conn->socks_request->address);
     if (!exitnode && !build_state->onehop_tunnel) {
       log_debug(LD_CIRC,"Not considering circuit with unknown router.");
       return 0; /* this circuit is screwed and doesn't know it yet,
@@ -198,6 +198,8 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
           return 0; /* this is a circuit to somewhere else */
         if (tor_digest_is_zero(digest)) {
           /* we don't know the digest; have to compare addr:port */
+          const int family = tor_addr_parse(&addr,
+                                            conn->socks_request->address);
           if (family < 0 ||
               !tor_addr_eq(&build_state->chosen_exit->addr, &addr) ||
               build_state->chosen_exit->port != conn->socks_request->port)
@@ -210,12 +212,14 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
         return 0;
       }
     }
-    if (origin_circ->prepend_policy && family != -1) {
-      int r = compare_tor_addr_to_addr_policy(&addr,
-                                              conn->socks_request->port,
-                                              origin_circ->prepend_policy);
-      if (r == ADDR_POLICY_REJECTED)
-        return 0;
+    if (origin_circ->prepend_policy) {
+      if (tor_addr_parse(&addr, conn->socks_request->address) != -1) {
+        int r = compare_tor_addr_to_addr_policy(&addr,
+                                                conn->socks_request->port,
+                                                origin_circ->prepend_policy);
+        if (r == ADDR_POLICY_REJECTED)
+          return 0;
+      }
     }
     if (exitnode && !connection_ap_can_use_exit(conn, exitnode)) {
       /* can't exit from this router */
@@ -1425,6 +1429,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
       if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
         hs_dec_rdv_stream_counter(origin_circ);
       }
+
+      /* If there are no more streams on this circ, tell circpad */
+      if (!origin_circ->p_streams)
+        circpad_machine_event_circ_has_no_streams(origin_circ);
+
       return;
     }
   } else {
@@ -1465,7 +1474,7 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
 /** Find each circuit that has been unused for too long, or dirty
  * for too long and has no streams on it: mark it for close.
  */
-static void
+STATIC void
 circuit_expire_old_circuits_clientside(void)
 {
   struct timeval cutoff, now;
@@ -1505,6 +1514,7 @@ circuit_expire_old_circuits_clientside(void)
                 circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT ||
                 circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
                 circ->purpose == CIRCUIT_PURPOSE_TESTING ||
+                circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING ||
                 (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
                 circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) ||
                 circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
@@ -1674,7 +1684,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop)
 void
 circuit_has_opened(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0);
+  circuit_event_status(circ, CIRC_EVENT_BUILT, 0);
 
   /* Remember that this circuit has finished building. Now if we start
    * it building again later (e.g. by extending it), we will know not
@@ -2523,8 +2533,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
           circ->rend_data = rend_data_dup(edge_conn->rend_data);
         } else if (edge_conn->hs_ident) {
           circ->hs_ident =
-            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk,
-                                 HS_IDENT_CIRCUIT_INTRO);
+            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk);
         }
         if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
             circ->base_.state == CIRCUIT_STATE_OPEN)
@@ -2596,6 +2605,12 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   /* add it into the linked list of streams on this circuit */
   log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.",
             (unsigned)circ->base_.n_circ_id);
+
+  /* If this is the first stream on this circuit, tell circpad
+   * that streams are attached */
+  if (!circ->p_streams)
+    circpad_machine_event_circ_has_streams(circ);
+
   /* reset it, so we can measure circ timeouts */
   ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL);
   ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams;
@@ -3080,6 +3095,8 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose)
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ),
                                           old_purpose);
+
+    circpad_machine_event_circ_purpose_changed(TO_ORIGIN_CIRCUIT(circ));
   }
 }
 
@@ -3113,7 +3130,9 @@ circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
 {
   if (!circ) return;
 
-  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+  tor_assertf_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE,
+                       "Wrong relay_body_len: %d (should be at most %d)",
+                       relay_body_len, RELAY_PAYLOAD_SIZE);
 
   circ->n_delivered_written_circ_bw =
       tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len);
diff --git a/src/core/or/command.c b/src/core/or/command.c
index 5fb6640c22..1c97437769 100644
--- a/src/core/or/command.c
+++ b/src/core/or/command.c
@@ -49,7 +49,7 @@
 #include "core/or/dos.h"
 #include "core/or/onion.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
@@ -182,7 +182,7 @@ command_process_cell(channel_t *chan, cell_t *cell)
     command_time_process_cell(cl, cn, & tp ## time ,            \
                               command_process_ ## tp ## _cell);  \
   } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
+#else /* !defined(KEEP_TIMING_STATS) */
 #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn)
 #endif /* defined(KEEP_TIMING_STATS) */
 
diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c
index 67a772be08..4f5f038e6c 100644
--- a/src/core/or/connection_edge.c
+++ b/src/core/or/connection_edge.c
@@ -62,21 +62,24 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/sendme.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/dnsserv.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
@@ -97,7 +100,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_util.h"
 
 #include "core/or/cell_st.h"
@@ -300,6 +303,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial)
       }
       return 0;
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
       if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) {
         /* (We already sent an end cell if possible) */
@@ -754,8 +762,13 @@ connection_edge_flushed_some(edge_connection_t *conn)
 {
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       break;
   }
   return 0;
@@ -779,7 +792,7 @@ connection_edge_finished_flushing(edge_connection_t *conn)
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       return 0;
     case AP_CONN_STATE_SOCKS_WAIT:
     case AP_CONN_STATE_NATD_WAIT:
@@ -1211,7 +1224,7 @@ connection_ap_rescan_and_attach_pending(void)
     entry_conn->marked_pending_circ_line = 0;   \
     entry_conn->marked_pending_circ_file = 0;   \
   } while (0)
-#else /* !(defined(DEBUGGING_17659)) */
+#else /* !defined(DEBUGGING_17659) */
 #define UNMARK() do { } while (0)
 #endif /* defined(DEBUGGING_17659) */
 
@@ -2803,6 +2816,31 @@ connection_ap_process_natd(entry_connection_t *conn)
   return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
 }
 
+static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 405 Method Not Allowed\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>This is an HTTP CONNECT tunnel, not a full HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>This is an HTTP CONNECT tunnel, not an HTTP proxy.</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use this Tor port as\n"
+  "an HTTP proxy.\n"
+  "</p><p>\n"
+  "This is not correct: This port is configured as a CONNECT tunnel, not\n"
+  "an HTTP proxy. Please configure your client accordingly.  You can also\n"
+  "use HTTPS; then the client should automatically use HTTP CONNECT."
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
 /** Called on an HTTP CONNECT entry connection when some bytes have arrived,
  * but we have not yet received a full HTTP CONNECT request.  Try to parse an
  * HTTP CONNECT request from the connection's inbuf.  On success, set up the
@@ -2843,7 +2881,7 @@ connection_ap_process_http_connect(entry_connection_t *conn)
   tor_assert(command);
   tor_assert(addrport);
   if (strcasecmp(command, "connect")) {
-    errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n";
+    errmsg = HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG;
     goto err;
   }
 
@@ -3706,6 +3744,10 @@ handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn)
   /* Link the circuit and the connection crypt path. */
   conn->cpath_layer = origin_circ->cpath->prev;
 
+  /* If this is the first stream on this circuit, tell circpad */
+  if (!origin_circ->p_streams)
+    circpad_machine_event_circ_has_streams(origin_circ);
+
   /* Add it into the linked list of p_streams on this circuit */
   conn->next_stream = origin_circ->p_streams;
   origin_circ->p_streams = conn;
@@ -3796,6 +3838,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
 
   if (! bcell.is_begindir) {
     /* Steal reference */
+    tor_assert(bcell.address);
     address = bcell.address;
     port = bcell.port;
 
@@ -4528,6 +4571,25 @@ circuit_clear_isolation(origin_circuit_t *circ)
   circ->socks_username_len = circ->socks_password_len = 0;
 }
 
+/** Send an END and mark for close the given edge connection conn using the
+ * given reason that has to be a stream reason.
+ *
+ * Note: We don't unattached the AP connection (if applicable) because we
+ * don't want to flush the remaining data. This function aims at ending
+ * everything quickly regardless of the connection state.
+ *
+ * This function can't fail and does nothing if conn is NULL. */
+void
+connection_edge_end_close(edge_connection_t *conn, uint8_t reason)
+{
+  if (!conn) {
+    return;
+  }
+
+  connection_edge_end(conn, reason);
+  connection_mark_for_close(TO_CONN(conn));
+}
+
 /** Free all storage held in module-scoped variables for connection_edge.c */
 void
 connection_edge_free_all(void)
diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h
index 68d8b19a11..e82b6bd765 100644
--- a/src/core/or/connection_edge.h
+++ b/src/core/or/connection_edge.h
@@ -80,6 +80,7 @@ int connection_edge_process_inbuf(edge_connection_t *conn,
 int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn);
 int connection_edge_end(edge_connection_t *conn, uint8_t reason);
 int connection_edge_end_errno(edge_connection_t *conn);
+void connection_edge_end_close(edge_connection_t *conn, uint8_t reason);
 int connection_edge_flushed_some(edge_connection_t *conn);
 int connection_edge_finished_flushing(edge_connection_t *conn);
 int connection_edge_finished_connecting(edge_connection_t *conn);
diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c
index 67157d8d0b..4c93351e31 100644
--- a/src/core/or/connection_or.c
+++ b/src/core/or/connection_or.c
@@ -22,13 +22,14 @@
  **/
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
 #define TOR_CHANNEL_INTERNAL_
 #define CONNECTION_OR_PRIVATE
+#define ORCONN_EVENT_PRIVATE
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitbuild.h"
@@ -38,7 +39,7 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/reachability.h"
@@ -46,6 +47,7 @@
 #include "lib/geoip/geoip.h"
 #include "core/mainloop/mainloop.h"
 #include "trunnel/link_handshake.h"
+#include "trunnel/netinfo.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -78,6 +80,8 @@
 #include "lib/tls/tortls.h"
 #include "lib/tls/x509.h"
 
+#include "core/or/orconn_event.h"
+
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_launch_v3_or_handshake(or_connection_t *conn);
 static int connection_or_process_cells_from_inbuf(or_connection_t *conn);
@@ -400,6 +404,55 @@ connection_or_report_broken_states(int severity, int domain)
   smartlist_free(items);
 }
 
+/**
+ * Helper function to publish an OR connection status event
+ *
+ * Publishes a messages to subscribers of ORCONN messages, and sends
+ * the control event.
+ **/
+void
+connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp,
+                           int reason)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  msg->status = tp;
+  msg->reason = reason;
+  orconn_status_publish(msg);
+  control_event_or_conn_status(conn, tp, reason);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * connection_or_change_state() calls this to notify subscribers about
+ * a change of an OR connection state.
+ **/
+static void
+connection_or_state_publish(const or_connection_t *conn, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  if (conn->is_pt) {
+    /* Do extra decoding because conn->proxy_type indicates the proxy
+     * protocol that tor uses to talk with the transport plugin,
+     * instead of PROXY_PLUGGABLE. */
+    tor_assert_nonfatal(conn->proxy_type != PROXY_NONE);
+    msg->proxy_type = PROXY_PLUGGABLE;
+  } else {
+    msg->proxy_type = conn->proxy_type;
+  }
+  msg->state = state;
+  if (conn->chan) {
+    msg->chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier;
+  } else {
+    msg->chan = 0;
+  }
+  orconn_state_publish(msg);
+}
+
 /** Call this to change or_connection_t states, so the owning channel_tls_t can
  * be notified.
  */
@@ -407,16 +460,13 @@ connection_or_report_broken_states(int severity, int domain)
 static void
 connection_or_change_state(or_connection_t *conn, uint8_t state)
 {
-  uint8_t old_state;
-
   tor_assert(conn);
 
-  old_state = conn->base_.state;
   conn->base_.state = state;
 
+  connection_or_state_publish(conn, state);
   if (conn->chan)
-    channel_tls_handle_state_change_on_orconn(conn->chan, conn,
-                                              old_state, state);
+    channel_tls_handle_state_change_on_orconn(conn->chan, conn, state);
 }
 
 /** Return the number of circuits using an or_connection_t; this used to
@@ -707,8 +757,6 @@ connection_or_finished_connecting(or_connection_t *or_conn)
 
   log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.",
             conn->address,conn->port);
-  control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  control_event_boot_first_orconn();
 
   if (proxy_type != PROXY_NONE) {
     /* start proxy handshake */
@@ -758,8 +806,8 @@ connection_or_about_to_close(or_connection_t *or_conn)
       entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan));
       if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) {
         int reason = tls_error_to_orconn_end_reason(or_conn->tls_error);
-        control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED,
-                                     reason);
+        connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED,
+                                   reason);
         if (!authdir_mode_tests_reachability(options))
           control_event_bootstrap_prob_or(
                 orconn_end_reason_to_control_string(reason),
@@ -769,10 +817,10 @@ connection_or_about_to_close(or_connection_t *or_conn)
   } else if (conn->hold_open_until_flushed) {
     /* We only set hold_open_until_flushed when we're intentionally
      * closing a connection. */
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   } else if (!tor_digest_is_zero(or_conn->identity_digest)) {
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   }
 }
@@ -1364,7 +1412,7 @@ void
 connection_or_connect_failed(or_connection_t *conn,
                              int reason, const char *msg)
 {
-  control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason);
+  connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason);
   if (!authdir_mode_tests_reachability(get_options()))
     control_event_bootstrap_prob_or(msg, reason, conn);
   note_or_connect_failed(conn);
@@ -1430,7 +1478,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
   int r;
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt = 0;
 
   tor_assert(_addr);
   tor_assert(id_digest);
@@ -1471,21 +1519,27 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
     return NULL;
   }
 
-  connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
-
   conn->is_outgoing = 1;
 
   /* If we are using a proxy server, find it and use it. */
-  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, TO_CONN(conn));
+  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         TO_CONN(conn));
   if (r == 0) {
     conn->proxy_type = proxy_type;
     if (proxy_type != PROXY_NONE) {
       tor_addr_copy(&addr, &proxy_addr);
       port = proxy_port;
       conn->base_.proxy_state = PROXY_INFANT;
+      conn->is_pt = is_pt;
     }
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
   } else {
+    /* This duplication of state change calls is necessary in case we
+     * run into an error condition below */
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
+
     /* get_proxy_addrport() might fail if we have a Bridge line that
        references a transport, but no ClientTransportPlugin lines
        defining its transport proxy. If this is the case, let's try to
@@ -1981,8 +2035,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
 
     /* Tell the new guard API about the channel failure */
     entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan));
-    control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED,
-                                 END_OR_CONN_REASON_OR_IDENTITY);
+    connection_or_event_status(conn, OR_CONN_EVENT_FAILED,
+                               END_OR_CONN_REASON_OR_IDENTITY);
     if (!authdir_mode_tests_reachability(options))
       control_event_bootstrap_prob_or(
                                 "Unexpected identity in router certificate",
@@ -2222,7 +2276,7 @@ int
 connection_or_set_state_open(or_connection_t *conn)
 {
   connection_or_change_state(conn, OR_CONN_STATE_OPEN);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0);
 
   /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection
    * that uses an earlier link protocol should not be treated as a relay. */
@@ -2252,6 +2306,8 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   cell_pack(&networkcell, cell, conn->wide_circ_ids);
 
+  /* We need to count padding cells from this non-packed code path
+   * since they are sent via chan->write_cell() (which is not packed) */
   rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   if (cell->command == CELL_PADDING)
     rep_hist_padding_count_write(PADDING_TYPE_CELL);
@@ -2262,7 +2318,7 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
   if (conn->chan) {
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
-    if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) {
+    if (TLS_CHAN_TO_BASE(conn->chan)->padding_enabled) {
       rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
       if (cell->command == CELL_PADDING)
         rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
@@ -2292,6 +2348,7 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0);
 
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   /* Touch the channel's active timestamp if there is one */
   if (conn->chan)
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
@@ -2428,6 +2485,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus)
   return 0;
 }
 
+static netinfo_addr_t *
+netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr)
+{
+  sa_family_t addr_family = tor_addr_family(tor_addr);
+
+  if (BUG(addr_family != AF_INET && addr_family != AF_INET6))
+    return NULL;
+
+  netinfo_addr_t *netinfo_addr = netinfo_addr_new();
+
+  if (addr_family == AF_INET) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4);
+    netinfo_addr_set_len(netinfo_addr, 4);
+    netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr));
+  } else if (addr_family == AF_INET6) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6);
+    netinfo_addr_set_len(netinfo_addr, 16);
+    uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr);
+    memcpy(ipv6_buf, in6_addr, 16);
+  }
+
+  return netinfo_addr;
+}
+
 /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know
  * about their address, our address, and the current time. */
 MOCK_IMPL(int,
@@ -2436,8 +2518,7 @@ connection_or_send_netinfo,(or_connection_t *conn))
   cell_t cell;
   time_t now = time(NULL);
   const routerinfo_t *me;
-  int len;
-  uint8_t *out;
+  int r = -1;
 
   tor_assert(conn->handshake_state);
 
@@ -2450,20 +2531,21 @@ connection_or_send_netinfo,(or_connection_t *conn))
   memset(&cell, 0, sizeof(cell_t));
   cell.command = CELL_NETINFO;
 
+  netinfo_cell_t *netinfo_cell = netinfo_cell_new();
+
   /* Timestamp, if we're a relay. */
   if (public_server_mode(get_options()) || ! conn->is_outgoing)
-    set_uint32(cell.payload, htonl((uint32_t)now));
+    netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now);
 
   /* Their address. */
-  out = cell.payload + 4;
+  const tor_addr_t *remote_tor_addr =
+    !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr;
   /* We use &conn->real_addr below, unless it hasn't yet been set. If it
    * hasn't yet been set, we know that base_.addr hasn't been tampered with
    * yet either. */
-  len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr)
-                                       ? &conn->real_addr : &conn->base_.addr);
-  if (len<0)
-    return -1;
-  out += len;
+  netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr);
+
+  netinfo_cell_set_other_addr(netinfo_cell, their_addr);
 
   /* My address -- only include it if I'm a public relay, or if I'm a
    * bridge and this is an incoming connection. If I'm a bridge and this
@@ -2471,28 +2553,42 @@ connection_or_send_netinfo,(or_connection_t *conn))
   if ((public_server_mode(get_options()) || !conn->is_outgoing) &&
       (me = router_get_my_routerinfo())) {
     tor_addr_t my_addr;
-    *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr);
-
     tor_addr_from_ipv4h(&my_addr, me->addr);
-    len = append_address_to_payload(out, &my_addr);
-    if (len < 0)
-      return -1;
-    out += len;
+
+    uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr);
+    netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs);
+
+    netinfo_cell_add_my_addrs(netinfo_cell,
+                              netinfo_addr_from_tor_addr(&my_addr));
 
     if (!tor_addr_is_null(&me->ipv6_addr)) {
-      len = append_address_to_payload(out, &me->ipv6_addr);
-      if (len < 0)
-        return -1;
+      netinfo_cell_add_my_addrs(netinfo_cell,
+                                netinfo_addr_from_tor_addr(&me->ipv6_addr));
     }
-  } else {
-    *out = 0;
+  }
+
+  const char *errmsg = NULL;
+  if ((errmsg = netinfo_cell_check(netinfo_cell))) {
+    log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s",
+                    errmsg);
+    goto cleanup;
+  }
+
+  if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                          netinfo_cell) < 0) {
+    log_warn(LD_OR, "Failed generating NETINFO cell");
+    goto cleanup;
   }
 
   conn->handshake_state->digest_sent_data = 0;
   conn->handshake_state->sent_netinfo = 1;
   connection_or_write_cell_to_buf(&cell, conn);
 
-  return 0;
+  r = 0;
+ cleanup:
+  netinfo_cell_free(netinfo_cell);
+
+  return r;
 }
 
 /** Helper used to add an encoded certs to a cert cell */
diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h
index 817bcdd317..272f536b83 100644
--- a/src/core/or/connection_or.h
+++ b/src/core/or/connection_or.h
@@ -17,32 +17,7 @@ struct ed25519_keypair_t;
 
 or_connection_t *TO_OR_CONN(connection_t *);
 
-#define OR_CONN_STATE_MIN_ 1
-/** State for a connection to an OR: waiting for connect() to finish. */
-#define OR_CONN_STATE_CONNECTING 1
-/** State for a connection to an OR: waiting for proxy handshake to complete */
-#define OR_CONN_STATE_PROXY_HANDSHAKING 2
-/** State for an OR connection client: SSL is handshaking, not done
- * yet. */
-#define OR_CONN_STATE_TLS_HANDSHAKING 3
-/** State for a connection to an OR: We're doing a second SSL handshake for
- * renegotiation purposes. (V2 handshake only.) */
-#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
-/** State for a connection at an OR: We're waiting for the client to
- * renegotiate (to indicate a v2 handshake) or send a versions cell (to
- * indicate a v3 handshake) */
-#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
-/** State for an OR connection: We're done with our SSL handshake, we've done
- * renegotiation, but we haven't yet negotiated link protocol versions and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
-/** State for an OR connection: We're done with our SSL handshake, but we
- * haven't yet negotiated link protocol versions, done a V3 handshake, and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
-/** State for an OR connection: Ready to send/receive cells. */
-#define OR_CONN_STATE_OPEN 8
-#define OR_CONN_STATE_MAX_ 8
+#include "core/or/orconn_event.h"
 
 void connection_or_clear_identity(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
@@ -81,6 +56,9 @@ MOCK_DECL(void,connection_or_close_for_error,
 
 void connection_or_report_broken_states(int severity, int domain);
 
+void connection_or_event_status(or_connection_t *conn,
+                                or_conn_status_event_t tp, int reason);
+
 MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn,
                                               int receiving));
 int connection_tls_continue_handshake(or_connection_t *conn);
diff --git a/src/core/or/connection_st.h b/src/core/or/connection_st.h
index d1430eda14..1c42a56d6b 100644
--- a/src/core/or/connection_st.h
+++ b/src/core/or/connection_st.h
@@ -146,4 +146,4 @@ struct connection_t {
  * directory connection. */
 #define DIR_CONN_IS_SERVER(conn) ((conn)->purpose == DIR_PURPOSE_SERVER)
 
-#endif
+#endif /* !defined(CONNECTION_ST_H) */
diff --git a/src/core/or/cpath_build_state_st.h b/src/core/or/cpath_build_state_st.h
index dbe596d851..4572a10430 100644
--- a/src/core/or/cpath_build_state_st.h
+++ b/src/core/or/cpath_build_state_st.h
@@ -34,5 +34,5 @@ struct cpath_build_state_t {
   time_t expiry_time;
 };
 
-#endif
+#endif /* !defined(CIRCUIT_BUILD_STATE_ST_ST_H) */
 
diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c
new file mode 100644
index 0000000000..9b7efc28bf
--- /dev/null
+++ b/src/core/or/crypt_path.c
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypt_path.c
+ *
+ * \brief Functions dealing with layered circuit encryption. This file aims to
+ *   provide an API around the crypt_path_t structure which holds crypto
+ *   information about a specific hop of a circuit.
+ *
+ * TODO: We should eventually move all functions dealing and manipulating
+ *   crypt_path_t to this file, so that eventually we encapsulate more and more
+ *   of crypt_path_t. Here are some more functions that can be moved here with
+ *   some more effort:
+ *
+ *   - circuit_list_path_impl()
+ *   - Functions dealing with cpaths in HSv2 create_rend_cpath() and
+ *     create_rend_cpath_legacy()
+ *   - The cpath related parts of rend_service_receive_introduction() and
+ *     rend_client_send_introduction().
+ **/
+
+#define CRYPT_PATH_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/crypt_path.h"
+
+#include "core/crypto/relay_crypto.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/cell_st.h"
+
+/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
+ * This function is used to extend cpath by another hop.
+ */
+void
+cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop)
+{
+  if (*head_ptr) {
+    new_hop->next = (*head_ptr);
+    new_hop->prev = (*head_ptr)->prev;
+    (*head_ptr)->prev->next = new_hop;
+    (*head_ptr)->prev = new_hop;
+  } else {
+    *head_ptr = new_hop;
+    new_hop->prev = new_hop->next = new_hop;
+  }
+}
+
+/** Create a new hop, annotate it with information about its
+ * corresponding router <b>choice</b>, and append it to the
+ * end of the cpath <b>head_ptr</b>. */
+int
+cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
+{
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+
+  /* link hop into the cpath, at the end. */
+  cpath_extend_linked_list(head_ptr, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_CLOSED;
+
+  hop->extend_info = extend_info_dup(choice);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  return 0;
+}
+
+/** Verify that cpath <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_ok(const crypt_path_t *cp)
+{
+  const crypt_path_t *start = cp;
+
+  do {
+    cpath_assert_layer_ok(cp);
+    /* layers must be in sequence of: "open* awaiting? closed*" */
+    if (cp != start) {
+      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      } else if (cp->state == CPATH_STATE_OPEN) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      }
+    }
+    cp = cp->next;
+    tor_assert(cp);
+  } while (cp != start);
+}
+
+/** Verify that cpath layer <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_layer_ok(const crypt_path_t *cp)
+{
+//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
+//  tor_assert(cp->port);
+  tor_assert(cp);
+  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
+  switch (cp->state)
+    {
+    case CPATH_STATE_OPEN:
+      relay_crypto_assert_ok(&cp->pvt_crypto);
+      FALLTHROUGH;
+    case CPATH_STATE_CLOSED:
+      /*XXXX Assert that there's no handshake_state either. */
+      tor_assert(!cp->rend_dh_handshake_state);
+      break;
+    case CPATH_STATE_AWAITING_KEYS:
+      /* tor_assert(cp->dh_handshake_state); */
+      break;
+    default:
+      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
+      tor_assert(0);
+    }
+  tor_assert(cp->package_window >= 0);
+  tor_assert(cp->deliver_window >= 0);
+}
+
+/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
+ *   - 20 to initialize f_digest
+ *   - 20 to initialize b_digest
+ *   - 16 to key f_crypto
+ *   - 16 to key b_crypto
+ *
+ * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
+ */
+int
+cpath_init_circuit_crypto(crypt_path_t *cpath,
+                          const char *key_data, size_t key_data_len,
+                          int reverse, int is_hs_v3)
+{
+
+  tor_assert(cpath);
+  return relay_crypto_init(&cpath->pvt_crypto, key_data, key_data_len,
+                           reverse, is_hs_v3);
+}
+
+/** Deallocate space associated with the cpath node <b>victim</b>. */
+void
+cpath_free(crypt_path_t *victim)
+{
+  if (!victim)
+    return;
+
+  relay_crypto_clear(&victim->pvt_crypto);
+  onion_handshake_state_release(&victim->handshake_state);
+  crypto_dh_free(victim->rend_dh_handshake_state);
+  extend_info_free(victim->extend_info);
+
+  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
+  tor_free(victim);
+}
+
+/********************** cpath crypto API *******************************/
+
+/** Encrypt or decrypt <b>payload</b> using the crypto of <b>cpath</b>. Actual
+ *  operation decided by <b>is_decrypt</b>.  */
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt)
+{
+  if (is_decrypt) {
+    relay_crypt_one_payload(cpath->pvt_crypto.b_crypto, payload);
+  } else {
+    relay_crypt_one_payload(cpath->pvt_crypto.f_crypto, payload);
+  }
+}
+
+/** Getter for the incoming digest of <b>cpath</b>. */
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath)
+{
+  return cpath->pvt_crypto.b_digest;
+}
+
+/** Set the right integrity digest on the outgoing <b>cell</b> based on the
+ *  cell payload and update the forward digest of <b>cpath</b>. */
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell)
+{
+  relay_set_digest(cpath->pvt_crypto.f_digest, cell);
+}
+
+/************ cpath sendme API ***************************/
+
+/** Return the sendme_digest of this <b>cpath</b>. */
+uint8_t *
+cpath_get_sendme_digest(crypt_path_t *cpath)
+{
+  return relay_crypto_get_sendme_digest(&cpath->pvt_crypto);
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+cpath_sendme_record_cell_digest(crypt_path_t *cpath, bool is_foward_digest)
+{
+  tor_assert(cpath);
+  relay_crypto_record_sendme_digest(&cpath->pvt_crypto, is_foward_digest);
+}
+
+/************ other cpath functions ***************************/
+
+/** Return the first non-open hop in cpath, or return NULL if all
+ * hops are open. */
+crypt_path_t *
+cpath_get_next_non_open_hop(crypt_path_t *cpath)
+{
+  crypt_path_t *hop = cpath;
+  do {
+    if (hop->state != CPATH_STATE_OPEN)
+      return hop;
+    hop = hop->next;
+  } while (hop != cpath);
+  return NULL;
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/** Unittest helper function: Count number of hops in cpath linked list. */
+unsigned int
+cpath_get_n_hops(crypt_path_t **head_ptr)
+{
+  unsigned int n_hops = 0;
+  crypt_path_t *tmp;
+
+  if (!*head_ptr) {
+    return 0;
+  }
+
+  tmp = *head_ptr;
+  do {
+    n_hops++;
+    tmp = tmp->next;
+  } while (tmp != *head_ptr);
+
+  return n_hops;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h
new file mode 100644
index 0000000000..7a95fec2b4
--- /dev/null
+++ b/src/core/or/crypt_path.h
@@ -0,0 +1,46 @@
+/**
+ * \file crypt_path.h
+ * \brief Header file for crypt_path.c.
+ **/
+
+#ifndef CRYPT_PATH_H
+#define CRYPT_PATH_H
+
+void cpath_assert_layer_ok(const crypt_path_t *cp);
+
+void cpath_assert_ok(const crypt_path_t *cp);
+
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+
+int cpath_init_circuit_crypto(crypt_path_t *cpath,
+                              const char *key_data, size_t key_data_len,
+                              int reverse, int is_hs_v3);
+
+void
+cpath_free(crypt_path_t *victim);
+
+void cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop);
+
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt);
+
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath);
+
+void cpath_sendme_record_cell_digest(crypt_path_t *cpath,
+                                     bool is_foward_digest);
+
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell);
+
+crypt_path_t *cpath_get_next_non_open_hop(crypt_path_t *cpath);
+
+void cpath_sendme_circuit_record_inbound_cell(crypt_path_t *cpath);
+
+uint8_t *cpath_get_sendme_digest(crypt_path_t *cpath);
+
+#if defined(TOR_UNIT_TESTS)
+unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(CRYPT_PATH_H) */
diff --git a/src/core/or/crypt_path_reference_st.h b/src/core/or/crypt_path_reference_st.h
index 3d79f26c1c..1827022b4e 100644
--- a/src/core/or/crypt_path_reference_st.h
+++ b/src/core/or/crypt_path_reference_st.h
@@ -19,5 +19,5 @@ struct crypt_path_reference_t {
   crypt_path_t *cpath;
 };
 
-#endif
+#endif /* !defined(CRYPT_PATH_REFERENCE_ST_H) */
 
diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h
index 429480f8ab..249ac6aaa3 100644
--- a/src/core/or/crypt_path_st.h
+++ b/src/core/or/crypt_path_st.h
@@ -24,15 +24,24 @@ struct onion_handshake_state_t {
   } u;
 };
 
+/** Macro to encapsulate private members of a struct.
+ *
+ *  Renames 'x' to 'x_crypt_path_private_field'.
+ */
+#define CRYPT_PATH_PRIV_FIELD(x) x ## _crypt_path_private_field
+
+#ifdef CRYPT_PATH_PRIVATE
+
+/* Helper macro to access private members of a struct. */
+#define pvt_crypto CRYPT_PATH_PRIV_FIELD(crypto)
+
+#endif /* defined(CRYPT_PATH_PRIVATE) */
+
 /** Holds accounting information for a single step in the layered encryption
  * performed by a circuit.  Used only at the client edge of a circuit. */
 struct crypt_path_t {
   uint32_t magic;
 
-  /** Cryptographic state used for encrypting and authenticating relay
-   * cells to and from this hop. */
-  relay_crypto_t crypto;
-
   /** Current state of the handshake as performed with the OR at this
    * step. */
   onion_handshake_state_t handshake_state;
@@ -65,6 +74,12 @@ struct crypt_path_t {
                        * at this step? */
   int deliver_window; /**< How many cells are we willing to deliver originating
                        * at this step? */
+
+  /*********************** Private members ****************************/
+
+  /** Private member: Cryptographic state used for encrypting and
+   * authenticating relay cells to and from this hop. */
+  relay_crypto_t CRYPT_PATH_PRIV_FIELD(crypto);
 };
 
-#endif
+#endif /* !defined(CRYPT_PATH_ST_H) */
diff --git a/src/core/or/destroy_cell_queue_st.h b/src/core/or/destroy_cell_queue_st.h
index 56630670ba..e917afc700 100644
--- a/src/core/or/destroy_cell_queue_st.h
+++ b/src/core/or/destroy_cell_queue_st.h
@@ -23,5 +23,5 @@ struct destroy_cell_queue_t {
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
+#endif /* !defined(DESTROY_CELL_QUEUE_ST_H) */
 
diff --git a/src/core/or/dos.h b/src/core/or/dos.h
index 058b7afce6..a46f65d767 100644
--- a/src/core/or/dos.h
+++ b/src/core/or/dos.h
@@ -135,7 +135,7 @@ MOCK_DECL(STATIC unsigned int, get_param_cc_enabled,
 MOCK_DECL(STATIC unsigned int, get_param_conn_enabled,
           (const networkstatus_t *ns));
 
-#endif /* TOR_DOS_PRIVATE */
+#endif /* defined(DOS_PRIVATE) */
 
-#endif /* TOR_DOS_H */
+#endif /* !defined(TOR_DOS_H) */
 
diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h
index 1665b8589f..8922a3a9cf 100644
--- a/src/core/or/edge_connection_st.h
+++ b/src/core/or/edge_connection_st.h
@@ -73,5 +73,5 @@ struct edge_connection_t {
   uint64_t dirreq_id;
 };
 
-#endif
+#endif /* !defined(EDGE_CONNECTION_ST_H) */
 
diff --git a/src/core/or/entry_connection_st.h b/src/core/or/entry_connection_st.h
index 45621fadbf..e65c545d17 100644
--- a/src/core/or/entry_connection_st.h
+++ b/src/core/or/entry_connection_st.h
@@ -96,5 +96,5 @@ struct entry_connection_t {
 /** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
 #define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
 
-#endif
+#endif /* !defined(ENTRY_CONNECTION_ST_H) */
 
diff --git a/src/core/or/entry_port_cfg_st.h b/src/core/or/entry_port_cfg_st.h
index 87dfb331e5..b84838d44f 100644
--- a/src/core/or/entry_port_cfg_st.h
+++ b/src/core/or/entry_port_cfg_st.h
@@ -50,5 +50,5 @@ struct entry_port_cfg_t {
 
 };
 
-#endif
+#endif /* !defined(ENTRY_PORT_CFG_ST_H) */
 
diff --git a/src/core/or/extend_info_st.h b/src/core/or/extend_info_st.h
index bc7a77b1b2..7704ff16b5 100644
--- a/src/core/or/extend_info_st.h
+++ b/src/core/or/extend_info_st.h
@@ -27,4 +27,4 @@ struct extend_info_t {
   curve25519_public_key_t curve25519_onion_key;
 };
 
-#endif
+#endif /* !defined(EXTEND_INFO_ST_H) */
diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h
index d4617be108..1fe47ad3f1 100644
--- a/src/core/or/half_edge_st.h
+++ b/src/core/or/half_edge_st.h
@@ -30,5 +30,5 @@ typedef struct half_edge_t {
   int connected_pending : 1;
 } half_edge_t;
 
-#endif
+#endif /* !defined(HALF_EDGE_ST_H) */
 
diff --git a/src/core/or/listener_connection_st.h b/src/core/or/listener_connection_st.h
index 8989a39dc8..1250d9c9b4 100644
--- a/src/core/or/listener_connection_st.h
+++ b/src/core/or/listener_connection_st.h
@@ -21,5 +21,5 @@ struct listener_connection_t {
 
 };
 
-#endif
+#endif /* !defined(LISTENER_CONNECTION_ST_H) */
 
diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c
new file mode 100644
index 0000000000..3cb9147134
--- /dev/null
+++ b/src/core/or/ocirc_event.c
@@ -0,0 +1,128 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.c
+ * \brief Publish state change messages for origin circuits
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of origin circuits.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/ocirc_event_sys.h"
+#include "core/or/origin_circuit_st.h"
+#include "lib/subsys/subsys.h"
+
+DECLARE_PUBLISH(ocirc_state);
+DECLARE_PUBLISH(ocirc_chan);
+DECLARE_PUBLISH(ocirc_cevent);
+
+static void
+ocirc_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+ocirc_state_fmt(msg_aux_data_t u)
+{
+  ocirc_state_msg_t *msg = (ocirc_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" state=%d onehop=%d>",
+               msg->gid, msg->state, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_chan_fmt(msg_aux_data_t u)
+{
+  ocirc_chan_msg_t *msg = (ocirc_chan_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" chan=%"PRIu64" onehop=%d>",
+               msg->gid, msg->chan, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_cevent_fmt(msg_aux_data_t u)
+{
+  ocirc_cevent_msg_t *msg = (ocirc_cevent_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" evtype=%d reason=%d onehop=%d>",
+               msg->gid, msg->evtype, msg->reason, msg->onehop);
+  return s;
+}
+
+static dispatch_typefns_t ocirc_state_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_state_fmt,
+};
+
+static dispatch_typefns_t ocirc_chan_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_chan_fmt,
+};
+
+static dispatch_typefns_t ocirc_cevent_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_cevent_fmt,
+};
+
+static int
+ocirc_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_state, &ocirc_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_chan, &ocirc_chan_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_cevent, &ocirc_cevent_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_state))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  return 0;
+}
+
+void
+ocirc_state_publish(ocirc_state_msg_t *msg)
+{
+  PUBLISH(ocirc_state, msg);
+}
+
+void
+ocirc_chan_publish(ocirc_chan_msg_t *msg)
+{
+  PUBLISH(ocirc_chan, msg);
+}
+
+void
+ocirc_cevent_publish(ocirc_cevent_msg_t *msg)
+{
+  PUBLISH(ocirc_cevent, msg);
+}
+
+const subsys_fns_t sys_ocirc_event = {
+  .name = "ocirc_event",
+  .supported = true,
+  .level = -32,
+  .add_pubsub = ocirc_add_pubsub,
+};
diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h
new file mode 100644
index 0000000000..8e9494874f
--- /dev/null
+++ b/src/core/or/ocirc_event.h
@@ -0,0 +1,72 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.h
+ * \brief Header file for ocirc_event.c
+ **/
+
+#ifndef TOR_OCIRC_EVENT_H
+#define TOR_OCIRC_EVENT_H
+
+#include <stdbool.h>
+
+#include "lib/cc/torint.h"
+#include "lib/pubsub/pubsub.h"
+
+/** Used to indicate the type of a circuit event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum circuit_status_event_t {
+  CIRC_EVENT_LAUNCHED = 0,
+  CIRC_EVENT_BUILT    = 1,
+  CIRC_EVENT_EXTENDED = 2,
+  CIRC_EVENT_FAILED   = 3,
+  CIRC_EVENT_CLOSED   = 4,
+} circuit_status_event_t;
+
+/** Message for origin circuit state update */
+typedef struct ocirc_state_msg_t {
+  uint32_t gid;       /**< global ID (only origin circuits have them) */
+  int state;          /**< new circuit state */
+  bool onehop;        /**< one-hop circuit? */
+} ocirc_state_msg_t;
+
+DECLARE_MESSAGE(ocirc_state, ocirc_state, ocirc_state_msg_t *);
+
+/**
+ * Message when a channel gets associated to a circuit.
+ *
+ * This doesn't always correspond to something in circuitbuild.c
+ * setting the n_chan field in the circuit.  For some reason, if
+ * circuit_handle_first_hop() launches a new circuit, it doesn't set
+ * the n_chan field.
+ */
+typedef struct ocirc_chan_msg_t {
+  uint32_t gid;                 /**< global ID */
+  uint64_t chan;                /**< channel ID */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_chan_msg_t;
+
+DECLARE_MESSAGE(ocirc_chan, ocirc_chan, ocirc_chan_msg_t *);
+
+/**
+ * Message for origin circuit status event
+ *
+ * This contains information that ends up in CIRC control protocol events.
+ */
+typedef struct ocirc_cevent_msg_t {
+  uint32_t gid;                 /**< global ID */
+  int evtype;                   /**< event type */
+  int reason;                   /**< reason */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_cevent_msg_t;
+
+DECLARE_MESSAGE(ocirc_cevent, ocirc_cevent, ocirc_cevent_msg_t *);
+
+#ifdef OCIRC_EVENT_PRIVATE
+void ocirc_state_publish(ocirc_state_msg_t *msg);
+void ocirc_chan_publish(ocirc_chan_msg_t *msg);
+void ocirc_cevent_publish(ocirc_cevent_msg_t *msg);
+#endif
+
+#endif /* !defined(TOR_OCIRC_EVENT_H) */
diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h
new file mode 100644
index 0000000000..61180496da
--- /dev/null
+++ b/src/core/or/ocirc_event_sys.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+
+/**
+ * \file ocirc_event_sys.h
+ * \brief Declare subsystem object for the origin circuit event module.
+ **/
+
+#ifndef TOR_OCIRC_EVENT_SYS_H
+#define TOR_OCIRC_EVENT_SYS_H
+
+extern const struct subsys_fns_t sys_ocirc_event;
+
+#endif /* !defined(TOR_OCIRC_EVENT_SYS_H) */
diff --git a/src/core/or/or.h b/src/core/or/or.h
index 7c601e49b3..990cfacbc0 100644
--- a/src/core/or/or.h
+++ b/src/core/or/or.h
@@ -26,7 +26,7 @@
 #include "lib/cc/compat_compiler.h"
 #include "lib/cc/torint.h"
 #include "lib/container/map.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/container/smartlist.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rsa.h"
@@ -97,6 +97,8 @@ struct curve25519_public_key_t;
 #define SIGNEWNYM 129
 #define SIGCLEARDNSCACHE 130
 #define SIGHEARTBEAT 131
+#define SIGACTIVE 132
+#define SIGDORMANT 133
 
 #if (SIZEOF_CELL_T != 0)
 /* On Irix, stdlib.h defines a cell_t type, so we need to make sure
@@ -205,6 +207,9 @@ struct curve25519_public_key_t;
 #define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39
 #define RELAY_COMMAND_INTRODUCE_ACK 40
 
+#define RELAY_COMMAND_PADDING_NEGOTIATE 41
+#define RELAY_COMMAND_PADDING_NEGOTIATED 42
+
 /* Reasons why an OR connection is closed. */
 #define END_OR_CONN_REASON_DONE           1
 #define END_OR_CONN_REASON_REFUSED        2 /* connection refused */
@@ -834,6 +839,14 @@ typedef struct protover_summary_flags_t {
    * service rendezvous point supporting version 3 as seen in proposal 224.
    * This requires HSRend=2. */
   unsigned int supports_v3_rendezvous_point: 1;
+
+  /** True iff this router has a protocol list that allows clients to
+   * negotiate hs circuit setup padding. Requires Padding>=2. */
+  unsigned int supports_hs_setup_padding : 1;
+
+  /** True iff this router has a protocol list that allows it to support the
+   * ESTABLISH_INTRO DoS cell extension. Requires HSIntro>=5. */
+  unsigned int supports_establish_intro_dos_extension : 1;
 } protover_summary_flags_t;
 
 typedef struct routerinfo_t routerinfo_t;
diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h
index 6b6feb9d89..f3eb861613 100644
--- a/src/core/or/or_circuit_st.h
+++ b/src/core/or/or_circuit_st.h
@@ -12,6 +12,8 @@
 #include "core/or/circuit_st.h"
 #include "core/or/crypt_path_st.h"
 
+#include "lib/evloop/token_bucket.h"
+
 struct onion_queue_t;
 
 /** An or_circuit_t holds information needed to implement a circuit at an
@@ -33,11 +35,6 @@ struct or_circuit_t {
   cell_queue_t p_chan_cells;
   /** The channel that is previous in this circuit. */
   channel_t *p_chan;
-  /**
-   * Circuit mux associated with p_chan to which this circuit is attached;
-   * NULL if we have no p_chan.
-   */
-  circuitmux_t *p_mux;
   /** Linked list of Exit streams associated with this circuit. */
   edge_connection_t *n_streams;
   /** Linked list of Exit streams associated with this circuit that are
@@ -74,7 +71,16 @@ struct or_circuit_t {
    * exit-ward queues of this circuit; reset every time when writing
    * buffer stats to disk. */
   uint64_t total_cell_waiting_time;
+
+  /** If set, the DoS defenses are enabled on this circuit meaning that the
+   * introduce2_bucket is initialized and used. */
+  unsigned int introduce2_dos_defense_enabled : 1;
+
+  /** INTRODUCE2 cell bucket controlling how much can go on this circuit. Only
+   * used if this is a service introduction circuit at the intro point
+   * (purpose = CIRCUIT_PURPOSE_INTRO_POINT). */
+  token_bucket_ctr_t introduce2_bucket;
 };
 
-#endif
+#endif /* !defined(OR_CIRCUIT_ST_H) */
 
diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h
index d5db5e8694..051fcd00d3 100644
--- a/src/core/or/or_connection_st.h
+++ b/src/core/or/or_connection_st.h
@@ -67,6 +67,8 @@ struct or_connection_t {
    * geoip cache and handled by the DoS mitigation subsystem. We use this to
    * insure we have a coherent count of concurrent connection. */
   unsigned int tracked_for_dos_mitigation : 1;
+  /** True iff this connection is using a pluggable transport */
+  unsigned int is_pt : 1;
 
   uint16_t link_proto; /**< What protocol version are we using? 0 for
                         * "none negotiated yet." */
@@ -89,4 +91,4 @@ struct or_connection_t {
   uint64_t bytes_xmitted, bytes_xmitted_by_tls;
 };
 
-#endif
+#endif /* !defined(OR_CONNECTION_ST_H) */
diff --git a/src/core/or/or_handshake_certs_st.h b/src/core/or/or_handshake_certs_st.h
index a93b7104aa..9deb6d6d59 100644
--- a/src/core/or/or_handshake_certs_st.h
+++ b/src/core/or/or_handshake_certs_st.h
@@ -37,4 +37,4 @@ struct or_handshake_certs_t {
   size_t ed_rsa_crosscert_len;
 };
 
-#endif
+#endif /* !defined(OR_HANDSHAKE_CERTS_ST) */
diff --git a/src/core/or/or_handshake_state_st.h b/src/core/or/or_handshake_state_st.h
index 09a8a34179..472ce8a302 100644
--- a/src/core/or/or_handshake_state_st.h
+++ b/src/core/or/or_handshake_state_st.h
@@ -74,5 +74,5 @@ struct or_handshake_state_t {
   or_handshake_certs_t *certs;
 };
 
-#endif
+#endif /* !defined(OR_HANDSHAKE_STATE_ST) */
 
diff --git a/src/core/or/or_periodic.c b/src/core/or/or_periodic.c
new file mode 100644
index 0000000000..fe28c99192
--- /dev/null
+++ b/src/core/or/or_periodic.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.c
+ * @brief Periodic callbacks for the onion routing subsystem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+
+#include "core/or/channel.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_periodic.h"
+
+#include "feature/relay/routermode.h"
+
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+
+#define CHANNEL_CHECK_INTERVAL (60*60)
+static int
+check_canonical_channels_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (public_server_mode(options))
+    channel_check_for_duplicates();
+
+  return CHANNEL_CHECK_INTERVAL;
+}
+
+DECLARE_EVENT(check_canonical_channels, RELAY, FL(NEED_NET));
+
+/**
+ * Periodic callback: as a server, see if we have any old unused circuits
+ * that should be expired */
+static int
+expire_old_circuits_serverside_callback(time_t now,
+                                        const or_options_t *options)
+{
+  (void)options;
+  /* every 11 seconds, so not usually the same second as other such events */
+  circuit_expire_old_circuits_serverside(now);
+  return 11;
+}
+
+DECLARE_EVENT(expire_old_circuits_serverside, ROUTER, FL(NEED_NET));
+
+void
+or_register_periodic_events(void)
+{
+  // These are router-only events, but they're owned by the OR subsystem. */
+  periodic_events_register(&check_canonical_channels_event);
+  periodic_events_register(&expire_old_circuits_serverside_event);
+}
diff --git a/src/core/or/or_periodic.h b/src/core/or/or_periodic.h
new file mode 100644
index 0000000000..c2f47cf5ef
--- /dev/null
+++ b/src/core/or/or_periodic.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.h
+ * @brief Header for core/or/or_periodic.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_PERIODIC_H
+#define TOR_CORE_OR_OR_PERIODIC_H
+
+void or_register_periodic_events(void);
+
+#endif /* !defined(TOR_CORE_OR_OR_PERIODIC_H) */
diff --git a/src/core/or/or_sys.c b/src/core/or/or_sys.c
new file mode 100644
index 0000000000..6f8c81a4df
--- /dev/null
+++ b/src/core/or/or_sys.c
@@ -0,0 +1,43 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.c
+ * @brief Subsystem definitions for OR module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/or/or_periodic.h"
+#include "core/or/or_sys.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/versions.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_or_initialize(void)
+{
+  or_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_or_shutdown(void)
+{
+  protover_free_all();
+  protover_summary_cache_free_all();
+  policies_free_all();
+}
+
+const struct subsys_fns_t sys_or = {
+  .name = "or",
+  .supported = true,
+  .level = 20,
+  .initialize = subsys_or_initialize,
+  .shutdown = subsys_or_shutdown,
+};
diff --git a/src/core/or/or_sys.h b/src/core/or/or_sys.h
new file mode 100644
index 0000000000..c37ef01858
--- /dev/null
+++ b/src/core/or/or_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.h
+ * @brief Header for core/or/or_sys.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_SYS_H
+#define TOR_CORE_OR_OR_SYS_H
+
+extern const struct subsys_fns_t sys_or;
+
+#endif /* !defined(TOR_CORE_OR_OR_SYS_H) */
diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c
new file mode 100644
index 0000000000..86f112fc09
--- /dev/null
+++ b/src/core/or/orconn_event.c
@@ -0,0 +1,99 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.c
+ * \brief Publish state change messages for OR connections
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of OR connections.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/orconn_event.h"
+#include "core/or/orconn_event_sys.h"
+
+DECLARE_PUBLISH(orconn_state);
+DECLARE_PUBLISH(orconn_status);
+
+static void
+orconn_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+orconn_state_fmt(msg_aux_data_t u)
+{
+  orconn_state_msg_t *msg = (orconn_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" chan=%"PRIu64" proxy_type=%d state=%d>",
+               msg->gid, msg->chan, msg->proxy_type, msg->state);
+  return s;
+}
+
+static char *
+orconn_status_fmt(msg_aux_data_t u)
+{
+  orconn_status_msg_t *msg = (orconn_status_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" status=%d reason=%d>",
+               msg->gid, msg->status, msg->reason);
+  return s;
+}
+
+static dispatch_typefns_t orconn_state_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_state_fmt,
+};
+
+static dispatch_typefns_t orconn_status_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_status_fmt,
+};
+
+static int
+orconn_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_state, &orconn_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_status, &orconn_status_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_state) != 0)
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_status) != 0)
+    return -1;
+  return 0;
+}
+
+void
+orconn_state_publish(orconn_state_msg_t *msg)
+{
+  PUBLISH(orconn_state, msg);
+}
+
+void
+orconn_status_publish(orconn_status_msg_t *msg)
+{
+  PUBLISH(orconn_status, msg);
+}
+
+const subsys_fns_t sys_orconn_event = {
+  .name = "orconn_event",
+  .supported = true,
+  .level = -33,
+  .add_pubsub = orconn_add_pubsub,
+};
diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h
new file mode 100644
index 0000000000..fb67a7d183
--- /dev/null
+++ b/src/core/or/orconn_event.h
@@ -0,0 +1,103 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.h
+ * \brief Header file for orconn_event.c
+ *
+ * The OR_CONN_STATE_* symbols are here to make it easier for
+ * subscribers to make decisions based on the messages that they
+ * receive.
+ **/
+
+#ifndef TOR_ORCONN_EVENT_H
+#define TOR_ORCONN_EVENT_H
+
+#include "lib/pubsub/pubsub.h"
+
+/**
+ * @name States of OR connections
+ *
+ * These must be in a partial ordering such that usually no OR
+ * connection will transition from a higher-numbered state to a
+ * lower-numbered one.  Code such as bto_update_best() depends on this
+ * ordering to determine the best state it's seen so far.
+ * @{ */
+#define OR_CONN_STATE_MIN_ 1
+/** State for a connection to an OR: waiting for connect() to finish. */
+#define OR_CONN_STATE_CONNECTING 1
+/** State for a connection to an OR: waiting for proxy handshake to complete */
+#define OR_CONN_STATE_PROXY_HANDSHAKING 2
+/** State for an OR connection client: SSL is handshaking, not done
+ * yet. */
+#define OR_CONN_STATE_TLS_HANDSHAKING 3
+/** State for a connection to an OR: We're doing a second SSL handshake for
+ * renegotiation purposes. (V2 handshake only.) */
+#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
+/** State for a connection at an OR: We're waiting for the client to
+ * renegotiate (to indicate a v2 handshake) or send a versions cell (to
+ * indicate a v3 handshake) */
+#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
+/** State for an OR connection: We're done with our SSL handshake, we've done
+ * renegotiation, but we haven't yet negotiated link protocol versions and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
+/** State for an OR connection: We're done with our SSL handshake, but we
+ * haven't yet negotiated link protocol versions, done a V3 handshake, and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
+/** State for an OR connection: Ready to send/receive cells. */
+#define OR_CONN_STATE_OPEN 8
+#define OR_CONN_STATE_MAX_ 8
+/** @} */
+
+/** Used to indicate the type of an OR connection event passed to the
+ * controller.  The various types are defined in control-spec.txt */
+typedef enum or_conn_status_event_t {
+  OR_CONN_EVENT_LAUNCHED     = 0,
+  OR_CONN_EVENT_CONNECTED    = 1,
+  OR_CONN_EVENT_FAILED       = 2,
+  OR_CONN_EVENT_CLOSED       = 3,
+  OR_CONN_EVENT_NEW          = 4,
+} or_conn_status_event_t;
+
+/**
+ * Message for orconn state update
+ *
+ * This contains information about internal state changes of
+ * or_connection_t objects.  The chan and proxy_type fields are
+ * additional information that a subscriber may need to make
+ * decisions.
+ **/
+typedef struct orconn_state_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  uint64_t chan;                /**< associated channel ID */
+  int proxy_type;               /**< connection's proxy type */
+  uint8_t state;                /**< new connection state */
+} orconn_state_msg_t;
+
+DECLARE_MESSAGE(orconn_state, orconn_state, orconn_state_msg_t *);
+
+/**
+ * Message for orconn status event
+ *
+ * This contains information that ends up in ORCONN control protocol
+ * events.
+ **/
+typedef struct orconn_status_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  int status;                   /**< or_conn_status_event_t */
+  int reason;                   /**< reason */
+} orconn_status_msg_t;
+
+DECLARE_MESSAGE(orconn_status, orconn_status, orconn_status_msg_t *);
+
+#ifdef ORCONN_EVENT_PRIVATE
+void orconn_state_publish(orconn_state_msg_t *);
+void orconn_status_publish(orconn_status_msg_t *);
+#endif
+
+#endif /* !defined(TOR_ORCONN_EVENT_H) */
diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h
new file mode 100644
index 0000000000..9703b2e3d1
--- /dev/null
+++ b/src/core/or/orconn_event_sys.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+
+/**
+ * \file orconn_event_sys.h
+ * \brief Declare subsystem object for the OR connection event module.
+ **/
+#ifndef TOR_ORCONN_EVENT_SYS_H
+#define TOR_ORCONN_EVENT_SYS_H
+
+extern const struct subsys_fns_t sys_orconn_event;
+
+#endif /* !defined(TOR_ORCONN_EVENT_SYS_H) */
diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h
index f55416db14..01bbc84ae2 100644
--- a/src/core/or/origin_circuit_st.h
+++ b/src/core/or/origin_circuit_st.h
@@ -161,6 +161,10 @@ struct origin_circuit_t {
    * connections to this circuit. */
   unsigned int unusable_for_new_conns : 1;
 
+  /* If this flag is set (due to padding negotiation failure), we should
+   * not try to negotiate further circuit padding. */
+  unsigned padding_negotiation_failed : 1;
+
   /**
    * Tristate variable to guard against pathbias miscounting
    * due to circuit purpose transitions changing the decision
@@ -291,4 +295,4 @@ struct origin_circuit_t {
 
 };
 
-#endif
+#endif /* !defined(ORIGIN_CIRCUIT_ST_H) */
diff --git a/src/core/or/policies.c b/src/core/or/policies.c
index 3ed282d785..39a0817c85 100644
--- a/src/core/or/policies.c
+++ b/src/core/or/policies.c
@@ -29,7 +29,9 @@
 #include "feature/relay/routermode.h"
 #include "lib/geoip/geoip.h"
 #include "ht.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/encoding/confline.h"
+#include "trunnel/ed25519_cert.h"
 
 #include "core/or/addr_policy_st.h"
 #include "feature/dirclient/dir_server_st.h"
@@ -461,7 +463,8 @@ fascist_firewall_use_ipv6(const or_options_t *options)
    * ClientPreferIPv6DirPort is deprecated, but check it anyway. */
   return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 ||
           options->ClientPreferIPv6ORPort == 1 ||
-          options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1);
+          options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1 ||
+          options->ClientAutoIPv6ORPort == 1);
 }
 
 /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and
@@ -488,6 +491,15 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options)
   return -1;
 }
 
+/* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address
+ * family. Return 0 for IPv4, and 1 for IPv6. */
+MOCK_IMPL(int,
+fascist_firewall_rand_prefer_ipv6_addr, (void))
+{
+  /* TODO: Check for failures, and infer our preference based on this. */
+  return crypto_rand_int(2);
+}
+
 /** Do we prefer to connect to IPv6 ORPorts?
  * Use node_ipv6_or_preferred() whenever possible: it supports bridge client
  * per-node IPv6 preferences.
@@ -502,7 +514,10 @@ fascist_firewall_prefer_ipv6_orport(const or_options_t *options)
   }
 
   /* We can use both IPv4 and IPv6 - which do we prefer? */
-  if (options->ClientPreferIPv6ORPort == 1) {
+  if (options->ClientAutoIPv6ORPort == 1) {
+    /* If ClientAutoIPv6ORPort is 1, we prefer IPv4 or IPv6 at random. */
+    return fascist_firewall_rand_prefer_ipv6_addr();
+  } else if (options->ClientPreferIPv6ORPort == 1) {
     return 1;
   }
 
@@ -1001,6 +1016,83 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs,
   }
 }
 
+/** Like fascist_firewall_choose_address_base(), but takes in a smartlist
+ * <b>lspecs</b> consisting of one or more link specifiers. We assume
+ * fw_connection is FIREWALL_OR_CONNECTION as link specifiers cannot
+ * contain DirPorts.
+ */
+void
+fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                   int pref_only, tor_addr_port_t* ap)
+{
+  int have_v4 = 0, have_v6 = 0;
+  uint16_t port_v4 = 0, port_v6 = 0;
+  tor_addr_t addr_v4, addr_v6;
+
+  tor_assert(ap);
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Unknown or missing link specifiers");
+    return;
+  }
+  if (smartlist_len(lspecs) == 0) {
+    log_warn(LD_PROTOCOL, "Link specifiers are empty");
+    return;
+  }
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
+  tor_addr_make_null(&addr_v4, AF_INET);
+  tor_addr_make_null(&addr_v6, AF_INET6);
+
+  SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
+    switch (link_specifier_get_ls_type(ls)) {
+    case LS_IPV4:
+      /* Skip if we already seen a v4. */
+      if (have_v4) continue;
+      tor_addr_from_ipv4h(&addr_v4,
+                          link_specifier_get_un_ipv4_addr(ls));
+      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      have_v4 = 1;
+      break;
+    case LS_IPV6:
+      /* Skip if we already seen a v6, or deliberately skip it if we're not a
+       * direct connection. */
+      if (have_v6) continue;
+      tor_addr_from_ipv6_bytes(&addr_v6,
+          (const char *) link_specifier_getconstarray_un_ipv6_addr(ls));
+      port_v6 = link_specifier_get_un_ipv6_port(ls);
+      have_v6 = 1;
+      break;
+    default:
+      /* Ignore unknown. */
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* If we don't have IPv4 or IPv6 in link specifiers, log a bug and return. */
+  if (!have_v4 && !have_v6) {
+    if (!have_v6) {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4 or IPv6");
+    } else {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4");
+    }
+    return;
+  }
+
+  /* Here, don't check for DirPorts as link specifiers are only used for
+   * ORPorts. */
+  const or_options_t *options = get_options();
+  int pref_ipv6 = fascist_firewall_prefer_ipv6_orport(options);
+  /* Assume that the DirPorts are zero as link specifiers only use ORPorts. */
+  fascist_firewall_choose_address_base(&addr_v4, port_v4, 0,
+                                       &addr_v6, port_v6, 0,
+                                       FIREWALL_OR_CONNECTION,
+                                       pref_only, pref_ipv6,
+                                       ap);
+}
+
 /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and
  * looks up the node's IPv6 preference rather than taking an argument
  * for pref_ipv6. */
@@ -1150,6 +1242,15 @@ authdir_policy_badexit_address(uint32_t addr, uint16_t port)
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END
 
+/** Check <b>or_options</b> to determine whether or not we are using the
+ * default options for exit policy. Return true if so, false otherwise. */
+static int
+policy_using_default_exit_options(const or_options_t *or_options)
+{
+  return (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 &&
+          or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0);
+}
+
 /** Config helper: If there's any problem with the policy configuration
  * options in <b>options</b>, return -1 and set <b>msg</b> to a newly
  * allocated description of the error. Else return 0. */
@@ -1168,9 +1269,8 @@ validate_addr_policies(const or_options_t *options, char **msg)
 
   static int warned_about_nonexit = 0;
 
-  if (public_server_mode(options) &&
-      !warned_about_nonexit && options->ExitPolicy == NULL &&
-      options->ExitRelay == -1 && options->ReducedExitPolicy == 0) {
+  if (public_server_mode(options) && !warned_about_nonexit &&
+      policy_using_default_exit_options(options)) {
     warned_about_nonexit = 1;
     log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. "
                "If you want to be an exit relay, "
@@ -2127,9 +2227,9 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options,
   int rv = 0;
 
   /* Short-circuit for non-exit relays, or for relays where we didn't specify
-   * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */
-  if (or_options->ExitRelay == 0 || (or_options->ExitPolicy == NULL &&
-      or_options->ExitRelay == -1 && or_options->ReducedExitPolicy == 0)) {
+   * ExitPolicy or ReducedExitPolicy or IPv6Exit and ExitRelay is auto. */
+  if (or_options->ExitRelay == 0 ||
+      policy_using_default_exit_options(or_options)) {
     append_exit_policy_string(result, "reject *4:*");
     append_exit_policy_string(result, "reject *6:*");
     return 0;
@@ -2706,7 +2806,7 @@ parse_short_policy(const char *summary)
   int is_accept;
   int n_entries;
   short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */
-  const char *next;
+  char *next;
 
   if (!strcmpstart(summary, "accept ")) {
     is_accept = 1;
@@ -2721,57 +2821,56 @@ parse_short_policy(const char *summary)
 
   n_entries = 0;
   for ( ; *summary; summary = next) {
-    const char *comma = strchr(summary, ',');
-    unsigned low, high;
-    char dummy;
-    char ent_buf[32];
-    size_t len;
-
-    next = comma ? comma+1 : strchr(summary, '\0');
-    len = comma ? (size_t)(comma - summary) : strlen(summary);
-
     if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) {
       log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s",
              escaped(orig_summary));
       return NULL;
     }
 
-    if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) {
-      /* unrecognized entry format. skip it. */
-      continue;
-    }
-    if (len < 1) {
-      /* empty; skip it. */
-      /* XXX This happens to be unreachable, since if len==0, then *summary is
-       * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */
-      continue;
+    unsigned low, high;
+    int ok;
+    low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next);
+    if (!ok) {
+      if (! TOR_ISDIGIT(*summary) || *summary == ',') {
+        /* Unrecognized format: skip it. */
+        goto skip_ent;
+      } else {
+        goto bad_ent;
+      }
     }
 
-    memcpy(ent_buf, summary, len);
-    ent_buf[len] = '\0';
+    switch (*next) {
+      case ',':
+        ++next;
+        FALLTHROUGH;
+      case '\0':
+        high = low;
+        break;
+      case '-':
+        high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next);
+        if (!ok)
+          goto bad_ent;
 
-    if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) {
-      if (low<1 || low>65535 || high<1 || high>65535 || low>high) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-    } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) {
-      if (low<1 || low>65535) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-      high = low;
-    } else {
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
-             escaped(orig_summary));
-      return NULL;
+        if (*next == ',')
+          ++next;
+        else if (*next != '\0')
+          goto bad_ent;
+
+        break;
+      default:
+        goto bad_ent;
     }
 
     entries[n_entries].min_port = low;
     entries[n_entries].max_port = high;
     n_entries++;
+
+    continue;
+  skip_ent:
+    next = strchr(next, ',');
+    if (!next)
+      break;
+    ++next;
   }
 
   if (n_entries == 0) {
@@ -2792,6 +2891,11 @@ parse_short_policy(const char *summary)
   result->n_entries = n_entries;
   memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries);
   return result;
+
+ bad_ent:
+  log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
+         escaped(orig_summary));
+  return NULL;
 }
 
 /** Write <b>policy</b> back out into a string. */
diff --git a/src/core/or/policies.h b/src/core/or/policies.h
index 2c38de362f..3c46363c04 100644
--- a/src/core/or/policies.h
+++ b/src/core/or/policies.h
@@ -70,6 +70,7 @@ typedef struct short_policy_t {
 int firewall_is_fascist_or(void);
 int firewall_is_fascist_dir(void);
 int fascist_firewall_use_ipv6(const or_options_t *options);
+MOCK_DECL(int, fascist_firewall_rand_prefer_ipv6_addr, (void));
 int fascist_firewall_prefer_ipv6_orport(const or_options_t *options);
 int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options);
 
@@ -91,6 +92,8 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds,
 void fascist_firewall_choose_address_rs(const routerstatus_t *rs,
                                         firewall_connection_t fw_connection,
                                         int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                        int pref_only, tor_addr_port_t* ap);
 void fascist_firewall_choose_address_node(const node_t *node,
                                           firewall_connection_t fw_connection,
                                           int pref_only, tor_addr_port_t* ap);
diff --git a/src/core/or/port_cfg_st.h b/src/core/or/port_cfg_st.h
index b67091ce32..e9e82bb1de 100644
--- a/src/core/or/port_cfg_st.h
+++ b/src/core/or/port_cfg_st.h
@@ -31,5 +31,5 @@ struct port_cfg_t {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
+#endif /* !defined(PORT_CFG_ST_H) */
 
diff --git a/src/core/or/protover.c b/src/core/or/protover.c
index 17979d04ea..905c5e9ed3 100644
--- a/src/core/or/protover.c
+++ b/src/core/or/protover.c
@@ -39,6 +39,9 @@ static int protocol_list_contains(const smartlist_t *protos,
 static const struct {
   protocol_type_t protover_type;
   const char *name;
+/* If you add a new protocol here, you probably also want to add
+ * parsing for it in routerstatus_parse_entry_from_string() so that
+ * it is set in routerstatus_t */
 } PROTOCOL_NAMES[] = {
   { PRT_LINK, "Link" },
   { PRT_LINKAUTH, "LinkAuth" },
@@ -49,7 +52,9 @@ static const struct {
   { PRT_HSREND, "HSRend" },
   { PRT_DESC, "Desc" },
   { PRT_MICRODESC, "Microdesc"},
-  { PRT_CONS, "Cons" }
+  { PRT_PADDING, "Padding"},
+  { PRT_CONS, "Cons" },
+  { PRT_FLOWCTRL, "FlowCtrl"},
 };
 
 #define N_PROTOCOL_NAMES ARRAY_LENGTH(PROTOCOL_NAMES)
@@ -387,7 +392,7 @@ protover_get_supported_protocols(void)
     "Desc=1-2 "
     "DirCache=1-2 "
     "HSDir=1-2 "
-    "HSIntro=3-4 "
+    "HSIntro=3-5 "
     "HSRend=1-2 "
     "Link=1-5 "
 #ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
@@ -396,7 +401,9 @@ protover_get_supported_protocols(void)
     "LinkAuth=3 "
 #endif
     "Microdesc=1-2 "
-    "Relay=1-2";
+    "Relay=1-2 "
+    "Padding=2 "
+    "FlowCtrl=1";
 }
 
 /** The protocols from protover_get_supported_protocols(), as parsed into a
@@ -815,6 +822,8 @@ protover_all_supported(const char *s, char **missing_out)
        * ones and, if so, add them to unsupported->ranges. */
       if (versions->low != 0 && versions->high != 0) {
         smartlist_add(unsupported->ranges, versions);
+      } else {
+        tor_free(versions);
       }
       /* Finally, if we had something unsupported, add it to the list of
        * missing_some things and mark that there was something missing. */
@@ -823,7 +832,6 @@ protover_all_supported(const char *s, char **missing_out)
         all_supported = 0;
       } else {
         proto_entry_free(unsupported);
-        tor_free(versions);
       }
     } SMARTLIST_FOREACH_END(range);
 
diff --git a/src/core/or/protover.h b/src/core/or/protover.h
index 7e181ba97a..af45a31aeb 100644
--- a/src/core/or/protover.h
+++ b/src/core/or/protover.h
@@ -28,21 +28,25 @@ struct smartlist_t;
 #define PROTOVER_HS_INTRO_V3 4
 /** The protover version number that signifies HSv3 rendezvous point support */
 #define PROTOVER_HS_RENDEZVOUS_POINT_V3 2
+/** The protover that signals support for HS circuit setup padding machines */
+#define PROTOVER_HS_SETUP_PADDING 2
 
 /** List of recognized subprotocols. */
 /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust`
 /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto`
 typedef enum protocol_type_t {
-  PRT_LINK,
-  PRT_LINKAUTH,
-  PRT_RELAY,
-  PRT_DIRCACHE,
-  PRT_HSDIR,
-  PRT_HSINTRO,
-  PRT_HSREND,
-  PRT_DESC,
-  PRT_MICRODESC,
-  PRT_CONS,
+  PRT_LINK      = 0,
+  PRT_LINKAUTH  = 1,
+  PRT_RELAY     = 2,
+  PRT_DIRCACHE  = 3,
+  PRT_HSDIR     = 4,
+  PRT_HSINTRO   = 5,
+  PRT_HSREND    = 6,
+  PRT_DESC      = 7,
+  PRT_MICRODESC = 8,
+  PRT_CONS      = 9,
+  PRT_PADDING   = 10,
+  PRT_FLOWCTRL  = 11,
 } protocol_type_t;
 
 bool protover_contains_long_protocol_names(const char *s);
diff --git a/src/core/or/relay.c b/src/core/or/relay.c
index f5fc1cfbb3..f83d2365e4 100644
--- a/src/core/or/relay.c
+++ b/src/core/or/relay.c
@@ -49,18 +49,19 @@
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "feature/client/circpathbias.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "lib/compress/compress.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircommon/directory.h"
@@ -80,7 +81,6 @@
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/routerlist.h"
 #include "core/or/scheduler.h"
-#include "feature/stats/rephist.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/cell_queue_st.h"
@@ -93,15 +93,12 @@
 #include "core/or/origin_circuit_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "core/or/socks_request_st.h"
-
-#include "lib/intmath/weakrng.h"
+#include "core/or/sendme.h"
 
 static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell,
                                             cell_direction_t cell_direction,
                                             crypt_path_t *layer_hint);
 
-static void circuit_consider_sending_sendme(circuit_t *circ,
-                                            crypt_path_t *layer_hint);
 static void circuit_resume_edge_reading(circuit_t *circ,
                                         crypt_path_t *layer_hint);
 static int circuit_resume_edge_reading_helper(edge_connection_t *conn,
@@ -134,9 +131,6 @@ uint64_t stats_n_relay_cells_delivered = 0;
  * reached (see append_cell_to_circuit_queue()) */
 uint64_t stats_n_circ_max_cell_reached = 0;
 
-/** Used to tell which stream to read from first on a circuit. */
-static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT;
-
 /**
  * Update channel usage state based on the type of relay cell and
  * circuit properties.
@@ -253,6 +247,10 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   if (recognized) {
     edge_connection_t *conn = NULL;
 
+    /* Recognized cell, the cell digest has been updated, we'll record it for
+     * the SENDME if need be. */
+    sendme_record_received_cell_digest(circ, layer_hint);
+
     if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
       if (pathbias_check_probe_response(circ, cell) == -1) {
         pathbias_count_valid_cells(circ, cell);
@@ -267,8 +265,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_OUT) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending away from origin.");
-      if ((reason=connection_edge_process_relay_cell(cell, circ, conn, NULL))
-          < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn, NULL);
+      if (reason < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "connection_edge_process_relay_cell (away from origin) "
                "failed.");
@@ -278,8 +276,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_IN) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending to origin.");
-      if ((reason = connection_edge_process_relay_cell(cell, circ, conn,
-                                                       layer_hint)) < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn,
+                                                  layer_hint);
+      if (reason < 0) {
         /* If a client is trying to connect to unknown hidden service port,
          * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit.
          * Do not log warn as this is an expected behavior for a service. */
@@ -293,7 +292,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     return 0;
   }
 
-  /* not recognized. pass it on. */
+  /* not recognized. inform circpad and pass it on. */
+  circpad_deliver_unrecognized_cell_events(circ, cell_direction);
+
   if (cell_direction == CELL_DIRECTION_OUT) {
     cell->circ_id = circ->n_circ_id; /* switch it */
     chan = circ->n_chan;
@@ -353,11 +354,11 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
  *  - Encrypt it to the right layer
  *  - Append it to the appropriate cell_queue on <b>circ</b>.
  */
-static int
-circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
+MOCK_IMPL(int,
+circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
                            cell_direction_t cell_direction,
                            crypt_path_t *layer_hint, streamid_t on_stream,
-                           const char *filename, int lineno)
+                           const char *filename, int lineno))
 {
   channel_t *chan; /* where to send the cell */
 
@@ -524,6 +525,8 @@ relay_command_to_string(uint8_t command)
     case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK";
     case RELAY_COMMAND_EXTEND2: return "EXTEND2";
     case RELAY_COMMAND_EXTENDED2: return "EXTENDED2";
+    case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE";
+    case RELAY_COMMAND_PADDING_NEGOTIATED: return "PADDING_NEGOTIATED";
     default:
       tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u",
                    (unsigned)command);
@@ -531,6 +534,64 @@ relay_command_to_string(uint8_t command)
   }
 }
 
+/** When padding a cell with randomness, leave this many zeros after the
+ * payload. */
+#define CELL_PADDING_GAP 4
+
+/** Return the offset where the padding should start. The <b>data_len</b> is
+ * the relay payload length expected to be put in the cell. It can not be
+ * bigger than RELAY_PAYLOAD_SIZE else this function assert().
+ *
+ * Value will always be smaller than CELL_PAYLOAD_SIZE because this offset is
+ * for the entire cell length not just the data payload length. Zero is
+ * returned if there is no room for padding.
+ *
+ * This function always skips the first 4 bytes after the payload because
+ * having some unused zero bytes has saved us a lot of times in the past. */
+
+STATIC size_t
+get_pad_cell_offset(size_t data_len)
+{
+  /* This is never supposed to happen but in case it does, stop right away
+   * because if tor is tricked somehow into not adding random bytes to the
+   * payload with this function returning 0 for a bad data_len, the entire
+   * authenticated SENDME design can be bypassed leading to bad denial of
+   * service attacks. */
+  tor_assert(data_len <= RELAY_PAYLOAD_SIZE);
+
+  /* If the offset is larger than the cell payload size, we return an offset
+   * of zero indicating that no padding needs to be added. */
+  size_t offset = RELAY_HEADER_SIZE + data_len + CELL_PADDING_GAP;
+  if (offset >= CELL_PAYLOAD_SIZE) {
+    return 0;
+  }
+  return offset;
+}
+
+/* Add random bytes to the unused portion of the payload, to foil attacks
+ * where the other side can predict all of the bytes in the payload and thus
+ * compute the authenticated SENDME cells without seeing the traffic. See
+ * proposal 289. */
+static void
+pad_cell_payload(uint8_t *cell_payload, size_t data_len)
+{
+  size_t pad_offset, pad_len;
+
+  tor_assert(cell_payload);
+
+  pad_offset = get_pad_cell_offset(data_len);
+  if (pad_offset == 0) {
+    /* We can't add padding so we are done. */
+    return;
+  }
+
+  /* Remember here that the cell_payload is the length of the header and
+   * payload size so we offset it using the full length of the cell. */
+  pad_len = CELL_PAYLOAD_SIZE - pad_offset;
+  crypto_fast_rng_getbytes(get_thread_fast_rng(),
+                           cell_payload + pad_offset, pad_len);
+}
+
 /** Make a relay cell out of <b>relay_command</b> and <b>payload</b>, and send
  * it onto the open circuit <b>circ</b>. <b>stream_id</b> is the ID on
  * <b>circ</b> for the stream that's sending the relay cell, or 0 if it's a
@@ -574,11 +635,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
   if (payload_len)
     memcpy(cell.payload+RELAY_HEADER_SIZE, payload, payload_len);
 
+  /* Add random padding to the cell if we can. */
+  pad_cell_payload(cell.payload, payload_len);
+
   log_debug(LD_OR,"delivering %d cell %s.", relay_command,
             cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward");
 
-  if (relay_command == RELAY_COMMAND_DROP)
-    rep_hist_padding_count_write(PADDING_TYPE_DROP);
+  /* Tell circpad we're sending a relay cell */
+  circpad_deliver_sent_relay_cell_events(circ, relay_command);
 
   /* If we are sending an END cell and this circuit is used for a tunneled
    * directory request, advance its state. */
@@ -602,7 +666,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
        * one of them.  Don't worry about the conn protocol version:
        * append_cell_to_circuit_queue will fix it up. */
       cell.command = CELL_RELAY_EARLY;
-      --origin_circ->remaining_relay_early_cells;
+      /* If we're out of relay early cells, tell circpad */
+      if (--origin_circ->remaining_relay_early_cells == 0)
+        circpad_machine_event_circ_has_no_relay_early(origin_circ);
       log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.",
                 (int)origin_circ->remaining_relay_early_cells);
       /* Memorize the command that is sent as RELAY_EARLY cell; helps debug
@@ -639,6 +705,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
     circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL);
     return -1;
   }
+
+  /* If applicable, note the cell digest for the SENDME version 1 purpose if
+   * we need to. This call needs to be after the circuit_package_relay_cell()
+   * because the cell digest is set within that function. */
+  if (relay_command == RELAY_COMMAND_DATA) {
+    sendme_record_cell_digest_on_circ(circ, cpath_layer);
+  }
+
   return 0;
 }
 
@@ -1428,85 +1502,108 @@ connection_edge_process_relay_cell_not_open(
 //  return -1;
 }
 
-/** An incoming relay cell has arrived on circuit <b>circ</b>. If
- * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
- * destined for <b>conn</b>.
+/** Process a SENDME cell that arrived on <b>circ</b>. If it is a stream level
+ * cell, it is destined for the given <b>conn</b>. If it is a circuit level
+ * cell, it is destined for the <b>layer_hint</b>. The <b>domain</b> is the
+ * logging domain that should be used.
  *
- * If <b>layer_hint</b> is defined, then we're the origin of the
- * circuit, and it specifies the hop that packaged <b>cell</b>.
- *
- * Return -reason if you want to warn and tear down the circuit, else 0.
- */
-STATIC int
-connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
-                                   edge_connection_t *conn,
-                                   crypt_path_t *layer_hint)
+ * Return 0 if everything went well or a negative value representing a circuit
+ * end reason on error for which the caller is responsible for closing it. */
+static int
+process_sendme_cell(const relay_header_t *rh, const cell_t *cell,
+                    circuit_t *circ, edge_connection_t *conn,
+                    crypt_path_t *layer_hint, int domain)
 {
-  static int num_seen=0;
-  relay_header_t rh;
-  unsigned domain = layer_hint?LD_APP:LD_EXIT;
-  int reason;
-  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
-                            * that's in the EXIT_CONN_STATE_RESOLVING
-                            * or EXIT_CONN_STATE_CONNECTING states. */
-
-  tor_assert(cell);
-  tor_assert(circ);
+  int ret;
 
-  relay_header_unpack(&rh, cell->payload);
-//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
-  num_seen++;
-  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
-            num_seen, rh.command, rh.stream_id);
+  tor_assert(rh);
 
-  if (rh.length > RELAY_PAYLOAD_SIZE) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Relay cell length field too long. Closing circuit.");
-    return - END_CIRC_REASON_TORPROTOCOL;
+  if (!rh->stream_id) {
+    /* Circuit level SENDME cell. */
+    ret = sendme_process_circuit_level(layer_hint, circ,
+                                       cell->payload + RELAY_HEADER_SIZE,
+                                       rh->length);
+    if (ret < 0) {
+      return ret;
+    }
+    /* Resume reading on any streams now that we've processed a valid
+     * SENDME cell that updated our package window. */
+    circuit_resume_edge_reading(circ, layer_hint);
+    /* We are done, the rest of the code is for the stream level. */
+    return 0;
   }
 
-  if (rh.stream_id == 0) {
-    switch (rh.command) {
-      case RELAY_COMMAND_BEGIN:
-      case RELAY_COMMAND_CONNECTED:
-      case RELAY_COMMAND_END:
-      case RELAY_COMMAND_RESOLVE:
-      case RELAY_COMMAND_RESOLVED:
-      case RELAY_COMMAND_BEGIN_DIR:
-        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
-               "stream_id. Dropping.", (int)rh.command);
-        return 0;
-      default:
-        ;
+  /* No connection, might be half edge state. We are done if so. */
+  if (!conn) {
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+      if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                               rh->stream_id)) {
+        circuit_read_valid_data(ocirc, rh->length);
+        log_info(domain, "Sendme cell on circ %u valid on half-closed "
+                         "stream id %d",
+                 ocirc->global_identifier, rh->stream_id);
+      }
     }
+
+    log_info(domain, "SENDME cell dropped, unknown stream (streamid %d).",
+             rh->stream_id);
+    return 0;
   }
 
-  /* either conn is NULL, in which case we've got a control cell, or else
-   * conn points to the recognized stream. */
+  /* Stream level SENDME cell. */
+  ret = sendme_process_stream_level(conn, circ, rh->length);
+  if (ret < 0) {
+    /* Means we need to close the circuit with reason ret. */
+    return ret;
+  }
 
-  if (conn && !connection_state_is_open(TO_CONN(conn))) {
-    if (conn->base_.type == CONN_TYPE_EXIT &&
-        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
-         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
-        rh.command == RELAY_COMMAND_DATA) {
-      /* Allow DATA cells to be delivered to an exit node in state
-       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
-       * This speeds up HTTP, for example. */
-      optimistic_data = 1;
-    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
-      log_warn(LD_BUG, "Somehow I had a connection that matched a "
-               "data cell with stream ID 0.");
-    } else {
-      return connection_edge_process_relay_cell_not_open(
-               &rh, cell, circ, conn, layer_hint);
-    }
+  /* We've now processed properly a SENDME cell, all windows have been
+   * properly updated, we'll read on the edge connection to see if we can
+   * get data out towards the end point (Exit or client) since we are now
+   * allowed to deliver more cells. */
+
+  if (circuit_queue_streams_are_blocked(circ)) {
+    /* Still waiting for queue to flush; don't touch conn */
+    return 0;
+  }
+  connection_start_reading(TO_CONN(conn));
+  /* handle whatever might still be on the inbuf */
+  if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
+    /* (We already sent an end cell if possible) */
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
   }
+  return 0;
+}
 
-  switch (rh.command) {
-    case RELAY_COMMAND_DROP:
-      rep_hist_padding_count_read(PADDING_TYPE_DROP);
-//      log_info(domain,"Got a relay-level padding cell. Dropping.");
-      return 0;
+/** A helper for connection_edge_process_relay_cell(): Actually handles the
+ *  cell that we received on the connection.
+ *
+ *  The arguments are the same as in the parent function
+ *  connection_edge_process_relay_cell(), plus the relay header <b>rh</b> as
+ *  unpacked by the parent function, and <b>optimistic_data</b> as set by the
+ *  parent function.
+ */
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data)
+{
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int reason;
+
+  tor_assert(rh);
+
+  /* First pass the cell to the circuit padding subsystem, in case it's a
+   * padding cell or circuit that should be handled there. */
+  if (circpad_check_received_cell(cell, circ, layer_hint, rh) == 0) {
+    log_debug(domain, "Cell handled as circuit padding");
+    return 0;
+  }
+
+  /* Now handle all the other commands */
+  switch (rh->command) {
     case RELAY_COMMAND_BEGIN:
     case RELAY_COMMAND_BEGIN_DIR:
       if (layer_hint &&
@@ -1527,7 +1624,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "Begin cell for known stream. Dropping.");
         return 0;
       }
-      if (rh.command == RELAY_COMMAND_BEGIN_DIR &&
+      if (rh->command == RELAY_COMMAND_BEGIN_DIR &&
           circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) {
         /* Assign this circuit and its app-ward OR connection a unique ID,
          * so that we can measure download times. The local edge and dir
@@ -1540,24 +1637,21 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       return connection_exit_begin_conn(cell, circ);
     case RELAY_COMMAND_DATA:
       ++stats_n_data_cells_received;
-      if (( layer_hint && --layer_hint->deliver_window < 0) ||
-          (!layer_hint && --circ->deliver_window < 0)) {
+
+      /* Update our circuit-level deliver window that we received a DATA cell.
+       * If the deliver window goes below 0, we end the circuit and stream due
+       * to a protocol failure. */
+      if (sendme_circuit_data_received(circ, layer_hint) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) circ deliver_window below 0. Killing.");
-        if (conn) {
-          /* XXXX Do we actually need to do this?  Will killing the circuit
-           * not send an END and mark the stream for close as appropriate? */
-          connection_edge_end(conn, END_STREAM_REASON_TORPROTOCOL);
-          connection_mark_for_close(TO_CONN(conn));
-        }
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
-      log_debug(domain,"circ deliver_window now %d.", layer_hint ?
-                layer_hint->deliver_window : circ->deliver_window);
 
-      circuit_consider_sending_sendme(circ, layer_hint);
+      /* Consider sending a circuit-level SENDME cell. */
+      sendme_circuit_consider_sending(circ, layer_hint);
 
-      if (rh.stream_id == 0) {
+      if (rh->stream_id == 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero "
                "stream_id. Dropping.");
         return 0;
@@ -1565,32 +1659,37 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (connection_half_edge_is_valid_data(ocirc->half_streams,
-                                                 rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
+                                                 rh->stream_id)) {
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "data cell on circ %u valid on half-closed "
-                     "stream id %d", ocirc->global_identifier, rh.stream_id);
+                     "stream id %d", ocirc->global_identifier, rh->stream_id);
           }
         }
 
         log_info(domain,"data cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
+                 rh->stream_id);
         return 0;
       }
 
-      if (--conn->deliver_window < 0) { /* is it below 0 after decrement? */
+      /* Update our stream-level deliver window that we just received a DATA
+       * cell. Going below 0 means we have a protocol level error so the
+       * stream and circuit are closed. */
+
+      if (sendme_stream_data_received(conn) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) conn deliver_window below 0. Killing.");
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
       /* Total all valid application bytes delivered */
-      if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      if (CIRCUIT_IS_ORIGIN(circ) && rh->length > 0) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
 
-      stats_n_data_bytes_received += rh.length;
+      stats_n_data_bytes_received += rh->length;
       connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE),
-                              rh.length, TO_CONN(conn));
+                              rh->length, TO_CONN(conn));
 
 #ifdef MEASUREMENTS_21206
       /* Count number of RELAY_DATA cells received on a linked directory
@@ -1606,25 +1705,25 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         /* Only send a SENDME if we're not getting optimistic data; otherwise
          * a SENDME could arrive before the CONNECTED.
          */
-        connection_edge_consider_sending_sendme(conn);
+        sendme_connection_edge_consider_sending(conn);
       }
 
       return 0;
     case RELAY_COMMAND_END:
-      reason = rh.length > 0 ?
+      reason = rh->length > 0 ?
         get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC;
       if (!conn) {
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (connection_half_edge_is_valid_end(ocirc->half_streams,
-                                                rh.stream_id)) {
+                                                rh->stream_id)) {
 
-            circuit_read_valid_data(ocirc, rh.length);
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "end cell (%s) on circ %u valid on half-closed "
                      "stream id %d",
                      stream_end_reason_to_string(reason),
-                     ocirc->global_identifier, rh.stream_id);
+                     ocirc->global_identifier, rh->stream_id);
             return 0;
           }
         }
@@ -1656,7 +1755,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
 
         /* Total all valid application bytes delivered */
         if (CIRCUIT_IS_ORIGIN(circ)) {
-          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
         }
       }
       return 0;
@@ -1664,7 +1763,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_EXTEND2: {
       static uint64_t total_n_extend=0, total_nonearly=0;
       total_n_extend++;
-      if (rh.stream_id) {
+      if (rh->stream_id) {
         log_fn(LOG_PROTOCOL_WARN, domain,
                "'extend' cell received for non-zero stream. Dropping.");
         return 0;
@@ -1705,9 +1804,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_debug(domain,"Got an extended cell! Yay.");
       {
         extended_cell_t extended_cell;
-        if (extended_cell_parse(&extended_cell, rh.command,
+        if (extended_cell_parse(&extended_cell, rh->command,
                         (const uint8_t*)cell->payload+RELAY_HEADER_SIZE,
-                        rh.length)<0) {
+                        rh->length)<0) {
           log_warn(LD_PROTOCOL,
                    "Can't parse EXTENDED cell; killing circuit.");
           return -END_CIRC_REASON_TORPROTOCOL;
@@ -1725,7 +1824,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       }
       /* Total all valid bytes delivered. */
       if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
       return 0;
     case RELAY_COMMAND_TRUNCATE:
@@ -1769,7 +1868,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
        * circuit is being torn down anyway, though.  */
       if (CIRCUIT_IS_ORIGIN(circ)) {
         circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
+                                rh->length);
       }
       circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
                         get_uint8(cell->payload + RELAY_HEADER_SIZE));
@@ -1784,11 +1883,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       if (CIRCUIT_IS_ORIGIN(circ)) {
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (connection_half_edge_is_valid_connected(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "connected cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1796,102 +1895,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_info(domain,
                "'connected' received on circid %u for streamid %d, "
                "no conn attached anymore. Ignoring.",
-               (unsigned)circ->n_circ_id, rh.stream_id);
+               (unsigned)circ->n_circ_id, rh->stream_id);
       return 0;
     case RELAY_COMMAND_SENDME:
-      if (!rh.stream_id) {
-        if (layer_hint) {
-          if (layer_hint->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from exit relay. "
-                   "Closing circ.");
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          layer_hint->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.",
-                    layer_hint->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-
-          /* We count circuit-level sendme's as valid delivered data because
-           * they are rate limited.
-           */
-          if (CIRCUIT_IS_ORIGIN(circ)) {
-            circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                    rh.length);
-          }
-
-        } else {
-          if (circ->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&client_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from client. "
-                   "Closing circ (window %d).",
-                   circ->package_window);
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          circ->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,
-                    "circ-level sendme at non-origin, packagewindow %d.",
-                    circ->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-        }
-        return 0;
-      }
-      if (!conn) {
-        if (CIRCUIT_IS_ORIGIN(circ)) {
-          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-          if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
-                                                   rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
-            log_info(domain,
-                    "sendme cell on circ %u valid on half-closed "
-                    "stream id %d", ocirc->global_identifier, rh.stream_id);
-          }
-        }
-
-        log_info(domain,"sendme cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
-        return 0;
-      }
-
-      /* Don't allow the other endpoint to request more than our maximum
-       * (i.e. initial) stream SENDME window worth of data. Well-behaved
-       * stock clients will not request more than this max (as per the check
-       * in the while loop of connection_edge_consider_sending_sendme()).
-       */
-      if (conn->package_window + STREAMWINDOW_INCREMENT >
-          STREAMWINDOW_START_MAX) {
-        static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
-        log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
-               "Unexpected stream sendme cell. Closing circ (window %d).",
-               conn->package_window);
-        return -END_CIRC_REASON_TORPROTOCOL;
-      }
-
-      /* At this point, the stream sendme is valid */
-      if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
-      }
-
-      conn->package_window += STREAMWINDOW_INCREMENT;
-      log_debug(domain,"stream-level sendme, packagewindow now %d.",
-                conn->package_window);
-      if (circuit_queue_streams_are_blocked(circ)) {
-        /* Still waiting for queue to flush; don't touch conn */
-        return 0;
-      }
-      connection_start_reading(TO_CONN(conn));
-      /* handle whatever might still be on the inbuf */
-      if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
-        /* (We already sent an end cell if possible) */
-        connection_mark_for_close(TO_CONN(conn));
-        return 0;
-      }
-      return 0;
+      return process_sendme_cell(rh, cell, circ, conn, layer_hint, domain);
     case RELAY_COMMAND_RESOLVE:
       if (layer_hint) {
         log_fn(LOG_PROTOCOL_WARN, LD_APP,
@@ -1919,11 +1926,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       if (CIRCUIT_IS_ORIGIN(circ)) {
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (connection_half_edge_is_valid_resolved(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "resolved cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1941,17 +1948,96 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_INTRO_ESTABLISHED:
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       rend_process_relay_cell(circ, layer_hint,
-                              rh.command, rh.length,
+                              rh->command, rh->length,
                               cell->payload+RELAY_HEADER_SIZE);
       return 0;
   }
   log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Received unknown relay command %d. Perhaps the other side is using "
          "a newer version of Tor? Dropping.",
-         rh.command);
+         rh->command);
   return 0; /* for forward compatibility, don't kill the circuit */
 }
 
+/** An incoming relay cell has arrived on circuit <b>circ</b>. If
+ * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
+ * destined for <b>conn</b>.
+ *
+ * If <b>layer_hint</b> is defined, then we're the origin of the
+ * circuit, and it specifies the hop that packaged <b>cell</b>.
+ *
+ * Return -reason if you want to warn and tear down the circuit, else 0.
+ */
+STATIC int
+connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
+                                   edge_connection_t *conn,
+                                   crypt_path_t *layer_hint)
+{
+  static int num_seen=0;
+  relay_header_t rh;
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
+                            * that's in the EXIT_CONN_STATE_RESOLVING
+                            * or EXIT_CONN_STATE_CONNECTING states. */
+
+  tor_assert(cell);
+  tor_assert(circ);
+
+  relay_header_unpack(&rh, cell->payload);
+//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
+  num_seen++;
+  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
+            num_seen, rh.command, rh.stream_id);
+
+  if (rh.length > RELAY_PAYLOAD_SIZE) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Relay cell length field too long. Closing circuit.");
+    return - END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  if (rh.stream_id == 0) {
+    switch (rh.command) {
+      case RELAY_COMMAND_BEGIN:
+      case RELAY_COMMAND_CONNECTED:
+      case RELAY_COMMAND_END:
+      case RELAY_COMMAND_RESOLVE:
+      case RELAY_COMMAND_RESOLVED:
+      case RELAY_COMMAND_BEGIN_DIR:
+        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
+               "stream_id. Dropping.", (int)rh.command);
+        return 0;
+      default:
+        ;
+    }
+  }
+
+  /* Tell circpad that we've received a recognized cell */
+  circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint);
+
+  /* either conn is NULL, in which case we've got a control cell, or else
+   * conn points to the recognized stream. */
+  if (conn && !connection_state_is_open(TO_CONN(conn))) {
+    if (conn->base_.type == CONN_TYPE_EXIT &&
+        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
+         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
+        rh.command == RELAY_COMMAND_DATA) {
+      /* Allow DATA cells to be delivered to an exit node in state
+       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
+       * This speeds up HTTP, for example. */
+      optimistic_data = 1;
+    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
+      log_warn(LD_BUG, "Somehow I had a connection that matched a "
+               "data cell with stream ID 0.");
+    } else {
+      return connection_edge_process_relay_cell_not_open(
+               &rh, cell, circ, conn, layer_hint);
+    }
+  }
+
+  return handle_relay_cell_command(cell, circ, conn, layer_hint,
+                              &rh, optimistic_data);
+}
+
 /** How many relay_data cells have we built, ever? */
 uint64_t stats_n_data_cells_packaged = 0;
 /** How many bytes of data have we put in relay_data cells have we built,
@@ -1965,6 +2051,84 @@ uint64_t stats_n_data_cells_received = 0;
  * ever received were completely full of data. */
 uint64_t stats_n_data_bytes_received = 0;
 
+/**
+ * Called when initializing a circuit, or when we have reached the end of the
+ * window in which we need to send some randomness so that incoming sendme
+ * cells will be unpredictable.  Resets the flags and picks a new window.
+ */
+void
+circuit_reset_sendme_randomness(circuit_t *circ)
+{
+  circ->have_sent_sufficiently_random_cell = 0;
+  circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 +
+    crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2);
+}
+
+/**
+ * Any relay data payload containing fewer than this many real bytes is
+ * considered to have enough randomness to.
+ **/
+#define RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES \
+  (RELAY_PAYLOAD_SIZE - CELL_PADDING_GAP - 16)
+
+/**
+ * Helper. Return the number of bytes that should be put into a cell from a
+ * given edge connection on which <b>n_available</b> bytes are available.
+ */
+STATIC size_t
+connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                           int package_partial,
+                                           circuit_t *on_circuit)
+{
+  if (!n_available)
+    return 0;
+
+  /* Do we need to force this payload to have space for randomness? */
+  const bool force_random_bytes =
+    (on_circuit->send_randomness_after_n_cells == 0) &&
+    (! on_circuit->have_sent_sufficiently_random_cell);
+
+  /* At most how much would we like to send in this cell? */
+  size_t target_length;
+  if (force_random_bytes) {
+    target_length = RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES;
+  } else {
+    target_length = RELAY_PAYLOAD_SIZE;
+  }
+
+  /* Decide how many bytes we will actually put into this cell. */
+  size_t package_length;
+  if (n_available >= target_length) { /* A full payload is available. */
+    package_length = target_length;
+  } else { /* not a full payload available */
+    if (package_partial)
+      package_length = n_available; /* just take whatever's available now */
+    else
+      return 0; /* nothing to do until we have a full payload */
+  }
+
+  /* If we reach this point, we will be definitely sending the cell. */
+  tor_assert_nonfatal(package_length > 0);
+
+  if (package_length <= RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES) {
+    /* This cell will have enough randomness in the padding to make a future
+     * sendme cell unpredictable. */
+    on_circuit->have_sent_sufficiently_random_cell = 1;
+  }
+
+  if (on_circuit->send_randomness_after_n_cells == 0) {
+    /* Either this cell, or some previous cell, had enough padding to
+     * ensure sendme unpredictability. */
+    tor_assert_nonfatal(on_circuit->have_sent_sufficiently_random_cell);
+    /* Pick a new interval in which we need to send randomness. */
+    circuit_reset_sendme_randomness(on_circuit);
+  }
+
+  --on_circuit->send_randomness_after_n_cells;
+
+  return package_length;
+}
+
 /** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or
  * <b>package_partial</b> is true), and the appropriate package windows aren't
  * empty, grab a cell and send it down the circuit.
@@ -2037,17 +2201,14 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     bytes_to_process = connection_get_inbuf_len(TO_CONN(conn));
   }
 
-  if (!bytes_to_process)
+  length = connection_edge_get_inbuf_bytes_to_package(bytes_to_process,
+                                                      package_partial, circ);
+  if (!length)
     return 0;
 
-  if (!package_partial && bytes_to_process < RELAY_PAYLOAD_SIZE)
-    return 0;
+  /* If we reach this point, we will definitely be packaging bytes into
+   * a cell. */
 
-  if (bytes_to_process > RELAY_PAYLOAD_SIZE) {
-    length = RELAY_PAYLOAD_SIZE;
-  } else {
-    length = bytes_to_process;
-  }
   stats_n_data_bytes_packaged += length;
   stats_n_data_cells_packaged += 1;
 
@@ -2082,15 +2243,17 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     return 0;
   }
 
-  if (!cpath_layer) { /* non-rendezvous exit */
-    tor_assert(circ->package_window > 0);
-    circ->package_window--;
-  } else { /* we're an AP, or an exit on a rendezvous circ */
-    tor_assert(cpath_layer->package_window > 0);
-    cpath_layer->package_window--;
+  /* Handle the circuit-level SENDME package window. */
+  if (sendme_note_circuit_data_packaged(circ, cpath_layer) < 0) {
+    /* Package window has gone under 0. Protocol issue. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Circuit package window is below 0. Closing circuit.");
+    conn->end_reason = END_STREAM_REASON_TORPROTOCOL;
+    return -1;
   }
 
-  if (--conn->package_window <= 0) { /* is it 0 after decrement? */
+  /* Handle the stream-level SENDME package window. */
+  if (sendme_note_stream_data_packaged(conn) < 0) {
     connection_stop_reading(TO_CONN(conn));
     log_debug(domain,"conn->package_window reached 0.");
     circuit_consider_stop_edge_reading(circ, cpath_layer);
@@ -2108,42 +2271,6 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
   goto repeat_connection_edge_package_raw_inbuf;
 }
 
-/** Called when we've just received a relay data cell, when
- * we've just finished flushing all bytes to stream <b>conn</b>,
- * or when we've flushed *some* bytes to the stream <b>conn</b>.
- *
- * If conn->outbuf is not too full, and our deliver window is
- * low, send back a suitable number of stream-level sendme cells.
- */
-void
-connection_edge_consider_sending_sendme(edge_connection_t *conn)
-{
-  circuit_t *circ;
-
-  if (connection_outbuf_too_full(TO_CONN(conn)))
-    return;
-
-  circ = circuit_get_by_edge_conn(conn);
-  if (!circ) {
-    /* this can legitimately happen if the destroy has already
-     * arrived and torn down the circuit */
-    log_info(LD_APP,"No circuit associated with conn. Skipping.");
-    return;
-  }
-
-  while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) {
-    log_debug(conn->base_.type == CONN_TYPE_AP ?LD_APP:LD_EXIT,
-              "Outbuf %d, Queuing stream sendme.",
-              (int)conn->base_.outbuf_flushlen);
-    conn->deliver_window += STREAMWINDOW_INCREMENT;
-    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
-                                     NULL, 0) < 0) {
-      log_warn(LD_APP,"connection_edge_send_command failed. Skipping.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The circuit <b>circ</b> has received a circuit-level sendme
  * (on hop <b>layer_hint</b>, if we're the OP). Go through all the
  * attached streams and let them resume reading and packaging, if
@@ -2166,12 +2293,6 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
                                        circ, layer_hint);
 }
 
-void
-stream_choice_seed_weak_rng(void)
-{
-  crypto_seed_weak_rng(&stream_choice_rng);
-}
-
 /** A helper function for circuit_resume_edge_reading() above.
  * The arguments are the same, except that <b>conn</b> is the head
  * of a linked list of edge streams that should each be considered.
@@ -2223,7 +2344,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn,
     int num_streams = 0;
     for (conn = first_conn; conn; conn = conn->next_stream) {
       num_streams++;
-      if (tor_weak_random_one_in_n(&stream_choice_rng, num_streams)) {
+
+      if (crypto_fast_rng_one_in_n(get_thread_fast_rng(), num_streams)) {
         chosen_stream = conn;
       }
       /* Invariant: chosen_stream has been chosen uniformly at random from
@@ -2365,33 +2487,6 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
   return 0;
 }
 
-/** Check if the deliver_window for circuit <b>circ</b> (at hop
- * <b>layer_hint</b> if it's defined) is low enough that we should
- * send a circuit-level sendme back down the circuit. If so, send
- * enough sendmes that the window would be overfull if we sent any
- * more.
- */
-static void
-circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint)
-{
-//  log_fn(LOG_INFO,"Considering: layer_hint is %s",
-//         layer_hint ? "defined" : "null");
-  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
-          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
-    log_debug(LD_CIRC,"Queuing circuit sendme.");
-    if (layer_hint)
-      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
-    else
-      circ->deliver_window += CIRCWINDOW_INCREMENT;
-    if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
-                                     NULL, 0, layer_hint) < 0) {
-      log_warn(LD_CIRC,
-               "relay_send_command_from_edge failed. Circuit's closed.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The total number of cells we have allocated. */
 static size_t total_cells_allocated = 0;
 
@@ -2766,7 +2861,7 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan,
 }
 
 /** Extract the command from a packed cell. */
-static uint8_t
+uint8_t
 packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids)
 {
   if (wide_circ_ids) {
diff --git a/src/core/or/relay.h b/src/core/or/relay.h
index 7cc3c43e43..99f7553013 100644
--- a/src/core/or/relay.h
+++ b/src/core/or/relay.h
@@ -42,6 +42,7 @@ int connection_edge_package_raw_inbuf(edge_connection_t *conn,
                                       int package_partial,
                                       int *max_cells);
 void connection_edge_consider_sending_sendme(edge_connection_t *conn);
+void circuit_reset_sendme_randomness(circuit_t *circ);
 
 extern uint64_t stats_n_data_cells_packaged;
 extern uint64_t stats_n_data_bytes_packaged;
@@ -78,6 +79,11 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue,
 void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out);
 MOCK_DECL(int, channel_flush_from_first_active_circuit,
           (channel_t *chan, int max));
+MOCK_DECL(int, circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno));
+
 void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
                              const char *file, int lineno);
 #define update_circuit_on_cmux(circ, direction) \
@@ -89,11 +95,15 @@ const uint8_t *decode_address_from_payload(tor_addr_t *addr_out,
                                         int payload_len);
 void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan);
 
-void stream_choice_seed_weak_rng(void);
-
 circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids);
+uint8_t packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids);
 
 #ifdef RELAY_PRIVATE
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data);
+
 STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
                          tor_addr_t *addr_out, int *ttl_out);
 /** An address-and-ttl tuple as yielded by resolved_cell_parse */
@@ -117,8 +127,11 @@ STATIC int cell_queues_check_size(void);
 STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                                    edge_connection_t *conn,
                                    crypt_path_t *layer_hint);
+STATIC size_t get_pad_cell_offset(size_t payload_len);
+STATIC size_t connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                                      int package_partial,
+                                                      circuit_t *on_circuit);
 
 #endif /* defined(RELAY_PRIVATE) */
 
 #endif /* !defined(TOR_RELAY_H) */
-
diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h
index dafce257c7..83bbd329a6 100644
--- a/src/core/or/relay_crypto_st.h
+++ b/src/core/or/relay_crypto_st.h
@@ -25,7 +25,9 @@ struct relay_crypto_t {
   /** Digest state for cells heading away from the OR at this step. */
   struct crypto_digest_t *b_digest;
 
+  /** Digest used for the next SENDME cell if any. */
+  uint8_t sendme_digest[DIGEST_LEN];
 };
 #undef crypto_cipher_t
 
-#endif
+#endif /* !defined(RELAY_CRYPTO_ST_H) */
diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c
index 9f1a27d501..0e3f0fe815 100644
--- a/src/core/or/scheduler.c
+++ b/src/core/or/scheduler.c
@@ -9,7 +9,7 @@
 #define SCHEDULER_KIST_PRIVATE
 #include "core/or/scheduler.h"
 #include "core/mainloop/mainloop.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #define TOR_CHANNEL_INTERNAL_
 #include "core/or/channeltls.h"
 #include "lib/evloop/compat_libevent.h"
@@ -267,7 +267,7 @@ select_scheduler(void)
           log_notice(LD_SCHED, "Scheduler type KIST has been disabled by "
                                "the consensus or no kernel support.");
         }
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
         log_info(LD_SCHED, "Scheduler type KIST not built in");
 #endif /* defined(HAVE_KIST_SUPPORT) */
         continue;
diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c
index 79ecb0bc7e..35b613cb8a 100644
--- a/src/core/or/scheduler_kist.c
+++ b/src/core/or/scheduler_kist.c
@@ -4,7 +4,7 @@
 #define SCHEDULER_KIST_PRIVATE
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/nodelist/networkstatus.h"
@@ -104,7 +104,7 @@ static unsigned int kist_lite_mode = 0;
  * changed and it doesn't recognized the values passed to the syscalls needed
  * by KIST. In that case, fallback to the naive approach. */
 static unsigned int kist_no_kernel_support = 0;
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 static unsigned int kist_lite_mode = 1;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -298,7 +298,7 @@ update_socket_info_impl, (socket_table_ent_t *ent))
   }
   return;
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
   goto fallback;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -724,7 +724,7 @@ kist_scheduler_run(void)
     SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
       scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
       if (!smartlist_contains(cp, readd_chan)) {
-        if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+        if (!SCHED_BUG(readd_chan->sched_heap_idx != -1, readd_chan)) {
           /* XXXX Note that the check above is in theory redundant with
            * the smartlist_contains check.  But let's make sure we're
            * not messing anything up, and leave them both for now. */
@@ -833,7 +833,7 @@ scheduler_can_use_kist(void)
   return run_interval > 0;
 }
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 
 int
 scheduler_can_use_kist(void)
diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c
new file mode 100644
index 0000000000..e61cf0c0f8
--- /dev/null
+++ b/src/core/or/sendme.c
@@ -0,0 +1,710 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.c
+ * \brief Code that is related to SENDME cells both in terms of
+ *        creating/parsing cells and handling the content.
+ */
+
+#define SENDME_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/mainloop/connection.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+#include "feature/nodelist/networkstatus.h"
+#include "lib/ctime/di_ops.h"
+#include "trunnel/sendme_cell.h"
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * used when emitting a SENDME cell. */
+STATIC int
+get_emit_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_emit_min_version",
+                                 SENDME_EMIT_MIN_VERSION_DEFAULT,
+                                 SENDME_EMIT_MIN_VERSION_MIN,
+                                 SENDME_EMIT_MIN_VERSION_MAX);
+}
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * accepted when receiving a SENDME cell. */
+STATIC int
+get_accept_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_accept_min_version",
+                                 SENDME_ACCEPT_MIN_VERSION_DEFAULT,
+                                 SENDME_ACCEPT_MIN_VERSION_MIN,
+                                 SENDME_ACCEPT_MIN_VERSION_MAX);
+}
+
+/* Pop the first cell digset on the given circuit from the SENDME last digests
+ * list. NULL is returned if the list is uninitialized or empty.
+ *
+ * The caller gets ownership of the returned digest thus is responsible for
+ * freeing the memory. */
+static uint8_t *
+pop_first_cell_digest(const circuit_t *circ)
+{
+  uint8_t *circ_digest;
+
+  tor_assert(circ);
+
+  if (circ->sendme_last_digests == NULL ||
+      smartlist_len(circ->sendme_last_digests) == 0) {
+    return NULL;
+  }
+
+  /* More cell digest than the SENDME window is never suppose to happen. The
+   * cell should have been rejected before reaching this point due to its
+   * package_window down to 0 leading to a circuit close. Scream loudly but
+   * still pop the element so we don't memory leak. */
+  tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <=
+                      CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT);
+
+  circ_digest = smartlist_get(circ->sendme_last_digests, 0);
+  smartlist_del_keeporder(circ->sendme_last_digests, 0);
+  return circ_digest;
+}
+
+/* Return true iff the given cell digest matches the first digest in the
+ * circuit sendme list. */
+static bool
+v1_digest_matches(const uint8_t *circ_digest, const uint8_t *cell_digest)
+{
+  tor_assert(circ_digest);
+  tor_assert(cell_digest);
+
+  /* Compare the digest with the one in the SENDME. This cell is invalid
+   * without a perfect match. */
+  if (tor_memneq(circ_digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "SENDME v1 cell digest do not match.");
+    return false;
+  }
+
+  /* Digests matches! */
+  return true;
+}
+
+/* Return true iff the given decoded SENDME version 1 cell is valid and
+ * matches the expected digest on the circuit.
+ *
+ * Validation is done by comparing the digest in the cell from the previous
+ * cell we saw which tells us that the other side has in fact seen that cell.
+ * See proposal 289 for more details. */
+static bool
+cell_v1_is_valid(const sendme_cell_t *cell, const uint8_t *circ_digest)
+{
+  tor_assert(cell);
+  tor_assert(circ_digest);
+
+  const uint8_t *cell_digest = sendme_cell_getconstarray_data_v1_digest(cell);
+  return v1_digest_matches(circ_digest, cell_digest);
+}
+
+/* Return true iff the given cell version can be handled or if the minimum
+ * accepted version from the consensus is known to us. */
+STATIC bool
+cell_version_can_be_handled(uint8_t cell_version)
+{
+  int accept_version = get_accept_min_version();
+
+  /* We will first check if the consensus minimum accepted version can be
+   * handled by us and if not, regardless of the cell version we got, we can't
+   * continue. */
+  if (accept_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Unable to accept SENDME version %u (from consensus). "
+           "We only support <= %u. Probably your tor is too old?",
+           accept_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  /* Then, is this version below the accepted version from the consensus? If
+   * yes, we must not handle it. */
+  if (cell_version < accept_version) {
+    log_info(LD_PROTOCOL, "Unacceptable SENDME version %u. Only "
+                          "accepting %u (from consensus). Closing circuit.",
+             cell_version, accept_version);
+    goto invalid;
+  }
+
+  /* Is this cell version supported by us? */
+  if (cell_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_info(LD_PROTOCOL, "SENDME cell version %u is not supported by us. "
+                          "We only support <= %u",
+             cell_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  return true;
+ invalid:
+  return false;
+}
+
+/* Return true iff the encoded SENDME cell in cell_payload of length
+ * cell_payload_len is valid. For each version:
+ *
+ *  0: No validation
+ *  1: Authenticated with last cell digest.
+ *
+ * This is the main critical function to make sure we can continue to
+ * send/recv cells on a circuit. If the SENDME is invalid, the circuit should
+ * be marked for close by the caller. */
+STATIC bool
+sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload,
+                size_t cell_payload_len)
+{
+  uint8_t cell_version;
+  uint8_t *circ_digest = NULL;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* An empty payload means version 0 so skip trunnel parsing. We won't be
+   * able to parse a 0 length buffer into a valid SENDME cell. */
+  if (cell_payload_len == 0) {
+    cell_version = 0;
+  } else {
+    /* First we'll decode the cell so we can get the version. */
+    if (sendme_cell_parse(&cell, cell_payload, cell_payload_len) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unparseable SENDME cell received. Closing circuit.");
+      goto invalid;
+    }
+    cell_version = sendme_cell_get_version(cell);
+  }
+
+  /* Validate that we can handle this cell version. */
+  if (!cell_version_can_be_handled(cell_version)) {
+    goto invalid;
+  }
+
+  /* Pop the first element that was added (FIFO). We do that regardless of the
+   * version so we don't accumulate on the circuit if v0 is used by the other
+   * end point. */
+  circ_digest = pop_first_cell_digest(circ);
+  if (circ_digest == NULL) {
+    /* We shouldn't have received a SENDME if we have no digests. Log at
+     * protocol warning because it can be tricked by sending many SENDMEs
+     * without prior data cell. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "We received a SENDME but we have no cell digests to match. "
+           "Closing circuit.");
+    goto invalid;
+  }
+
+  /* Validate depending on the version now. */
+  switch (cell_version) {
+  case 0x01:
+    if (!cell_v1_is_valid(cell, circ_digest)) {
+      goto invalid;
+    }
+    break;
+  case 0x00:
+    /* Version 0, there is no work to be done on the payload so it is
+     * necessarily valid if we pass the version validation. */
+    break;
+  default:
+    log_warn(LD_PROTOCOL, "Unknown SENDME cell version %d received.",
+             cell_version);
+    tor_assert_nonfatal_unreached();
+    break;
+  }
+
+  /* Valid cell. */
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return true;
+ invalid:
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return false;
+}
+
+/* Build and encode a version 1 SENDME cell into payload, which must be at
+ * least of RELAY_PAYLOAD_SIZE bytes, using the digest for the cell data.
+ *
+ * Return the size in bytes of the encoded cell in payload. A negative value
+ * is returned on encoding failure. */
+STATIC ssize_t
+build_cell_payload_v1(const uint8_t *cell_digest, uint8_t *payload)
+{
+  ssize_t len = -1;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(cell_digest);
+  tor_assert(payload);
+
+  cell = sendme_cell_new();
+
+  /* Building a payload for version 1. */
+  sendme_cell_set_version(cell, 0x01);
+  /* Set the data length field for v1. */
+  sendme_cell_set_data_len(cell, TRUNNEL_SENDME_V1_DIGEST_LEN);
+
+  /* Copy the digest into the data payload. */
+  memcpy(sendme_cell_getarray_data_v1_digest(cell), cell_digest,
+         sendme_cell_get_data_len(cell));
+
+  /* Finally, encode the cell into the payload. */
+  len = sendme_cell_encode(payload, RELAY_PAYLOAD_SIZE, cell);
+
+  sendme_cell_free(cell);
+  return len;
+}
+
+/* Send a circuit-level SENDME on the given circuit using the layer_hint if
+ * not NULL. The digest is only used for version 1.
+ *
+ * Return 0 on success else a negative value and the circuit will be closed
+ * because we failed to send the cell on it. */
+static int
+send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint,
+                          const uint8_t *cell_digest)
+{
+  uint8_t emit_version;
+  uint8_t payload[RELAY_PAYLOAD_SIZE];
+  ssize_t payload_len;
+
+  tor_assert(circ);
+  tor_assert(cell_digest);
+
+  emit_version = get_emit_min_version();
+  switch (emit_version) {
+  case 0x01:
+    payload_len = build_cell_payload_v1(cell_digest, payload);
+    if (BUG(payload_len < 0)) {
+      /* Unable to encode the cell, abort. We can recover from this by closing
+       * the circuit but in theory it should never happen. */
+      return -1;
+    }
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 1 cell.");
+    break;
+  case 0x00:
+    FALLTHROUGH;
+  default:
+    /* Unknown version, fallback to version 0 meaning no payload. */
+    payload_len = 0;
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 0 cell. "
+                           "Consensus emit version is %d", emit_version);
+    break;
+  }
+
+  if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
+                                   (char *) payload, payload_len,
+                                   layer_hint) < 0) {
+    log_warn(LD_CIRC,
+             "SENDME relay_send_command_from_edge failed. Circuit's closed.");
+    return -1; /* the circuit's closed, don't continue */
+  }
+  return 0;
+}
+
+/* Record the cell digest only if the next cell is expected to be a SENDME. */
+static void
+record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest)
+{
+  tor_assert(circ);
+  tor_assert(sendme_digest);
+
+  /* Add the digest to the last seen list in the circuit. */
+  if (circ->sendme_last_digests == NULL) {
+    circ->sendme_last_digests = smartlist_new();
+  }
+  smartlist_add(circ->sendme_last_digests,
+                tor_memdup(sendme_digest, DIGEST_LEN));
+}
+
+/*
+ * Public API
+ */
+
+/** Return true iff the next cell for the given cell window is expected to be
+ * a SENDME.
+ *
+ * We are able to know that because the package or deliver window value minus
+ * one cell (the possible SENDME cell) should be a multiple of the increment
+ * window value. */
+static bool
+circuit_sendme_cell_is_next(int window)
+{
+  /* At the start of the window, no SENDME will be expected. */
+  if (window == CIRCWINDOW_START) {
+    return false;
+  }
+
+  /* Are we at the limit of the increment and if not, we don't expect next
+   * cell is a SENDME.
+   *
+   * We test against the window minus 1 because when we are looking if the
+   * next cell is a SENDME, the window (either package or deliver) hasn't been
+   * decremented just yet so when this is called, we are currently processing
+   * the "window - 1" cell.
+   *
+   * This function is used when recording a cell digest and this is done quite
+   * low in the stack when decrypting or encrypting a cell. The window is only
+   * updated once the cell is actually put in the outbuf. */
+  if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) {
+    return false;
+  }
+
+  /* Next cell is expected to be a SENDME. */
+  return true;
+}
+
+/** Called when we've just received a relay data cell, when we've just
+ * finished flushing all bytes to stream <b>conn</b>, or when we've flushed
+ * *some* bytes to the stream <b>conn</b>.
+ *
+ * If conn->outbuf is not too full, and our deliver window is low, send back a
+ * suitable number of stream-level sendme cells.
+ */
+void
+sendme_connection_edge_consider_sending(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  int log_domain = TO_CONN(conn)->type == CONN_TYPE_AP ? LD_APP : LD_EXIT;
+
+  /* Don't send it if we still have data to deliver. */
+  if (connection_outbuf_too_full(TO_CONN(conn))) {
+    goto end;
+  }
+
+  if (circuit_get_by_edge_conn(conn) == NULL) {
+    /* This can legitimately happen if the destroy has already arrived and
+     * torn down the circuit. */
+    log_info(log_domain, "No circuit associated with edge connection. "
+                         "Skipping sending SENDME.");
+    goto end;
+  }
+
+  while (conn->deliver_window <=
+         (STREAMWINDOW_START - STREAMWINDOW_INCREMENT)) {
+    log_debug(log_domain, "Outbuf %" TOR_PRIuSZ ", queuing stream SENDME.",
+              TO_CONN(conn)->outbuf_flushlen);
+    conn->deliver_window += STREAMWINDOW_INCREMENT;
+    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
+                                     NULL, 0) < 0) {
+      log_warn(LD_BUG, "connection_edge_send_command failed while sending "
+                       "a SENDME. Circuit probably closed, skipping.");
+      goto end; /* The circuit's closed, don't continue */
+    }
+  }
+
+ end:
+  return;
+}
+
+/** Check if the deliver_window for circuit <b>circ</b> (at hop
+ * <b>layer_hint</b> if it's defined) is low enough that we should
+ * send a circuit-level sendme back down the circuit. If so, send
+ * enough sendmes that the window would be overfull if we sent any
+ * more.
+ */
+void
+sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  bool sent_one_sendme = false;
+  const uint8_t *digest;
+
+  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
+          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
+    log_debug(LD_CIRC,"Queuing circuit sendme.");
+    if (layer_hint) {
+      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = cpath_get_sendme_digest(layer_hint);
+    } else {
+      circ->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+    }
+    if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) {
+      return; /* The circuit's closed, don't continue */
+    }
+    /* Current implementation is not suppose to send multiple SENDME at once
+     * because this means we would use the same relay crypto digest for each
+     * SENDME leading to a mismatch on the other side and the circuit to
+     * collapse. Scream loudly if it ever happens so we can address it. */
+    tor_assert_nonfatal(!sent_one_sendme);
+    sent_one_sendme = true;
+  }
+}
+
+/* Process a circuit-level SENDME cell that we just received. The layer_hint,
+ * if not NULL, is the Exit hop of the connection which means that we are a
+ * client. In that case, circ must be an origin circuit. The cell_body_len is
+ * the length of the SENDME cell payload (excluding the header). The
+ * cell_payload is the payload.
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_circuit_level(crypt_path_t *layer_hint,
+                             circuit_t *circ, const uint8_t *cell_payload,
+                             uint16_t cell_payload_len)
+{
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* Validate the SENDME cell. Depending on the version, different validation
+   * can be done. An invalid SENDME requires us to close the circuit. */
+  if (!sendme_is_valid(circ, cell_payload, cell_payload_len)) {
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  /* If we are the origin of the circuit, we are the Client so we use the
+   * layer hint (the Exit hop) for the package window tracking. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* If we are the origin of the circuit, it is impossible to not have a
+     * cpath. Just in case, bug on it and close the circuit. */
+    if (BUG(layer_hint == NULL)) {
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    if ((layer_hint->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from exit relay. "
+                     "Closing circ.");
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    layer_hint->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.",
+              layer_hint->package_window);
+
+    /* We count circuit-level sendme's as valid delivered data because they
+     * are rate limited. */
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len);
+  } else {
+    /* We aren't the origin of this circuit so we are the Exit and thus we
+     * track the package window with the circuit object. */
+    if ((circ->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&client_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from client. "
+                     "Closing circ (window %d).", circ->package_window);
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    circ->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_EXIT, "circ-level sendme at non-origin, packagewindow %d.",
+              circ->package_window);
+  }
+
+  return 0;
+}
+
+/* Process a stream-level SENDME cell that we just received. The conn is the
+ * edge connection (stream) that the circuit circ is associated with. The
+ * cell_body_len is the length of the payload (excluding the header).
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                            uint16_t cell_body_len)
+{
+  tor_assert(conn);
+  tor_assert(circ);
+
+  /* Don't allow the other endpoint to request more than our maximum (i.e.
+   * initial) stream SENDME window worth of data. Well-behaved stock clients
+   * will not request more than this max (as per the check in the while loop
+   * of sendme_connection_edge_consider_sending()). */
+  if ((conn->package_window + STREAMWINDOW_INCREMENT) >
+      STREAMWINDOW_START_MAX) {
+    static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
+    log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                   "Unexpected stream sendme cell. Closing circ (window %d).",
+                   conn->package_window);
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+  /* At this point, the stream sendme is valid */
+  conn->package_window += STREAMWINDOW_INCREMENT;
+
+  /* We count circuit-level sendme's as valid delivered data because they are
+   * rate limited. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_body_len);
+  }
+
+  log_debug(CIRCUIT_IS_ORIGIN(circ) ? LD_APP : LD_EXIT,
+            "stream-level sendme, package_window now %d.",
+            conn->package_window);
+  return 0;
+}
+
+/* Called when a relay DATA cell is received on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the deliver window and return its new value. */
+int
+sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int deliver_window, domain;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    tor_assert(layer_hint);
+    --layer_hint->deliver_window;
+    deliver_window = layer_hint->deliver_window;
+    domain = LD_APP;
+  } else {
+    tor_assert(!layer_hint);
+    --circ->deliver_window;
+    deliver_window = circ->deliver_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit deliver_window now %d.", deliver_window);
+  return deliver_window;
+}
+
+/* Called when a relay DATA cell is received for the given edge connection
+ * conn. Update the deliver window and return its new value. */
+int
+sendme_stream_data_received(edge_connection_t *conn)
+{
+  tor_assert(conn);
+  return --conn->deliver_window;
+}
+
+/* Called when a relay DATA cell is packaged on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the package window and return its new value. */
+int
+sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int package_window, domain;
+
+  tor_assert(circ);
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* Client side. */
+    tor_assert(layer_hint);
+    --layer_hint->package_window;
+    package_window = layer_hint->package_window;
+    domain = LD_APP;
+  } else {
+    /* Exit side. */
+    tor_assert(!layer_hint);
+    --circ->package_window;
+    package_window = circ->package_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit package_window now %d.", package_window);
+  return package_window;
+}
+
+/* Called when a relay DATA cell is packaged for the given edge connection
+ * conn. Update the package window and return its new value. */
+int
+sendme_note_stream_data_packaged(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  --conn->package_window;
+  log_debug(LD_APP, "Stream package_window now %d.", conn->package_window);
+  return conn->package_window;
+}
+
+/* Record the cell digest into the circuit sendme digest list depending on
+ * which edge we are. The digest is recorded only if we expect the next cell
+ * that we will receive is a SENDME so we can match the digest. */
+void
+sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath)
+{
+  int package_window;
+  uint8_t *sendme_digest;
+
+  tor_assert(circ);
+
+  package_window = circ->package_window;
+  if (cpath) {
+    package_window = cpath->package_window;
+  }
+
+  /* Is this the last cell before a SENDME? The idea is that if the
+   * package_window reaches a multiple of the increment, after this cell, we
+   * should expect a SENDME. */
+  if (!circuit_sendme_cell_is_next(package_window)) {
+    return;
+  }
+
+  /* Getting the digest is expensive so we only do it once we are certain to
+   * record it on the circuit. */
+  if (cpath) {
+    sendme_digest = cpath_get_sendme_digest(cpath);
+  } else {
+    sendme_digest =
+      relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+  }
+
+  record_cell_digest_on_circ(circ, sendme_digest);
+}
+
+/* Called once we decrypted a cell and recognized it. Record the cell digest
+ * as the next sendme digest only if the next cell we'll send on the circuit
+ * is expected to be a SENDME. */
+void
+sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window :
+                                           circ->deliver_window)) {
+    return;
+  }
+
+  if (cpath) {
+    /* Record incoming digest. */
+    cpath_sendme_record_cell_digest(cpath, false);
+  } else {
+    /* Record foward digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, true);
+  }
+}
+
+/* Called once we encrypted a cell. Record the cell digest as the next sendme
+ * digest only if the next cell we expect to receive is a SENDME so we can
+ * match the digests. */
+void
+sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window :
+                                           circ->package_window)) {
+    goto end;
+  }
+
+  if (cpath) {
+    /* Record the forward digest. */
+    cpath_sendme_record_cell_digest(cpath, true);
+  } else {
+    /* Record the incoming digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, false);
+  }
+
+ end:
+  return;
+}
diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h
new file mode 100644
index 0000000000..9d757ee435
--- /dev/null
+++ b/src/core/or/sendme.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.h
+ * \brief Header file for sendme.c.
+ **/
+
+#ifndef TOR_SENDME_H
+#define TOR_SENDME_H
+
+#include "core/or/edge_connection_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+
+/* Sending SENDME cell. */
+void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn);
+void sendme_circuit_consider_sending(circuit_t *circ,
+                                     crypt_path_t *layer_hint);
+
+/* Processing SENDME cell. */
+int sendme_process_circuit_level(crypt_path_t *layer_hint,
+                                 circuit_t *circ, const uint8_t *cell_payload,
+                                 uint16_t cell_payload_len);
+int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                                uint16_t cell_body_len);
+
+/* Update deliver window functions. */
+int sendme_stream_data_received(edge_connection_t *conn);
+int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint);
+
+/* Update package window functions. */
+int sendme_note_circuit_data_packaged(circuit_t *circ,
+                                      crypt_path_t *layer_hint);
+int sendme_note_stream_data_packaged(edge_connection_t *conn);
+
+/* Record cell digest on circuit. */
+void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath);
+/* Record cell digest as the SENDME digest. */
+void sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+void sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+
+/* Private section starts. */
+#ifdef SENDME_PRIVATE
+
+/* The maximum supported version. Above that value, the cell can't be
+ * recognized as a valid SENDME. */
+#define SENDME_MAX_SUPPORTED_VERSION 1
+
+/* The cell version constants for when emitting a cell. */
+#define SENDME_EMIT_MIN_VERSION_DEFAULT 1
+#define SENDME_EMIT_MIN_VERSION_MIN 0
+#define SENDME_EMIT_MIN_VERSION_MAX UINT8_MAX
+
+/* The cell version constants for when accepting a cell. */
+#define SENDME_ACCEPT_MIN_VERSION_DEFAULT 0
+#define SENDME_ACCEPT_MIN_VERSION_MIN 0
+#define SENDME_ACCEPT_MIN_VERSION_MAX UINT8_MAX
+
+/*
+ * Unit tests declaractions.
+ */
+#ifdef TOR_UNIT_TESTS
+
+STATIC int get_emit_min_version(void);
+STATIC int get_accept_min_version(void);
+
+STATIC bool cell_version_can_be_handled(uint8_t cell_version);
+
+STATIC ssize_t build_cell_payload_v1(const uint8_t *cell_digest,
+                                     uint8_t *payload);
+STATIC bool sendme_is_valid(const circuit_t *circ,
+                            const uint8_t *cell_payload,
+                            size_t cell_payload_len);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(SENDME_PRIVATE) */
+
+#endif /* !defined(TOR_SENDME_H) */
diff --git a/src/core/or/server_port_cfg_st.h b/src/core/or/server_port_cfg_st.h
index bd026af7ee..0738735c61 100644
--- a/src/core/or/server_port_cfg_st.h
+++ b/src/core/or/server_port_cfg_st.h
@@ -16,5 +16,5 @@ struct server_port_cfg_t {
   unsigned int bind_ipv6_only : 1;
 };
 
-#endif
+#endif /* !defined(SERVER_PORT_CFG_ST_H) */
 
diff --git a/src/core/or/socks_request_st.h b/src/core/or/socks_request_st.h
index 5922870c61..9fb941ff7e 100644
--- a/src/core/or/socks_request_st.h
+++ b/src/core/or/socks_request_st.h
@@ -74,4 +74,4 @@ struct socks_request_t {
   uint8_t socks5_atyp; /* SOCKS5 address type */
 };
 
-#endif
+#endif /* !defined(SOCKS_REQUEST_ST_H) */
diff --git a/src/core/or/tor_version_st.h b/src/core/or/tor_version_st.h
index 716429bd32..c5bdcaf07b 100644
--- a/src/core/or/tor_version_st.h
+++ b/src/core/or/tor_version_st.h
@@ -28,5 +28,5 @@ struct tor_version_t {
   char git_tag[DIGEST_LEN];
 };
 
-#endif
+#endif /* !defined(TOR_VERSION_ST_H) */
 
diff --git a/src/core/or/var_cell_st.h b/src/core/or/var_cell_st.h
index 4287c83f6d..607c0d6c83 100644
--- a/src/core/or/var_cell_st.h
+++ b/src/core/or/var_cell_st.h
@@ -19,5 +19,5 @@ struct var_cell_t {
   uint8_t payload[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
+#endif /* !defined(VAR_CELL_ST_H) */
 
diff --git a/src/core/or/versions.c b/src/core/or/versions.c
index 33273a5294..2c32b529f7 100644
--- a/src/core/or/versions.c
+++ b/src/core/or/versions.c
@@ -16,6 +16,25 @@
 
 #include "core/or/tor_version_st.h"
 
+/**
+ * Return the approximate date when this release came out, or was
+ * scheduled to come out, according to the APPROX_RELEASE_DATE set in
+ * configure.ac
+ **/
+time_t
+tor_get_approx_release_date(void)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  tor_snprintf(tbuf, sizeof(tbuf),
+               "%s 00:00:00", APPROX_RELEASE_DATE);
+  time_t result = 0;
+  int r = parse_iso_time(tbuf, &result);
+  if (BUG(r < 0)) {
+    result = 0;
+  }
+  return result;
+}
+
 /** Return VS_RECOMMENDED if <b>myversion</b> is contained in
  * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
  * entries. Else, return VS_OLD if every member of
@@ -377,6 +396,69 @@ sort_version_list(smartlist_t *versions, int remove_duplicates)
     smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
 }
 
+/** If there are more than this many entries, we're probably under
+ * some kind of weird DoS. */
+static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024;
+
+/**
+ * Map from protover string to protover_summary_flags_t.
+ */
+static strmap_t *protover_summary_map = NULL;
+
+/**
+ * Helper.  Given a non-NULL protover string <b>protocols</b>, set <b>out</b>
+ * to its summary, and memoize the result in <b>protover_summary_map</b>.
+ */
+static void
+memoize_protover_summary(protover_summary_flags_t *out,
+                         const char *protocols)
+{
+  if (!protover_summary_map)
+    protover_summary_map = strmap_new();
+
+  if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) {
+    protover_summary_cache_free_all();
+    tor_assert(protover_summary_map == NULL);
+    protover_summary_map = strmap_new();
+  }
+
+  const protover_summary_flags_t *cached =
+    strmap_get(protover_summary_map, protocols);
+
+  if (cached != NULL) {
+    /* We found a cached entry; no need to parse this one. */
+    memcpy(out, cached, sizeof(protover_summary_flags_t));
+    tor_assert(out->protocols_known);
+    return;
+  }
+
+  memset(out, 0, sizeof(*out));
+  out->protocols_known = 1;
+  out->supports_extend2_cells =
+    protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+  out->supports_ed25519_link_handshake_compat =
+    protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_link_handshake_any =
+    protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_hs_intro =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+  out->supports_v3_hsdir =
+    protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                    PROTOVER_HSDIR_V3);
+  out->supports_v3_rendezvous_point =
+    protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                    PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  out->supports_hs_setup_padding =
+    protocol_list_supports_protocol(protocols, PRT_PADDING,
+                                    PROTOVER_HS_SETUP_PADDING);
+  out->supports_establish_intro_dos_extension =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 5);
+
+  protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out));
+  cached = strmap_set(protover_summary_map, protocols, new_cached);
+  tor_assert(!cached);
+}
+
 /** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
  * falling back or correcting them based on <b>version</b> as appropriate.
  */
@@ -388,21 +470,7 @@ summarize_protover_flags(protover_summary_flags_t *out,
   tor_assert(out);
   memset(out, 0, sizeof(*out));
   if (protocols) {
-    out->protocols_known = 1;
-    out->supports_extend2_cells =
-      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
-    out->supports_ed25519_link_handshake_compat =
-      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_link_handshake_any =
-      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_hs_intro =
-      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
-    out->supports_v3_hsdir =
-      protocol_list_supports_protocol(protocols, PRT_HSDIR,
-                                      PROTOVER_HSDIR_V3);
-    out->supports_v3_rendezvous_point =
-      protocol_list_supports_protocol(protocols, PRT_HSREND,
-                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+    memoize_protover_summary(out, protocols);
   }
   if (version && !strcmpstart(version, "Tor ")) {
     if (!out->protocols_known) {
@@ -420,3 +488,13 @@ summarize_protover_flags(protover_summary_flags_t *out,
     }
   }
 }
+
+/**
+ * Free all space held in the protover_summary_map.
+ */
+void
+protover_summary_cache_free_all(void)
+{
+  strmap_free(protover_summary_map, tor_free_);
+  protover_summary_map = NULL;
+}
diff --git a/src/core/or/versions.h b/src/core/or/versions.h
index 22f3be176f..9aa7a0db87 100644
--- a/src/core/or/versions.h
+++ b/src/core/or/versions.h
@@ -26,6 +26,8 @@ typedef enum version_status_t {
   VS_UNKNOWN, /**< We have no idea. */
 } version_status_t;
 
+time_t tor_get_approx_release_date(void);
+
 version_status_t tor_version_is_obsolete(const char *myversion,
                                          const char *versionlist);
 int tor_version_parse_platform(const char *platform,
@@ -41,4 +43,6 @@ void summarize_protover_flags(protover_summary_flags_t *out,
                               const char *protocols,
                               const char *version);
 
+void protover_summary_cache_free_all(void);
+
 #endif /* !defined(TOR_VERSIONS_H) */
diff --git a/src/core/proto/.may_include b/src/core/proto/.may_include
new file mode 100644
index 0000000000..c1647a5cf9
--- /dev/null
+++ b/src/core/proto/.may_include
@@ -0,0 +1,10 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/buf/*.h
+
+trunnel/*.h
+
+core/proto/*.h
\ No newline at end of file
diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c
index 0442e2c6ee..697fed29e1 100644
--- a/src/core/proto/proto_cell.c
+++ b/src/core/proto/proto_cell.c
@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_cell.h"
 
 #include "core/or/connection_or.h"
diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c
index 21fa328f02..d741f28f09 100644
--- a/src/core/proto/proto_control0.c
+++ b/src/core/proto/proto_control0.c
@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_control0.h"
 
 /** Return 1 iff buf looks more like it has an (obsolete) v0 controller
diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c
index edbc51b10c..4213bc14dd 100644
--- a/src/core/proto/proto_ext_or.c
+++ b/src/core/proto/proto_ext_or.c
@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_ext_or.h"
 
diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c
index 5c86fc4979..88c59ef561 100644
--- a/src/core/proto/proto_http.c
+++ b/src/core/proto/proto_http.c
@@ -6,7 +6,7 @@
 
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 
 /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c
index c7bf13b9f4..c06e6b1a41 100644
--- a/src/core/proto/proto_socks.c
+++ b/src/core/proto/proto_socks.c
@@ -6,9 +6,9 @@
 
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/relay/ext_orport.h"
diff --git a/src/ext/.may_include b/src/ext/.may_include
new file mode 100644
index 0000000000..1eafff2eeb
--- /dev/null
+++ b/src/ext/.may_include
@@ -0,0 +1,10 @@
+
+orconfig.h
+
+lib/err/*.h
+lib/cc/*.h
+
+tinytest*.h
+ext/siphash.h
+ext/byteorder.h
+ext/tor_readpassphrase.h
\ No newline at end of file
diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c
index a65b6fcbe6..e0f5b2e5c9 100644
--- a/src/ext/csiphash.c
+++ b/src/ext/csiphash.c
@@ -30,12 +30,12 @@
 */
 
 #include "lib/cc/torint.h"
-#include "lib/log/util_bug.h"
+#include "lib/err/torerr.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 #include <string.h>
 #include <stdlib.h>
-#include "byteorder.h"
+#include "ext/byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
@@ -87,6 +87,13 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		v0 ^= mi;
 	}
 
+#ifdef __COVERITY__
+	{
+		uint64_t mi = 0;
+		memcpy(&mi, m+i, (src_sz-blocks));
+		last7 = _le64toh(mi) | (uint64_t)(src_sz & 0xff) << 56;
+	}
+#else
 	switch (src_sz - blocks) {
 		case 7: last7 |= (uint64_t)m[i + 6] << 48; FALLTHROUGH;
 		case 6: last7 |= (uint64_t)m[i + 5] << 40; FALLTHROUGH;
@@ -98,6 +105,7 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		case 0:
 		default:;
 	}
+#endif
 	v3 ^= last7;
 	DOUBLE_ROUND(v0,v1,v2,v3);
 	v0 ^= last7;
@@ -112,13 +120,13 @@ static int the_siphash_key_is_set = 0;
 static struct sipkey the_siphash_key;
 
 uint64_t siphash24g(const void *src, unsigned long src_sz) {
-	tor_assert(the_siphash_key_is_set);
+	raw_assert(the_siphash_key_is_set);
 	return siphash24(src, src_sz, &the_siphash_key);
 }
 
 void siphash_set_global_key(const struct sipkey *key)
 {
-	tor_assert(! the_siphash_key_is_set);
+	raw_assert(! the_siphash_key_is_set);
 	the_siphash_key.k0 = key->k0;
 	the_siphash_key.k1 = key->k1;
 	the_siphash_key_is_set = 1;
diff --git a/src/ext/include.am b/src/ext/include.am
index 6bdce2d79e..317e25d78e 100644
--- a/src/ext/include.am
+++ b/src/ext/include.am
@@ -143,6 +143,7 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS)
 LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a
 noinst_LIBRARIES += $(LIBED25519_DONNA)
 
+if BUILD_KECCAK_TINY
 src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\
   @CFLAGS_CONSTTIME@
 
@@ -156,6 +157,7 @@ noinst_HEADERS += $(LIBKECCAK_TINY_HDRS)
 
 LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a
 noinst_LIBRARIES += $(LIBKECCAK_TINY)
+endif
 
 EXTRA_DIST += \
 	src/ext/timeouts/bench/bench-add.lua 		\
diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c
index e0df05d7b7..16611af1e2 100644
--- a/src/ext/readpassphrase.c
+++ b/src/ext/readpassphrase.c
@@ -30,7 +30,7 @@
 #include <signal.h>
 #include <ctype.h>
 #include <fcntl.h>
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include
new file mode 100644
index 0000000000..92c7116555
--- /dev/null
+++ b/src/ext/timeouts/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+
+ext/tor_queue.h
+ext/timeouts/*.h
+ext/timeouts/timeout-bitops.c
diff --git a/src/ext/timeouts/test-timeout.c b/src/ext/timeouts/test-timeout.c
index 8077129376..52d2e31e0c 100644
--- a/src/ext/timeouts/test-timeout.c
+++ b/src/ext/timeouts/test-timeout.c
@@ -4,7 +4,7 @@
 #include <assert.h>
 #include <limits.h>
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #define THE_END_OF_TIME ((timeout_t)-1)
 
diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c
index d4b514d2c5..79fcc168ed 100644
--- a/src/ext/timeouts/timeout.c
+++ b/src/ext/timeouts/timeout.c
@@ -38,16 +38,16 @@
 
 #include <errno.h>     /* errno */
 
-#include "tor_queue.h" /* TAILQ(3) */
+#include "ext/tor_queue.h" /* TAILQ(3) */
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #ifndef TIMEOUT_DEBUG
 #define TIMEOUT_DEBUG 0
 #endif
 
 #if TIMEOUT_DEBUG - 0
-#include "timeout-debug.h"
+#include "ext/timeouts/timeout-debug.h"
 #endif
 
 #ifdef TIMEOUT_DISABLE_RELATIVE_ACCESS
@@ -141,7 +141,7 @@
 #define WHEEL_MASK (WHEEL_LEN - 1)
 #define TIMEOUT_MAX ((TIMEOUT_C(1) << (WHEEL_BIT * WHEEL_NUM)) - 1)
 
-#include "timeout-bitops.c"
+#include "ext/timeouts/timeout-bitops.c"
 
 #if WHEEL_BIT == 6
 #define ctz(n) ctz64(n)
@@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) {
 			timeout = MIN(_timeout, timeout);
 		}
 
-		relmask <<= WHEEL_BIT; 
+		relmask <<= WHEEL_BIT;
 		relmask |= WHEEL_MASK;
 	}
 
@@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) {
 TIMEOUT_PUBLIC int timeout_v_api(void) {
 	return TIMEOUT_V_API;
 } /* timeout_version() */
-
diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h
index b35874e153..1ed309fd08 100644
--- a/src/ext/timeouts/timeout.h
+++ b/src/ext/timeouts/timeout.h
@@ -31,7 +31,7 @@
 
 #include <inttypes.h>   /* PRIu64 PRIx64 PRIX64 uint64_t */
 
-#include "tor_queue.h"  /* TAILQ(3) */
+#include "ext/tor_queue.h"  /* TAILQ(3) */
 
 
 /*
@@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int);
 #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS
 TIMEOUT_PUBLIC bool timeout_pending(struct timeout *);
 /* true if on timing wheel, false otherwise */
- 
+
 TIMEOUT_PUBLIC bool timeout_expired(struct timeout *);
 /* true if on expired queue, false otherwise */
 
diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c
index 16f11e4639..239fdd0a38 100644
--- a/src/ext/tinytest.c
+++ b/src/ext/tinytest.c
@@ -492,6 +492,12 @@ tinytest_set_test_skipped_(void)
 		cur_test_outcome = SKIP;
 }
 
+int
+tinytest_cur_test_has_failed(void)
+{
+	return (cur_test_outcome == FAIL);
+}
+
 char *
 tinytest_format_hex_(const void *val_, unsigned long len)
 {
diff --git a/src/ext/tinytest.h b/src/ext/tinytest.h
index ed07b26bc0..05c2fda052 100644
--- a/src/ext/tinytest.h
+++ b/src/ext/tinytest.h
@@ -72,6 +72,9 @@ struct testlist_alias_t {
 };
 #define END_OF_ALIASES { NULL, NULL }
 
+/** Return true iff the current test has failed. */
+int tinytest_cur_test_has_failed(void);
+
 /** Implementation: called from a test to indicate failure, before logging. */
 void tinytest_set_test_failed_(void);
 /** Implementation: called from a test to indicate that we're skipping. */
diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h
index 15d1c8633e..52afa9ccd4 100644
--- a/src/ext/trunnel/trunnel-impl.h
+++ b/src/ext/trunnel/trunnel-impl.h
@@ -1,4 +1,4 @@
-/* trunnel-impl.h -- copied from Trunnel v1.5.2
+/* trunnel-impl.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c
index 3ae3fe02c8..01a55c5bec 100644
--- a/src/ext/trunnel/trunnel.c
+++ b/src/ext/trunnel/trunnel.c
@@ -1,4 +1,4 @@
-/* trunnel.c -- copied from Trunnel v1.5.2
+/* trunnel.c -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h
index 9b708437b8..87c75f4ec3 100644
--- a/src/ext/trunnel/trunnel.h
+++ b/src/ext/trunnel/trunnel.h
@@ -1,4 +1,4 @@
-/* trunnel.h -- copied from Trunnel v1.5.2
+/* trunnel.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c
index 697397d46b..e270c51ac9 100644
--- a/src/feature/api/tor_api.c
+++ b/src/feature/api/tor_api.c
@@ -40,10 +40,10 @@
 #define raw_socketpair tor_ersatz_socketpair
 #define raw_closesocket closesocket
 #define snprintf _snprintf
-#else
+#else /* !defined(_WIN32) */
 #define raw_socketpair socketpair
 #define raw_closesocket close
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h
index 2bf130c376..cb84853a52 100644
--- a/src/feature/api/tor_api.h
+++ b/src/feature/api/tor_api.h
@@ -55,7 +55,7 @@ typedef SOCKET tor_control_socket_t;
 #else
 typedef int tor_control_socket_t;
 #define INVALID_TOR_CONTROL_SOCKET (-1)
-#endif
+#endif /* defined(_WIN32) */
 
 /** DOCDOC */
 tor_control_socket_t tor_main_configuration_setup_control_socket(
diff --git a/src/feature/client/addressmap.c b/src/feature/client/addressmap.c
index bbe786a6a2..c5a27ce8c6 100644
--- a/src/feature/client/addressmap.c
+++ b/src/feature/client/addressmap.c
@@ -22,7 +22,7 @@
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/relay/dns.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
index 626c5efcae..f517876609 100644
--- a/src/feature/client/bridges.c
+++ b/src/feature/client/bridges.c
@@ -348,7 +348,7 @@ int
 node_is_a_configured_bridge(const node_t *node)
 {
   /* First, let's try searching for a bridge with matching identity. */
-  if (BUG(tor_digest_is_zero(node->identity)))
+  if (BUG(fast_mem_is_zero(node->identity, DIGEST_LEN)))
     return 0;
 
   if (find_bridge_by_digest(node->identity) != NULL)
@@ -844,7 +844,8 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
       }
     }
 
-    if (options->ClientPreferIPv6ORPort == -1) {
+    if (options->ClientPreferIPv6ORPort == -1 ||
+        options->ClientAutoIPv6ORPort == 0) {
       /* Mark which address to use based on which bridge_t we got. */
       node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
                               !tor_addr_is_null(&node->ri->ipv6_addr));
diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c
index 1743ab5a81..60a52664f1 100644
--- a/src/feature/client/circpathbias.c
+++ b/src/feature/client/circpathbias.c
@@ -176,6 +176,7 @@ pathbias_get_scale_threshold(const or_options_t *options)
 static double
 pathbias_get_scale_ratio(const or_options_t *options)
 {
+  (void) options;
   /*
    * The scale factor is the denominator for our scaling
    * of circuit counts for our path bias window.
@@ -185,7 +186,8 @@ pathbias_get_scale_ratio(const or_options_t *options)
    */
   int denominator = networkstatus_get_param(NULL, "pb_scalefactor",
                               2, 2, INT32_MAX);
-  (void) options;
+  tor_assert(denominator > 0);
+
   /**
    * The mult factor is the numerator for our scaling
    * of circuit counts for our path bias window. It
@@ -301,7 +303,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
   return circ->cpath &&
          circ->cpath->next != circ->cpath &&
          circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS;
-#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */
+#else /* !defined(N2N_TAGGING_IS_POSSIBLE) */
   /* If tagging attacks are no longer possible, we probably want to
    * count bias from the first hop. However, one could argue that
    * timing-based tagging is still more useful than per-hop failure.
@@ -369,8 +371,9 @@ pathbias_should_count(origin_circuit_t *circ)
         !circ->build_state->onehop_tunnel) {
       if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) {
         log_info(LD_BUG,
-               "One-hop circuit has length %d. Path state is %s. "
+               "One-hop circuit %d has length %d. Path state is %s. "
                "Circuit is a %s currently %s.%s",
+               circ->global_identifier,
                circ->build_state->desired_path_len,
                pathbias_state_to_string(circ->path_state),
                circuit_purpose_to_string(circ->base_.purpose),
@@ -398,12 +401,13 @@ pathbias_should_count(origin_circuit_t *circ)
   /* Check to see if the shouldcount result has changed due to a
    * unexpected purpose change that would affect our results */
   if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) {
-      log_info(LD_BUG,
-              "Circuit %d is now being counted despite being ignored "
-              "in the past. Purpose is %s, path state is %s",
-              circ->global_identifier,
-              circuit_purpose_to_string(circ->base_.purpose),
-              pathbias_state_to_string(circ->path_state));
+    log_info(LD_CIRC,
+            "Circuit %d is not being counted by pathbias because it was "
+            "ignored in the past. Purpose is %s, path state is %s",
+            circ->global_identifier,
+            circuit_purpose_to_string(circ->base_.purpose),
+            pathbias_state_to_string(circ->path_state));
+    return 0;
   }
   circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED;
 
@@ -434,8 +438,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                                      approx_time()))) {
         log_info(LD_BUG,
-                "Opened circuit is in strange path state %s. "
+                "Opened circuit %d is in strange path state %s. "
                 "Circuit is a %s currently %s.%s",
+                circ->global_identifier,
                 pathbias_state_to_string(circ->path_state),
                 circuit_purpose_to_string(circ->base_.purpose),
                 circuit_state_to_string(circ->base_.state),
@@ -468,8 +473,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
           if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                   approx_time()))) {
             log_info(LD_BUG,
-                   "Unopened circuit has strange path state %s. "
+                   "Unopened circuit %d has strange path state %s. "
                    "Circuit is a %s currently %s.%s",
+                   circ->global_identifier,
                    pathbias_state_to_string(circ->path_state),
                    circuit_purpose_to_string(circ->base_.purpose),
                    circuit_state_to_string(circ->base_.state),
@@ -538,8 +544,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
         if ((rate_msg = rate_limit_log(&success_notice_limit,
                 approx_time()))) {
           log_info(LD_BUG,
-              "Succeeded circuit is in strange path state %s. "
+              "Succeeded circuit %d is in strange path state %s. "
               "Circuit is a %s currently %s.%s",
+              circ->global_identifier,
               pathbias_state_to_string(circ->path_state),
               circuit_purpose_to_string(circ->base_.purpose),
               circuit_state_to_string(circ->base_.state),
@@ -574,8 +581,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&success_notice_limit,
               approx_time()))) {
         log_info(LD_BUG,
-            "Opened circuit is in strange path state %s. "
+            "Opened circuit %d is in strange path state %s. "
             "Circuit is a %s currently %s.%s",
+            circ->global_identifier,
             pathbias_state_to_string(circ->path_state),
             circuit_purpose_to_string(circ->base_.purpose),
             circuit_state_to_string(circ->base_.state),
@@ -601,8 +609,9 @@ pathbias_count_use_attempt(origin_circuit_t *circ)
 
   if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) {
     log_notice(LD_BUG,
-        "Used circuit is in strange path state %s. "
+        "Used circuit %d is in strange path state %s. "
         "Circuit is a %s currently %s.",
+        circ->global_identifier,
         pathbias_state_to_string(circ->path_state),
         circuit_purpose_to_string(circ->base_.purpose),
         circuit_state_to_string(circ->base_.state));
diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c
index cf593cfb68..f9e436e88f 100644
--- a/src/feature/client/dnsserv.c
+++ b/src/feature/client/dnsserv.c
@@ -26,8 +26,9 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/policies.h"
 
 #include "feature/control/control_connection_st.h"
@@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse,
   edge_connection_t *conn;
   char *q_name;
 
+  /* Launching a request for a user counts as user activity. */
+  note_user_activity(approx_time());
+
   /* Make a new dummy AP connection, and attach the request to it. */
   entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
   entry_conn->entry_cfg.dns_request = 1;
@@ -234,7 +238,7 @@ dnsserv_launch_request(const char *name, int reverse,
     TO_CONN(conn)->port = control_conn->base_.port;
     TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
   }
-#else /* !(defined(AF_UNIX)) */
+#else /* !defined(AF_UNIX) */
   TO_CONN(conn)->port = control_conn->base_.port;
   TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
 #endif /* defined(AF_UNIX) */
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
index 8d9230b66b..115d871843 100644
--- a/src/feature/client/entrynodes.c
+++ b/src/feature/client/entrynodes.c
@@ -114,7 +114,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "app/config/statefile.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
@@ -128,7 +128,7 @@
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c
index 0d1342c87b..3f731ac7d4 100644
--- a/src/feature/client/transports.c
+++ b/src/feature/client/transports.c
@@ -100,12 +100,14 @@
 #include "app/config/statefile.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
+#include "lib/process/process.h"
 #include "lib/process/env.h"
-#include "lib/process/subprocess.h"
 
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp);
 
 static inline int proxy_configuration_finished(const managed_proxy_t *mp);
@@ -127,6 +129,8 @@ static void parse_method_error(const char *line, int is_server_method);
 #define PROTO_SMETHODS_DONE "SMETHODS DONE"
 #define PROTO_PROXY_DONE "PROXY DONE"
 #define PROTO_PROXY_ERROR "PROXY-ERROR"
+#define PROTO_LOG "LOG"
+#define PROTO_STATUS "STATUS"
 
 /** The first and only supported - at the moment - configuration
     protocol version. */
@@ -490,8 +494,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
   tor_assert(mp->conf_state == PT_PROTO_COMPLETED);
 
   /* destroy the process handle and terminate the process. */
-  tor_process_handle_destroy(mp->process_handle, 1);
-  mp->process_handle = NULL;
+  process_set_data(mp->process, NULL);
+  process_terminate(mp->process);
 
   /* destroy all its registered transports, since we will no longer
      use them. */
@@ -520,34 +524,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
 static int
 launch_managed_proxy(managed_proxy_t *mp)
 {
-  int retval;
-
-  process_environment_t *env = create_managed_proxy_environment(mp);
-
-#ifdef _WIN32
-  /* Passing NULL as lpApplicationName makes Windows search for the .exe */
-  retval = tor_spawn_background(NULL,
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#else /* !(defined(_WIN32)) */
-  retval = tor_spawn_background(mp->argv[0],
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#endif /* defined(_WIN32) */
-
-  process_environment_free(env);
-
-  if (retval == PROCESS_STATUS_ERROR) {
-    log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.",
+  tor_assert(mp);
+
+  smartlist_t *env = create_managed_proxy_environment(mp);
+
+  /* Configure our process. */
+  process_set_data(mp->process, mp);
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback);
+  process_set_exit_callback(mp->process, managed_proxy_exit_callback);
+  process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE);
+  process_reset_environment(mp->process, env);
+
+  /* Cleanup our env. */
+  SMARTLIST_FOREACH(env, char *, x, tor_free(x));
+  smartlist_free(env);
+
+  /* Skip the argv[0] as we get that from process_new(argv[0]). */
+  for (int i = 1; mp->argv[i] != NULL; ++i)
+    process_append_argument(mp->process, mp->argv[i]);
+
+  if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.",
              mp->argv[0]);
     return -1;
   }
 
-  log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.",
-           mp->argv[0], tor_process_get_pid(mp->process_handle));
-
+  log_info(LD_CONFIG,
+           "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.",
+           mp->argv[0], process_get_pid(mp->process));
   mp->conf_state = PT_PROTO_LAUNCHED;
 
   return 0;
@@ -615,10 +620,6 @@ pt_configure_remaining_proxies(void)
 STATIC int
 configure_proxy(managed_proxy_t *mp)
 {
-  int configuration_finished = 0;
-  smartlist_t *proxy_output = NULL;
-  enum stream_status stream_status = 0;
-
   /* if we haven't launched the proxy yet, do it now */
   if (mp->conf_state == PT_PROTO_INFANT) {
     if (launch_managed_proxy(mp) < 0) { /* launch fail */
@@ -629,45 +630,8 @@ configure_proxy(managed_proxy_t *mp)
   }
 
   tor_assert(mp->conf_state != PT_PROTO_INFANT);
-  tor_assert(mp->process_handle);
-
-  proxy_output =
-    tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle),
-                              &stream_status);
-  if (!proxy_output) { /* failed to get input from proxy */
-    if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */
-      mp->conf_state = PT_PROTO_BROKEN;
-      log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' "
-               "is '%s'. Most probably the managed proxy stopped running. "
-               "This might be a bug of the managed proxy, a bug of Tor, or "
-               "a misconfiguration. Please enable logging on your managed "
-               "proxy and check the logs for errors.",
-               mp->argv[0], stream_status_to_string(stream_status));
-    }
-
-    goto done;
-  }
-
-  /* Handle lines. */
-  SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) {
-    handle_proxy_line(line, mp);
-    if (proxy_configuration_finished(mp))
-      goto done;
-  } SMARTLIST_FOREACH_END(line);
-
- done:
-  /* if the proxy finished configuring, exit the loop. */
-  if (proxy_configuration_finished(mp)) {
-    handle_finished_proxy(mp);
-    configuration_finished = 1;
-  }
-
-  if (proxy_output) {
-    SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp));
-    smartlist_free(proxy_output);
-  }
-
-  return configuration_finished;
+  tor_assert(mp->process);
+  return mp->conf_state == PT_PROTO_COMPLETED;
 }
 
 /** Register server managed proxy <b>mp</b> transports to state */
@@ -748,8 +712,14 @@ managed_proxy_destroy(managed_proxy_t *mp,
   /* free the outgoing proxy URI */
   tor_free(mp->proxy_uri);
 
-  tor_process_handle_destroy(mp->process_handle, also_terminate_process);
-  mp->process_handle = NULL;
+  /* do we want to terminate our process if it's still running? */
+  if (also_terminate_process && mp->process) {
+    /* Note that we do not call process_free(mp->process) here because we let
+     * the exit handler in managed_proxy_exit_callback() return `true` which
+     * makes the process subsystem deallocate the process_t. */
+    process_set_data(mp->process, NULL);
+    process_terminate(mp->process);
+  }
 
   tor_free(mp);
 }
@@ -945,21 +915,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp)
 
     parse_proxy_error(line);
     goto err;
-  } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) {
-    /* managed proxy launch failed: parse error message to learn why. */
-    int retval, child_state, saved_errno;
-    retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x",
-                        &child_state, &saved_errno);
-    if (retval == 2) {
-      log_warn(LD_GENERAL,
-               "Could not launch managed proxy executable at '%s' ('%s').",
-               mp->argv[0], strerror(saved_errno));
-    } else { /* failed to parse error message */
-      log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.",
-               mp->argv[0]);
-    }
 
-    mp->conf_state = PT_PROTO_FAILED_LAUNCH;
+    /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS
+     * string to make sure we can later extend this big if/else-if table with
+     * something that begins with "LOG" without having to get the order right.
+     * */
+  } else if (!strcmpstart(line, PROTO_LOG " ")) {
+    parse_log_line(line, mp);
+    return;
+  } else if (!strcmpstart(line, PROTO_STATUS " ")) {
+    parse_status_line(line, mp);
     return;
   }
 
@@ -1182,6 +1147,121 @@ parse_proxy_error(const char *line)
            line+strlen(PROTO_PROXY_ERROR)+1);
 }
 
+/** Parses a LOG <b>line</b> and emit log events accordingly. */
+STATIC void
+parse_log_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *log_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_LOG) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_LOG);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_LOG) + 1;
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s",
+             mp->argv[0], data);
+    goto done;
+  }
+
+  const config_line_t *severity = config_line_find(values, "SEVERITY");
+  const config_line_t *message = config_line_find(values, "MESSAGE");
+
+  /* Check if we got a message. */
+  if (! message) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "MESSAGE: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Check if severity is there and whether it's valid. */
+  if (! severity) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "SEVERITY: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  int log_severity = managed_proxy_severity_parse(severity->value);
+
+  if (log_severity == -1) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an "
+                    "invalid severity level: %s",
+                    mp->argv[0], severity->value);
+    goto done;
+  }
+
+  tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s",
+          mp->argv[0], message->value);
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  log_message = kvline_encode(values, KV_QUOTED);
+
+  /* Emit control port event. */
+  control_event_pt_log(log_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(log_message);
+}
+
+/** Parses a STATUS <b>line</b> and emit control events accordingly. */
+STATIC void
+parse_status_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *status_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_STATUS) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_STATUS);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_STATUS) + 1;
+
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid "
+             "STATUS message: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* We check if we received the TRANSPORT parameter, which is the only
+   * *required* value. */
+  const config_line_t *type = config_line_find(values, "TRANSPORT");
+
+  if (! type) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without "
+                    "TRANSPORT: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  status_message = kvline_encode(values, KV_QUOTED);
+
+  /* We have checked that TRANSPORT is there, we can now emit the STATUS event
+   * via the control port. */
+  control_event_pt_status(status_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(status_message);
+}
+
 /** Return a newly allocated string that tor should place in
  * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server
  * manged proxy in <b>mp</b>. Return NULL if no such options are found. */
@@ -1257,7 +1337,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp)
 
 /** Return a newly allocated process_environment_t * for <b>mp</b>'s
  * process. */
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp)
 {
   const or_options_t *options = get_options();
@@ -1272,8 +1352,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
   /* The final environment to be passed to mp. */
   smartlist_t *merged_env_vars = get_current_process_environment_variables();
 
-  process_environment_t *env;
-
   {
     char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */
     smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp);
@@ -1346,11 +1424,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     } else {
       smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=");
     }
-
-    /* All new versions of tor will keep stdin open, so PTs can use it
-     * as a reliable termination detection mechanism.
-     */
-    smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
   } else {
     /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the
      * TOR_PT_PROXY line.
@@ -1361,19 +1434,19 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     }
   }
 
+  /* All new versions of tor will keep stdin open, so PTs can use it
+   * as a reliable termination detection mechanism.
+   */
+  smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
+
   SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) {
     set_environment_variable_in_smartlist(merged_env_vars, env_var,
                                           tor_free_, 1);
   } SMARTLIST_FOREACH_END(env_var);
 
-  env = process_environment_make(merged_env_vars);
-
   smartlist_free(envs);
 
-  SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x));
-  smartlist_free(merged_env_vars);
-
-  return env;
+  return merged_env_vars;
 }
 
 /** Create and return a new managed proxy for <b>transport</b> using
@@ -1392,6 +1465,7 @@ managed_proxy_create(const smartlist_t *with_transport_list,
   mp->argv = proxy_argv;
   mp->transports = smartlist_new();
   mp->proxy_uri = get_pt_proxy_uri();
+  mp->process = process_new(proxy_argv[0]);
 
   mp->transports_to_launch = smartlist_new();
   SMARTLIST_FOREACH(with_transport_list, const char *, transport,
@@ -1736,3 +1810,93 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape)
 
   return new_string;
 }
+
+/** Callback function that is called when our PT process have data on its
+ * stdout. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stdout_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (mp == NULL)
+    return;
+
+  handle_proxy_line(line, mp);
+
+  if (proxy_configuration_finished(mp))
+    handle_finished_proxy(mp);
+}
+
+/** Callback function that is called when our PT process have data on its
+ * stderr. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stderr_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (BUG(mp == NULL))
+    return;
+
+  log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line);
+}
+
+/** Callback function that is called when our PT process terminates. The
+ * process exit code can be found in <b>exit_code</b> and our process can be
+ * found in <b>process</b>. Returns true iff we want the process subsystem to
+ * free our process_t handle for us. */
+STATIC bool
+managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_warn(LD_PT,
+          "Pluggable Transport process terminated with status code %" PRIu64,
+          exit_code);
+
+  /* Returning true here means that the process subsystem will take care of
+   * calling process_free() on our process_t. */
+  return true;
+}
+
+/** Returns a valid integer log severity level from <b>severity</b> that
+ * is compatible with Tor's logging functions. Returns <b>-1</b> on
+ * error. */
+STATIC int
+managed_proxy_severity_parse(const char *severity)
+{
+  tor_assert(severity);
+
+  /* Slightly different than log.c's parse_log_level :-( */
+  if (! strcmp(severity, "debug"))
+    return LOG_DEBUG;
+
+  if (! strcmp(severity, "info"))
+    return LOG_INFO;
+
+  if (! strcmp(severity, "notice"))
+    return LOG_NOTICE;
+
+  if (! strcmp(severity, "warning"))
+    return LOG_WARN;
+
+  if (! strcmp(severity, "error"))
+    return LOG_ERR;
+
+  return -1;
+}
diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h
index e2fa45828f..900dd9288e 100644
--- a/src/feature/client/transports.h
+++ b/src/feature/client/transports.h
@@ -11,6 +11,8 @@
 #ifndef TOR_TRANSPORTS_H
 #define TOR_TRANSPORTS_H
 
+#include "lib/process/process.h"
+
 /** Represents a pluggable transport used by a bridge. */
 typedef struct transport_t {
   /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */
@@ -81,7 +83,7 @@ enum pt_proto_state {
   PT_PROTO_FAILED_LAUNCH /* failed while launching */
 };
 
-struct process_handle_t;
+struct process_t;
 
 /** Structure containing information of a managed proxy. */
 typedef struct {
@@ -94,10 +96,8 @@ typedef struct {
 
   int is_server; /* is it a server proxy? */
 
-  /* A pointer to the process handle of this managed proxy. */
-  struct process_handle_t *process_handle;
-
-  int pid; /* The Process ID this managed proxy is using. */
+  /* A pointer to the process of this managed proxy. */
+  struct process_t *process;
 
   /** Boolean: We are re-parsing our config, and we are going to
    * remove this managed proxy if we don't find it any transport
@@ -128,6 +128,8 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp);
 STATIC void parse_env_error(const char *line);
 STATIC void parse_proxy_error(const char *line);
 STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_log_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_status_line(const char *line, managed_proxy_t *mp);
 STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp);
 
 STATIC void managed_proxy_destroy(managed_proxy_t *mp,
@@ -142,6 +144,12 @@ STATIC char* get_pt_proxy_uri(void);
 
 STATIC void free_execve_args(char **arg);
 
+STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t);
+STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t);
+STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t);
+
+STATIC int managed_proxy_severity_parse(const char *);
+
 #endif /* defined(PT_PRIVATE) */
 
 #endif /* !defined(TOR_TRANSPORTS_H) */
diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c
new file mode 100644
index 0000000000..3ce97dc855
--- /dev/null
+++ b/src/feature/control/btrack.c
@@ -0,0 +1,64 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack.c
+ * \brief Bootstrap trackers
+ *
+ * Initializes and shuts down the specific bootstrap trackers.  These
+ * trackers help the reporting of bootstrap progress by maintaining
+ * state information about various subsystems within tor.  When the
+ * correct state changes happen, these trackers emit controller
+ * events.
+ *
+ * These trackers avoid referring directly to the internals of state
+ * objects of other subsystems.
+ *
+ * btrack_circuit.c contains the tracker for origin circuits.
+ *
+ * btrack_orconn.c contains the tracker for OR connections.
+ *
+ * Eventually there will be a tracker for directory downloads as well.
+ **/
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_sys.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+static int
+btrack_init(void)
+{
+  if (btrack_orconn_init())
+    return -1;
+
+  return 0;
+}
+
+static void
+btrack_fini(void)
+{
+  btrack_orconn_fini();
+  btrack_circ_fini();
+}
+
+static int
+btrack_add_pubsub(pubsub_connector_t *connector)
+{
+  if (btrack_orconn_add_pubsub(connector))
+    return -1;
+  if (btrack_circ_add_pubsub(connector))
+    return -1;
+
+  return 0;
+}
+
+const subsys_fns_t sys_btrack = {
+  .name = "btrack",
+  .supported = true,
+  .level = -30,
+  .initialize = btrack_init,
+  .shutdown = btrack_fini,
+  .add_pubsub = btrack_add_pubsub,
+};
diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c
new file mode 100644
index 0000000000..2980c77ddc
--- /dev/null
+++ b/src/feature/control/btrack_circuit.c
@@ -0,0 +1,166 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.c
+ * \brief Bootstrap tracker for origin circuits
+ *
+ * Track state changes of origin circuits, as published by the circuit
+ * subsystem.
+ **/
+
+#include "core/or/or.h"
+
+#include "core/or/ocirc_event.h"
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/control.h"
+#include "lib/log/log.h"
+
+/** Pair of a best origin circuit GID with its state or status */
+typedef struct btc_best_t {
+  uint32_t gid;
+  int val;
+} btc_best_t;
+
+/** GID and state of the best origin circuit we've seen so far */
+static btc_best_t best_any_state = { 0, -1 };
+/** GID and state of the best application circuit we've seen so far */
+static btc_best_t best_ap_state = { 0, -1 };
+/** GID and status of the best origin circuit we've seen so far */
+static btc_best_t best_any_evtype = { 0, -1 };
+/** GID and status of the best application circuit we've seen so far */
+static btc_best_t best_ap_evtype = { 0, -1 };
+
+/** Reset cached "best" values */
+static void
+btc_reset_bests(void)
+{
+  best_any_state.gid = best_ap_state.gid = 0;
+  best_any_state.val = best_ap_state.val = -1;
+  best_any_evtype.gid = best_ap_state.gid = 0;
+  best_any_evtype.val = best_ap_evtype.val = -1;
+}
+
+/** True if @a state is a "better" origin circuit state than @a best->val */
+static bool
+btc_state_better(int state, const btc_best_t *best)
+{
+  return state > best->val;
+}
+
+/**
+ * Definine an ordering on circuit status events
+ *
+ * The CIRC_EVENT_ constants aren't sorted in a useful order, so this
+ * array helps to decode them.  This approach depends on the statuses
+ * being nonnegative and dense.
+ **/
+static int circ_event_order[] = {
+  [CIRC_EVENT_FAILED] = -1,
+  [CIRC_EVENT_CLOSED] = -1,
+  [CIRC_EVENT_LAUNCHED] = 1,
+  [CIRC_EVENT_EXTENDED] = 2,
+  [CIRC_EVENT_BUILT] = 3,
+};
+#define N_CIRC_EVENT_ORDER \
+  (sizeof(circ_event_order) / sizeof(circ_event_order[0]))
+
+/** True if @a state is a "better" origin circuit event status than @a
+    best->val */
+static bool
+btc_evtype_better(int state, const btc_best_t *best)
+{
+  if (state < 0)
+    return false;
+  if (best->val < 0)
+    return true;
+
+  tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER);
+  tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER);
+  return circ_event_order[state] > circ_event_order[best->val];
+}
+
+static bool
+btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best,
+                 const char *type)
+{
+  if (btc_state_better(msg->state, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type,
+             best->val, msg->state, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->state;
+    return true;
+  }
+  return false;
+}
+
+static bool
+btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best,
+                  const char *type)
+{
+  if (btc_evtype_better(msg->evtype, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type,
+             best->val, msg->evtype, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->evtype;
+    return true;
+  }
+  return false;
+}
+
+DECLARE_SUBSCRIBE(ocirc_state, btc_state_rcvr);
+DECLARE_SUBSCRIBE(ocirc_cevent, btc_cevent_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, btc_chan_rcvr);
+
+static void
+btc_state_rcvr(const msg_t *msg, const ocirc_state_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d",
+            arg->gid, arg->state, arg->onehop);
+
+  btc_update_state(arg, &best_any_state, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_state(arg, &best_ap_state, "AP");
+}
+
+static void
+btc_cevent_rcvr(const msg_t *msg, const ocirc_cevent_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d",
+            arg->gid, arg->evtype, arg->reason, arg->onehop);
+
+  btc_update_evtype(arg, &best_any_evtype, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_evtype(arg, &best_ap_evtype, "AP");
+}
+
+static void
+btc_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d",
+            arg->gid, arg->chan, arg->onehop);
+}
+
+int
+btrack_circ_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_state))
+    return -1;
+  return 0;
+}
+
+void
+btrack_circ_fini(void)
+{
+  btc_reset_bests();
+}
diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h
new file mode 100644
index 0000000000..b326c22ccf
--- /dev/null
+++ b/src/feature/control/btrack_circuit.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.h
+ * \brief Header file for btrack_circuit.c
+ **/
+
+#ifndef TOR_BTRACK_CIRCUIT_H
+#define TOR_BTRACK_CIRCUIT_H
+
+#include "lib/pubsub/pubsub.h"
+
+int btrack_circ_init(void);
+void btrack_circ_fini(void);
+int btrack_circ_add_pubsub(pubsub_connector_t *);
+
+#endif /* !defined(TOR_BTRACK_CIRCUIT_H) */
diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c
new file mode 100644
index 0000000000..922b542a0c
--- /dev/null
+++ b/src/feature/control/btrack_orconn.c
@@ -0,0 +1,206 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.c
+ * \brief Bootstrap tracker for OR connections
+ *
+ * Track state changes of OR connections, as published by the
+ * connection subsystem.  Also track circuit launch events, because
+ * they're one of the few ways to discover the association between a
+ * channel (and OR connection) and a circuit.
+ *
+ * We track all OR connections that we receive events for, whether or
+ * not they're carrying origin circuits.  (An OR connection might
+ * carry origin circuits only after we first find out about that
+ * connection.)
+ *
+ * All origin ORCONN events update the "any" state variables, while
+ * only application ORCONN events update the "ap" state variables (and
+ * also update the "any") variables.
+ *
+ * We do this because we want to report the first increments of
+ * connection progress as the earliest bootstrap phases.  This results
+ * in a better user experience because failures here translate into
+ * zero or very small amounts of displayed progress, instead of
+ * progress stuck near completion.  The first connection to a relay
+ * might be a one-hop circuit for directory lookups, or it might be a
+ * connection for an application circuit because we already have
+ * enough directory info to build an application circuit.
+ *
+ * We call functions in btrack_orconn_cevent.c to generate the actual
+ * controller events, because some of the state decoding we need to do
+ * is complicated.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+#include "lib/pubsub/pubsub.h"
+
+DECLARE_SUBSCRIBE(orconn_state, bto_state_rcvr);
+DECLARE_SUBSCRIBE(orconn_status, bto_status_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, bto_chan_rcvr);
+
+/** Pair of a best ORCONN GID and with its state */
+typedef struct bto_best_t {
+  uint64_t gid;
+  int state;
+} bto_best_t;
+
+/** GID and state of the best ORCONN we've seen so far */
+static bto_best_t best_any = { 0, -1 };
+/** GID and state of the best application circuit ORCONN we've seen so far */
+static bto_best_t best_ap = { 0, -1 };
+
+/**
+ * Update a cached state of a best ORCONN progress we've seen so far.
+ *
+ * Return true if the new state is better than the old.
+ **/
+static bool
+bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type)
+{
+  if (bto->state < best->state)
+    return false;
+  /* Update even if we won't change best->state, because it's more
+   * recent information that a particular connection transitioned to
+   * that state. */
+  best->gid = bto->gid;
+  if (bto->state > best->state) {
+    log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type,
+             best->state, bto->state, bto->gid);
+    best->state = bto->state;
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Update cached states of best ORCONN progress we've seen
+ *
+ * Only update the application ORCONN state if we know it's carrying
+ * an application circuit.
+ **/
+static void
+bto_update_bests(const bt_orconn_t *bto)
+{
+  tor_assert(bto->is_orig);
+
+  if (bto_update_best(bto, &best_any, "ANY"))
+    bto_cevent_anyconn(bto);
+  if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP"))
+    bto_cevent_apconn(bto);
+}
+
+/** Reset cached "best" values */
+static void
+bto_reset_bests(void)
+{
+  best_any.gid = best_ap.gid = 0;
+  best_any.state = best_ap.state = -1;
+}
+
+/**
+ * Update cached states of ORCONNs from the incoming message.  This
+ * message comes from code in connection_or.c.
+ **/
+static void
+bto_state_rcvr(const msg_t *msg, const orconn_state_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(arg->gid, arg->chan);
+  log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64
+            " proxy_type=%d state=%d",
+            arg->gid, arg->chan, arg->proxy_type, arg->state);
+  bto->proxy_type = arg->proxy_type;
+  bto->state = arg->state;
+  if (bto->is_orig)
+    bto_update_bests(bto);
+}
+
+/**
+ * Delete a cached ORCONN state if we get an incoming message saying
+ * the ORCONN is failed or closed.  This message comes from code in
+ * control.c.
+ **/
+static void
+bto_status_rcvr(const msg_t *msg, const orconn_status_msg_t *arg)
+{
+  (void)msg;
+  switch (arg->status) {
+  case OR_CONN_EVENT_FAILED:
+  case OR_CONN_EVENT_CLOSED:
+    log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d",
+             arg->gid, arg->status, arg->reason);
+    return bto_delete(arg->gid);
+  default:
+    break;
+  }
+}
+
+/**
+ * Create or update a cached ORCONN state for a newly launched
+ * connection, including whether it's launched by an origin circuit
+ * and whether it's a one-hop circuit.
+ **/
+static void
+bto_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(0, arg->chan);
+  if (!bto->is_orig || (bto->is_onehop && !arg->onehop)) {
+    log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d",
+              arg->chan, arg->onehop);
+  }
+  bto->is_orig = true;
+  if (!arg->onehop)
+    bto->is_onehop = false;
+  bto_update_bests(bto);
+}
+
+/**
+ * Initialize the hash maps and subscribe to ORCONN and origin
+ * circuit events.
+ **/
+int
+btrack_orconn_init(void)
+{
+  bto_init_maps();
+
+  return 0;
+}
+
+int
+btrack_orconn_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_state))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_status))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  return 0;
+}
+
+/** Clear the hash maps and reset the "best" states */
+void
+btrack_orconn_fini(void)
+{
+  bto_clear_maps();
+  bto_reset_bests();
+  bto_cevent_reset();
+}
diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h
new file mode 100644
index 0000000000..07b1b755f3
--- /dev/null
+++ b/src/feature/control/btrack_orconn.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.h
+ * \brief Header file for btrack_orconn.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_H
+#define TOR_BTRACK_ORCONN_H
+
+#include "lib/pubsub/pubsub.h"
+
+#ifdef BTRACK_ORCONN_PRIVATE
+
+#include "ht.h"
+
+/**
+ * Structure for tracking OR connection states
+ *
+ * This gets linked into two hash maps: one with connection IDs, and
+ * another with channel IDs.
+ **/
+typedef struct bt_orconn_t {
+  HT_ENTRY(bt_orconn_t) node;   /**< Hash map entry indexed by gid */
+  HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */
+  uint64_t gid;                    /**< Global ID of this ORCONN */
+  uint64_t chan;                   /**< Channel ID, if known */
+  int proxy_type;                  /**< Proxy type */
+  uint8_t state;                   /**< State of this ORCONN */
+  bool is_orig;             /**< Does this carry an origin circuit? */
+  bool is_onehop;           /**< Is this for a one-hop circuit? */
+} bt_orconn_t;
+
+#endif /* defined(BTRACK_ORCONN_PRIVATE) */
+
+int btrack_orconn_init(void);
+int btrack_orconn_add_pubsub(pubsub_connector_t *);
+void btrack_orconn_fini(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_H) */
diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c
new file mode 100644
index 0000000000..a00eb042d5
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.c
@@ -0,0 +1,160 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.c
+ * \brief Emit bootstrap status events for OR connections
+ *
+ * We do some decoding of the raw OR_CONN_STATE_* values.  For
+ * example, OR_CONN_STATE_CONNECTING means the first TCP connect()
+ * completing, regardless of whether it's directly to a relay instead
+ * of a proxy or a PT.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/control_events.h"
+
+/**
+ * Have we completed our first OR connection?
+ *
+ * Block display of application circuit progress until we do, to avoid
+ * some misleading behavior of jumping to high progress.
+ **/
+static bool bto_first_orconn = false;
+
+/** Is the ORCONN using a pluggable transport? */
+static bool
+using_pt(const bt_orconn_t *bto)
+{
+  return bto->proxy_type == PROXY_PLUGGABLE;
+}
+
+/** Is the ORCONN using a non-PT proxy? */
+static bool
+using_proxy(const bt_orconn_t *bto)
+{
+  switch (bto->proxy_type) {
+  case PROXY_CONNECT:
+  case PROXY_SOCKS4:
+  case PROXY_SOCKS5:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_anyconn(const bt_orconn_t *bto)
+{
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0);
+    /* Unblock directory progress display */
+    control_event_boot_first_orconn();
+    /* Unblock apconn progress display */
+    bto_first_orconn = true;
+    break;
+  default:
+    break;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any application circuit OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_apconn(const bt_orconn_t *bto)
+{
+  if (!bto_first_orconn)
+    return;
+
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0);
+    break;
+  default:
+    break;
+  }
+}
+
+/** Forget that we completed our first OR connection */
+void
+bto_cevent_reset(void)
+{
+  bto_first_orconn = false;
+}
diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h
new file mode 100644
index 0000000000..afec55581e
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.h
+ * \brief Header file for btrack_orconn_cevent.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_CEVENT_H
+#define TOR_BTRACK_ORCONN_CEVENT_H
+
+#include "feature/control/btrack_orconn.h"
+
+void bto_cevent_anyconn(const bt_orconn_t *);
+void bto_cevent_apconn(const bt_orconn_t *);
+void bto_cevent_reset(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_CEVENT_H) */
diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c
new file mode 100644
index 0000000000..e64bd3f0fe
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.c
@@ -0,0 +1,223 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.c
+ * \brief Hash map implementation for btrack_orconn.c
+ *
+ * These functions manipulate the hash maps that contain bt_orconn
+ * objects.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+
+static inline unsigned int
+bto_gid_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid));
+}
+
+static inline int
+bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->gid == b->gid;
+}
+
+static inline unsigned int
+bto_chan_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan));
+}
+
+static inline int
+bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->chan == b->chan;
+}
+
+HT_HEAD(bto_gid_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_)
+HT_GENERATE2(bto_gid_ht, bt_orconn_t, node,
+             bto_gid_hash_, bto_gid_eq_, 0.6,
+             tor_reallocarray_, tor_free_)
+static struct bto_gid_ht *bto_gid_map;
+
+HT_HEAD(bto_chan_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_, bto_chan_eq_)
+HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node,
+             bto_chan_hash_, bto_chan_eq_, 0.6,
+             tor_reallocarray_, tor_free_)
+static struct bto_chan_ht *bto_chan_map;
+
+/** Clear the GID hash map, freeing any bt_orconn_t objects that become
+ * unreferenced */
+static void
+bto_gid_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_gid_ht, bto_gid_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt);
+
+    c->gid = 0;
+    /* Don't delete if chan ID isn't zero: it's still in the chan hash map */
+    if (!c->chan)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_gid_ht, bto_gid_map);
+  tor_free(bto_gid_map);
+}
+
+/** Clear the chan ID hash map, freeing any bt_orconn_t objects that
+ * become unreferenced */
+static void
+bto_chan_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_chan_ht, bto_chan_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt);
+
+    c->chan = 0;
+    /* Don't delete if GID isn't zero, it's still in the GID hash map */
+    if (!c->gid)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_chan_ht, bto_chan_map);
+  tor_free(bto_chan_map);
+}
+
+/** Delete a bt_orconn from the hash maps by GID */
+void
+bto_delete(uint64_t gid)
+{
+  bt_orconn_t key, *bto;
+
+  key.gid = gid;
+  key.chan = 0;
+  bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto) {
+    /* The orconn might be unregistered because it's an EXT_OR_CONN? */
+    log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64,
+              gid);
+    return;
+  }
+  HT_REMOVE(bto_gid_ht, bto_gid_map, &key);
+  if (bto->chan) {
+    key.chan = bto->chan;
+    HT_REMOVE(bto_chan_ht, bto_chan_map, &key);
+  }
+  tor_free(bto);
+}
+
+/**
+ * Helper for bto_find_or_new().
+ *
+ * Update GID and chan ID of an existing bt_orconn object if needed,
+ * given a search key previously used within bto_find_or_new().
+ **/
+static bt_orconn_t *
+bto_update(bt_orconn_t *bto, const bt_orconn_t *key)
+{
+  /* ORCONN GIDs shouldn't change once assigned */
+  tor_assert(!bto->gid || !key->gid || bto->gid == key->gid);
+  if (!bto->gid && key->gid) {
+    /* Got a gid when we didn't already have one; insert into gid map */
+    log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan,
+              key->gid);
+    bto->gid = key->gid;
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  }
+  /* association of ORCONN with channel shouldn't change */
+  tor_assert(!bto->chan || !key->chan || bto->chan == key->chan);
+  if (!bto->chan && key->chan) {
+    /* Got a chan when we didn't already have one; insert into chan map */
+    log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64,
+              bto->gid, key->chan);
+    bto->chan = key->chan;
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+  }
+  return bto;
+}
+
+/** Helper for bto_find_or_new() */
+static bt_orconn_t *
+bto_new(const bt_orconn_t *key)
+{
+  struct bt_orconn_t *bto = tor_malloc(sizeof(*bto));
+
+  bto->gid = key->gid;
+  bto->chan = key->chan;
+  bto->state = 0;
+  bto->proxy_type = 0;
+  bto->is_orig = false;
+  bto->is_onehop = true;
+
+  if (bto->gid)
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  if (bto->chan)
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+
+  return bto;
+}
+
+/**
+ * Insert a new bt_orconn with the given GID and chan ID, or update
+ * the GID and chan ID if one already exists.
+ *
+ * Return the found or allocated bt_orconn.
+ **/
+bt_orconn_t *
+bto_find_or_new(uint64_t gid, uint64_t chan)
+{
+  bt_orconn_t key, *bto = NULL;
+
+  tor_assert(gid || chan);
+  key.gid = gid;
+  key.chan = chan;
+  if (key.gid)
+    bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto && key.chan) {
+    /* Not found by GID; look up by chan ID */
+    bto = HT_FIND(bto_chan_ht, bto_chan_map, &key);
+  }
+  if (bto)
+    return bto_update(bto, &key);
+  else
+    return bto_new(&key);
+}
+
+/** Initialize the hash maps  */
+void
+bto_init_maps(void)
+{
+  bto_gid_map = tor_malloc(sizeof(*bto_gid_map));
+  HT_INIT(bto_gid_ht, bto_gid_map);
+  bto_chan_map = tor_malloc(sizeof(*bto_chan_map));
+  HT_INIT(bto_chan_ht, bto_chan_map);
+}
+
+/** Clear the hash maps, freeing all associated storage */
+void
+bto_clear_maps(void)
+{
+  bto_gid_clear_map();
+  bto_chan_clear_map();
+}
diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h
new file mode 100644
index 0000000000..c2043fa153
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.h
+ * \brief Header file for btrack_orconn_maps.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_MAPS_H
+#define TOR_BTRACK_ORCONN_MAPS_H
+
+void bto_delete(uint64_t);
+bt_orconn_t *bto_find_or_new(uint64_t, uint64_t);
+
+void bto_init_maps(void);
+void bto_clear_maps(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_MAPS_H) */
diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h
new file mode 100644
index 0000000000..3f831d0640
--- /dev/null
+++ b/src/feature/control/btrack_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_sys.h
+ * \brief Declare subsystem object for the bootstrap tracker susbystem.
+ **/
+
+#ifndef TOR_BTRACK_SYS_H
+#define TOR_BTRACK_SYS_H
+
+extern const struct subsys_fns_t sys_btrack;
+
+#endif /* !defined(TOR_BTRACK_SYS_H) */
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
index 26ac12d307..ece5616907 100644
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@ -1,4 +1,3 @@
-
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
@@ -33,83 +32,27 @@
  * stack.
  **/
 
+#define CONTROL_MODULE_PRIVATE
 #define CONTROL_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
 #include "app/main/main.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
-#include "core/or/channel.h"
-#include "core/or/channeltls.h"
-#include "core/or/circuitbuild.h"
-#include "core/or/circuitlist.h"
-#include "core/or/circuitstats.h"
-#include "core/or/circuituse.h"
-#include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/policies.h"
-#include "core/or/reasons.h"
-#include "core/or/versions.h"
 #include "core/proto/proto_control0.h"
 #include "core/proto/proto_http.h"
-#include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/dnsserv.h"
-#include "feature/client/entrynodes.h"
 #include "feature/control/control.h"
-#include "feature/control/fmt_serverstatus.h"
-#include "feature/control/getinfo_geoip.h"
-#include "feature/dircache/dirserv.h"
-#include "feature/dirclient/dirclient.h"
-#include "feature/dirclient/dlstatus.h"
-#include "feature/dircommon/directory.h"
-#include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
-#include "feature/hs/hs_common.h"
-#include "feature/hs/hs_control.h"
-#include "feature/hs_common/shared_random_client.h"
-#include "feature/nodelist/authcert.h"
-#include "feature/nodelist/dirlist.h"
-#include "feature/nodelist/microdesc.h"
-#include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
-#include "feature/nodelist/routerlist.h"
-#include "feature/relay/router.h"
-#include "feature/relay/routermode.h"
-#include "feature/relay/selftest.h"
-#include "feature/rend/rendclient.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_proto.h"
 #include "feature/rend/rendcommon.h"
-#include "feature/rend/rendparse.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
-#include "feature/stats/predict_ports.h"
-#include "lib/container/buffers.h"
-#include "lib/crypt_ops/crypto_rand.h"
-#include "lib/crypt_ops/crypto_util.h"
-#include "lib/encoding/confline.h"
-#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/procmon.h"
 
-#include "feature/dircache/cached_dir_st.h"
 #include "feature/control/control_connection_st.h"
-#include "core/or/cpath_build_state_st.h"
-#include "core/or/entry_connection_st.h"
-#include "feature/nodelist/extrainfo_st.h"
-#include "feature/nodelist/networkstatus_st.h"
-#include "feature/nodelist/node_st.h"
-#include "core/or/or_connection_st.h"
-#include "core/or/or_circuit_st.h"
-#include "core/or/origin_circuit_st.h"
-#include "feature/nodelist/microdesc_st.h"
-#include "feature/rend/rend_authorized_client_st.h"
-#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
-#include "feature/rend/rend_service_descriptor_st.h"
-#include "feature/nodelist/routerinfo_st.h"
-#include "feature/nodelist/routerlist_st.h"
-#include "core/or/socks_request_st.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -118,152 +61,6 @@
 #include <sys/stat.h>
 #endif
 
-#ifndef _WIN32
-#include <pwd.h>
-#include <sys/resource.h>
-#endif
-
-#include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/evloop/procmon.h"
-#include "lib/evloop/compat_libevent.h"
-
-/** Yield true iff <b>s</b> is the state of a control_connection_t that has
- * finished authentication and is accepting commands. */
-#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
-
-/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
- * connection is interested in events of type <b>e</b>.  We use this
- * so that we can decide to skip generating event messages that nobody
- * has interest in without having to walk over the global connection
- * list to find out.
- **/
-typedef uint64_t event_mask_t;
-
-/** An event mask of all the events that any controller is interested in
- * receiving. */
-static event_mask_t global_event_mask = 0;
-
-/** True iff we have disabled log messages from being sent to the controller */
-static int disable_log_messages = 0;
-
-/** Macro: true if any control connection is interested in events of type
- * <b>e</b>. */
-#define EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & EVENT_MASK_(e)))
-
-/** Macro: true if any event from the bitfield 'e' is interesting. */
-#define ANY_EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & (e)))
-
-/** If we're using cookie-type authentication, how long should our cookies be?
- */
-#define AUTHENTICATION_COOKIE_LEN 32
-
-/** If true, we've set authentication_cookie to a secret code and
- * stored it to disk. */
-static int authentication_cookie_is_set = 0;
-/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
- * and which we're using to authenticate controllers.  (If the controller can
- * read it off disk, it has permission to connect.) */
-static uint8_t *authentication_cookie = NULL;
-
-#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
-  "Tor safe cookie authentication server-to-controller hash"
-#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
-  "Tor safe cookie authentication controller-to-server hash"
-#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
-
-/** The list of onion services that have been added via ADD_ONION that do not
- * belong to any particular control connection.
- */
-static smartlist_t *detached_onion_services = NULL;
-
-/** A sufficiently large size to record the last bootstrap phase string. */
-#define BOOTSTRAP_MSG_LEN 1024
-
-/** What was the last bootstrap phase message we sent? We keep track
- * of this so we can respond to getinfo status/bootstrap-phase queries. */
-static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
-
-static void connection_printf_to_buf(control_connection_t *conn,
-                                     const char *format, ...)
-  CHECK_PRINTF(2,3);
-static void send_control_event_impl(uint16_t event,
-                                    const char *format, va_list ap)
-  CHECK_PRINTF(2,0);
-static int control_event_status(int type, int severity, const char *format,
-                                va_list args)
-  CHECK_PRINTF(3,0);
-
-static void send_control_done(control_connection_t *conn);
-static void send_control_event(uint16_t event,
-                               const char *format, ...)
-  CHECK_PRINTF(2,3);
-static int handle_control_setconf(control_connection_t *conn, uint32_t len,
-                                  char *body);
-static int handle_control_resetconf(control_connection_t *conn, uint32_t len,
-                                    char *body);
-static int handle_control_getconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_setevents(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_authenticate(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_signal(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                                     const char *body);
-static char *list_getinfo_options(void);
-static int handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_extendcircuit(control_connection_t *conn,
-                                        uint32_t len,
-                                        const char *body);
-static int handle_control_setcircuitpurpose(control_connection_t *conn,
-                                            uint32_t len, const char *body);
-static int handle_control_attachstream(control_connection_t *conn,
-                                       uint32_t len,
-                                        const char *body);
-static int handle_control_postdescriptor(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_redirectstream(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_closestream(control_connection_t *conn, uint32_t len,
-                                      const char *body);
-static int handle_control_closecircuit(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_resolve(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_usefeature(control_connection_t *conn,
-                                     uint32_t len,
-                                     const char *body);
-static int handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_hspost(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_add_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_del_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
-                                      size_t len);
-static void orconn_target_get_name(char *buf, size_t len,
-                                   or_connection_t *conn);
-
-static int get_cached_network_liveness(void);
-static void set_cached_network_liveness(int liveness);
-
-static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
-
-static char * download_status_to_string(const download_status_t *dl);
-static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
-
 /** Convert a connection_t* to an control_connection_t*; assert if the cast is
  * invalid. */
 control_connection_t *
@@ -273,410 +70,6 @@ TO_CONTROL_CONN(connection_t *c)
   return DOWNCAST(control_connection_t, c);
 }
 
-/** Given a control event code for a message event, return the corresponding
- * log severity. */
-static inline int
-event_to_log_severity(int event)
-{
-  switch (event) {
-    case EVENT_DEBUG_MSG: return LOG_DEBUG;
-    case EVENT_INFO_MSG: return LOG_INFO;
-    case EVENT_NOTICE_MSG: return LOG_NOTICE;
-    case EVENT_WARN_MSG: return LOG_WARN;
-    case EVENT_ERR_MSG: return LOG_ERR;
-    default: return -1;
-  }
-}
-
-/** Given a log severity, return the corresponding control event code. */
-static inline int
-log_severity_to_event(int severity)
-{
-  switch (severity) {
-    case LOG_DEBUG: return EVENT_DEBUG_MSG;
-    case LOG_INFO: return EVENT_INFO_MSG;
-    case LOG_NOTICE: return EVENT_NOTICE_MSG;
-    case LOG_WARN: return EVENT_WARN_MSG;
-    case LOG_ERR: return EVENT_ERR_MSG;
-    default: return -1;
-  }
-}
-
-/** Helper: clear bandwidth counters of all origin circuits. */
-static void
-clear_circ_bw_fields(void)
-{
-  origin_circuit_t *ocirc;
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-    ocirc = TO_ORIGIN_CIRCUIT(circ);
-    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-  }
-  SMARTLIST_FOREACH_END(circ);
-}
-
-/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
- * connection's event_mask field. */
-void
-control_update_global_event_mask(void)
-{
-  smartlist_t *conns = get_connection_array();
-  event_mask_t old_mask, new_mask;
-  old_mask = global_event_mask;
-  int any_old_per_sec_events = control_any_per_second_event_enabled();
-
-  global_event_mask = 0;
-  SMARTLIST_FOREACH(conns, connection_t *, _conn,
-  {
-    if (_conn->type == CONN_TYPE_CONTROL &&
-        STATE_IS_OPEN(_conn->state)) {
-      control_connection_t *conn = TO_CONTROL_CONN(_conn);
-      global_event_mask |= conn->event_mask;
-    }
-  });
-
-  new_mask = global_event_mask;
-
-  /* Handle the aftermath.  Set up the log callback to tell us only what
-   * we want to hear...*/
-  control_adjust_event_log_severity();
-
-  /* Macro: true if ev was false before and is true now. */
-#define NEWLY_ENABLED(ev) \
-  (! (old_mask & (ev)) && (new_mask & (ev)))
-
-  /* ...then, if we've started logging stream or circ bw, clear the
-   * appropriate fields. */
-  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
-    SMARTLIST_FOREACH(conns, connection_t *, conn,
-    {
-      if (conn->type == CONN_TYPE_AP) {
-        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-        edge_conn->n_written = edge_conn->n_read = 0;
-      }
-    });
-  }
-  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
-    clear_circ_bw_fields();
-  }
-  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
-    uint64_t r, w;
-    control_get_bytes_rw_last_sec(&r, &w);
-  }
-  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
-    reschedule_per_second_timer();
-  }
-
-#undef NEWLY_ENABLED
-}
-
-/** Adjust the log severities that result in control_event_logmsg being called
- * to match the severity of log messages that any controllers are interested
- * in. */
-void
-control_adjust_event_log_severity(void)
-{
-  int i;
-  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
-
-  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      min_log_event = i;
-      break;
-    }
-  }
-  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      max_log_event = i;
-      break;
-    }
-  }
-  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
-    if (min_log_event > EVENT_NOTICE_MSG)
-      min_log_event = EVENT_NOTICE_MSG;
-    if (max_log_event < EVENT_ERR_MSG)
-      max_log_event = EVENT_ERR_MSG;
-  }
-  if (min_log_event <= max_log_event)
-    change_callback_log_severity(event_to_log_severity(min_log_event),
-                                 event_to_log_severity(max_log_event),
-                                 control_event_logmsg);
-  else
-    change_callback_log_severity(LOG_ERR, LOG_ERR,
-                                 control_event_logmsg);
-}
-
-/** Return true iff the event with code <b>c</b> is being sent to any current
- * control connection.  This is useful if the amount of work needed to prepare
- * to call the appropriate control_event_...() function is high.
- */
-int
-control_event_is_interesting(int event)
-{
-  return EVENT_IS_INTERESTING(event);
-}
-
-/** Return true if any event that needs to fire once a second is enabled. */
-int
-control_any_per_second_event_enabled(void)
-{
-  return ANY_EVENT_IS_INTERESTING(
-      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CELL_STATS) |
-      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CONN_BW) |
-      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
-  );
-}
-
-/* The value of 'get_bytes_read()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_read = 0;
-/* The value of 'get_bytes_written()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_written = 0;
-
-/**
- * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
- * and written by Tor since the last call to this function.
- *
- * Call this only from the main thread.
- */
-static void
-control_get_bytes_rw_last_sec(uint64_t *n_read,
-                              uint64_t *n_written)
-{
-  const uint64_t stats_n_bytes_read = get_bytes_read();
-  const uint64_t stats_n_bytes_written = get_bytes_written();
-
-  *n_read = stats_n_bytes_read - stats_prev_n_read;
-  *n_written = stats_n_bytes_written - stats_prev_n_written;
-  stats_prev_n_read = stats_n_bytes_read;
-  stats_prev_n_written = stats_n_bytes_written;
-}
-
-/**
- * Run all the controller events (if any) that are scheduled to trigger once
- * per second.
- */
-void
-control_per_second_events(void)
-{
-  if (!control_any_per_second_event_enabled())
-    return;
-
-  uint64_t bytes_read, bytes_written;
-  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
-  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
-
-  control_event_stream_bandwidth_used();
-  control_event_conn_bandwidth_used();
-  control_event_circ_bandwidth_used();
-  control_event_circuit_cell_stats();
-}
-
-/** Append a NUL-terminated string <b>s</b> to the end of
- * <b>conn</b>-\>outbuf.
- */
-static inline void
-connection_write_str_to_buf(const char *s, control_connection_t *conn)
-{
-  size_t len = strlen(s);
-  connection_buf_add(s, len, TO_CONN(conn));
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
- * contents of <b>data</b> into *<b>out</b>, adding a period before any period
- * that appears at the start of a line, and adding a period-CRLF line at
- * the end. Replace all LF characters sequences with CRLF.  Return the number
- * of bytes in *<b>out</b>.
- */
-STATIC size_t
-write_escaped_data(const char *data, size_t len, char **out)
-{
-  tor_assert(len < SIZE_MAX - 9);
-  size_t sz_out = len+8+1;
-  char *outp;
-  const char *start = data, *end;
-  size_t i;
-  int start_of_line;
-  for (i=0; i < len; ++i) {
-    if (data[i] == '\n') {
-      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
-      if (sz_out >= SIZE_T_CEILING) {
-        log_warn(LD_BUG, "Input to write_escaped_data was too long");
-        *out = tor_strdup(".\r\n");
-        return 3;
-      }
-    }
-  }
-  *out = outp = tor_malloc(sz_out);
-  end = data+len;
-  start_of_line = 1;
-  while (data < end) {
-    if (*data == '\n') {
-      if (data > start && data[-1] != '\r')
-        *outp++ = '\r';
-      start_of_line = 1;
-    } else if (*data == '.') {
-      if (start_of_line) {
-        start_of_line = 0;
-        *outp++ = '.';
-      }
-    } else {
-      start_of_line = 0;
-    }
-    *outp++ = *data++;
-  }
-  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
-    *outp++ = '\r';
-    *outp++ = '\n';
-  }
-  *outp++ = '.';
-  *outp++ = '\r';
-  *outp++ = '\n';
-  *outp = '\0'; /* NUL-terminate just in case. */
-  tor_assert(outp >= *out);
-  tor_assert((size_t)(outp - *out) <= sz_out);
-  return outp - *out;
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
- * the contents of <b>data</b> into *<b>out</b>, removing any period
- * that appears at the start of a line, and replacing all CRLF sequences
- * with LF.   Return the number of
- * bytes in *<b>out</b>. */
-STATIC size_t
-read_escaped_data(const char *data, size_t len, char **out)
-{
-  char *outp;
-  const char *next;
-  const char *end;
-
-  *out = outp = tor_malloc(len+1);
-
-  end = data+len;
-
-  while (data < end) {
-    /* we're at the start of a line. */
-    if (*data == '.')
-      ++data;
-    next = memchr(data, '\n', end-data);
-    if (next) {
-      size_t n_to_copy = next-data;
-      /* Don't copy a CR that precedes this LF. */
-      if (n_to_copy && *(next-1) == '\r')
-        --n_to_copy;
-      memcpy(outp, data, n_to_copy);
-      outp += n_to_copy;
-      data = next+1; /* This will point at the start of the next line,
-                      * or the end of the string, or a period. */
-    } else {
-      memcpy(outp, data, end-data);
-      outp += (end-data);
-      *outp = '\0';
-      return outp - *out;
-    }
-    *outp++ = '\n';
-  }
-
-  *outp = '\0';
-  return outp - *out;
-}
-
-/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
- * double-quoted string with escaped characters, return the length of that
- * string (as encoded, including quotes).  Otherwise return -1. */
-static inline int
-get_escaped_string_length(const char *start, size_t in_len_max,
-                          int *chars_out)
-{
-  const char *cp, *end;
-  int chars = 0;
-
-  if (*start != '\"')
-    return -1;
-
-  cp = start+1;
-  end = start+in_len_max;
-
-  /* Calculate length. */
-  while (1) {
-    if (cp >= end) {
-      return -1; /* Too long. */
-    } else if (*cp == '\\') {
-      if (++cp == end)
-        return -1; /* Can't escape EOS. */
-      ++cp;
-      ++chars;
-    } else if (*cp == '\"') {
-      break;
-    } else {
-      ++cp;
-      ++chars;
-    }
-  }
-  if (chars_out)
-    *chars_out = chars;
-  return (int)(cp - start+1);
-}
-
-/** As decode_escaped_string, but does not decode the string: copies the
- * entire thing, including quotation marks. */
-static const char *
-extract_escaped_string(const char *start, size_t in_len_max,
-                       char **out, size_t *out_len)
-{
-  int length = get_escaped_string_length(start, in_len_max, NULL);
-  if (length<0)
-    return NULL;
-  *out_len = length;
-  *out = tor_strndup(start, *out_len);
-  return start+length;
-}
-
-/** Given a pointer to a string starting at <b>start</b> containing
- * <b>in_len_max</b> characters, decode a string beginning with one double
- * quote, containing any number of non-quote characters or characters escaped
- * with a backslash, and ending with a final double quote.  Place the resulting
- * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
- * store its length in <b>out_len</b>.  On success, return a pointer to the
- * character immediately following the escaped string.  On failure, return
- * NULL. */
-static const char *
-decode_escaped_string(const char *start, size_t in_len_max,
-                   char **out, size_t *out_len)
-{
-  const char *cp, *end;
-  char *outp;
-  int len, n_chars = 0;
-
-  len = get_escaped_string_length(start, in_len_max, &n_chars);
-  if (len<0)
-    return NULL;
-
-  end = start+len-1; /* Index of last quote. */
-  tor_assert(*end == '\"');
-  outp = *out = tor_malloc(len+1);
-  *out_len = n_chars;
-
-  cp = start+1;
-  while (cp < end) {
-    if (*cp == '\\')
-      ++cp;
-    *outp++ = *cp++;
-  }
-  *outp = '\0';
-  tor_assert((outp - *out) == (int)*out_len);
-
-  return end+1;
-}
-
 /** Create and add a new controller connection on <b>sock</b>.  If
  * <b>CC_LOCAL_FD_IS_OWNER</b> is set in <b>flags</b>, this Tor process should
  * exit when the connection closes.  If <b>CC_LOCAL_FD_IS_AUTHENTICATED</b>
@@ -720,29 +113,6 @@ control_connection_add_local_fd(tor_socket_t sock, unsigned flags)
   return 0;
 }
 
-/** Acts like sprintf, but writes its formatted string to the end of
- * <b>conn</b>-\>outbuf. */
-static void
-connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
-{
-  va_list ap;
-  char *buf = NULL;
-  int len;
-
-  va_start(ap,format);
-  len = tor_vasprintf(&buf, format, ap);
-  va_end(ap);
-
-  if (len < 0) {
-    log_err(LD_BUG, "Unable to format string for controller.");
-    tor_assert(0);
-  }
-
-  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
-
-  tor_free(buf);
-}
-
 /** Write all of the open control ports to ControlPortWriteToFile */
 void
 control_ports_write_to_file(void)
@@ -787,886 +157,11 @@ control_ports_write_to_file(void)
   smartlist_free(lines);
 }
 
-/** Send a "DONE" message down the control connection <b>conn</b>. */
-static void
-send_control_done(control_connection_t *conn)
-{
-  connection_write_str_to_buf("250 OK\r\n", conn);
-}
-
-/** Represents an event that's queued to be sent to one or more
- * controllers. */
-typedef struct queued_event_s {
-  uint16_t event;
-  char *msg;
-} queued_event_t;
-
-/** Pointer to int. If this is greater than 0, we don't allow new events to be
- * queued. */
-static tor_threadlocal_t block_event_queue_flag;
-
-/** Holds a smartlist of queued_event_t objects that may need to be sent
- * to one or more controllers */
-static smartlist_t *queued_control_events = NULL;
-
-/** True if the flush_queued_events_event is pending. */
-static int flush_queued_event_pending = 0;
-
-/** Lock to protect the above fields. */
-static tor_mutex_t *queued_control_events_lock = NULL;
-
-/** An event that should fire in order to flush the contents of
- * queued_control_events. */
-static mainloop_event_t *flush_queued_events_event = NULL;
-
-void
-control_initialize_event_queue(void)
-{
-  if (queued_control_events == NULL) {
-    queued_control_events = smartlist_new();
-  }
-
-  if (flush_queued_events_event == NULL) {
-    struct event_base *b = tor_libevent_get_base();
-    if (b) {
-      flush_queued_events_event =
-        mainloop_event_new(flush_queued_events_cb, NULL);
-      tor_assert(flush_queued_events_event);
-    }
-  }
-
-  if (queued_control_events_lock == NULL) {
-    queued_control_events_lock = tor_mutex_new();
-    tor_threadlocal_init(&block_event_queue_flag);
-  }
-}
-
-static int *
-get_block_event_queue(void)
-{
-  int *val = tor_threadlocal_get(&block_event_queue_flag);
-  if (PREDICT_UNLIKELY(val == NULL)) {
-    val = tor_malloc_zero(sizeof(int));
-    tor_threadlocal_set(&block_event_queue_flag, val);
-  }
-  return val;
-}
-
-/** Helper: inserts an event on the list of events queued to be sent to
- * one or more controllers, and schedules the events to be flushed if needed.
- *
- * This function takes ownership of <b>msg</b>, and may free it.
- *
- * We queue these events rather than send them immediately in order to break
- * the dependency in our callgraph from code that generates events for the
- * controller, and the network layer at large.  Otherwise, nearly every
- * interesting part of Tor would potentially call every other interesting part
- * of Tor.
- */
-MOCK_IMPL(STATIC void,
-queue_control_event_string,(uint16_t event, char *msg))
-{
-  /* This is redundant with checks done elsewhere, but it's a last-ditch
-   * attempt to avoid queueing something we shouldn't have to queue. */
-  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
-    tor_free(msg);
-    return;
-  }
-
-  int *block_event_queue = get_block_event_queue();
-  if (*block_event_queue) {
-    tor_free(msg);
-    return;
-  }
-
-  queued_event_t *ev = tor_malloc(sizeof(*ev));
-  ev->event = event;
-  ev->msg = msg;
-
-  /* No queueing an event while queueing an event */
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  tor_assert(queued_control_events);
-  smartlist_add(queued_control_events, ev);
-
-  int activate_event = 0;
-  if (! flush_queued_event_pending && in_main_thread()) {
-    activate_event = 1;
-    flush_queued_event_pending = 1;
-  }
-
-  tor_mutex_release(queued_control_events_lock);
-
-  --*block_event_queue;
-
-  /* We just put an event on the queue; mark the queue to be
-   * flushed.  We only do this from the main thread for now; otherwise,
-   * we'd need to incur locking overhead in Libevent or use a socket.
-   */
-  if (activate_event) {
-    tor_assert(flush_queued_events_event);
-    mainloop_event_activate(flush_queued_events_event);
-  }
-}
-
-#define queued_event_free(ev) \
-  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
-
-/** Release all storage held by <b>ev</b>. */
-static void
-queued_event_free_(queued_event_t *ev)
-{
-  if (ev == NULL)
-    return;
-
-  tor_free(ev->msg);
-  tor_free(ev);
-}
-
-/** Send every queued event to every controller that's interested in it,
- * and remove the events from the queue.  If <b>force</b> is true,
- * then make all controllers send their data out immediately, since we
- * may be about to shut down. */
-static void
-queued_events_flush_all(int force)
-{
-  /* Make sure that we get all the pending log events, if there are any. */
-  flush_pending_log_callbacks();
-
-  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
-    return;
-  }
-  smartlist_t *all_conns = get_connection_array();
-  smartlist_t *controllers = smartlist_new();
-  smartlist_t *queued_events;
-
-  int *block_event_queue = get_block_event_queue();
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  /* No queueing an event while flushing events. */
-  flush_queued_event_pending = 0;
-  queued_events = queued_control_events;
-  queued_control_events = smartlist_new();
-  tor_mutex_release(queued_control_events_lock);
-
-  /* Gather all the controllers that will care... */
-  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
-    if (conn->type == CONN_TYPE_CONTROL &&
-        !conn->marked_for_close &&
-        conn->state == CONTROL_CONN_STATE_OPEN) {
-      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
-
-      smartlist_add(controllers, control_conn);
-    }
-  } SMARTLIST_FOREACH_END(conn);
-
-  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
-    const event_mask_t bit = ((event_mask_t)1) << ev->event;
-    const size_t msg_len = strlen(ev->msg);
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      if (control_conn->event_mask & bit) {
-        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
-      }
-    } SMARTLIST_FOREACH_END(control_conn);
-
-    queued_event_free(ev);
-  } SMARTLIST_FOREACH_END(ev);
-
-  if (force) {
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      connection_flush(TO_CONN(control_conn));
-    } SMARTLIST_FOREACH_END(control_conn);
-  }
-
-  smartlist_free(queued_events);
-  smartlist_free(controllers);
-
-  --*block_event_queue;
-}
-
-/** Libevent callback: Flushes pending events to controllers that are
- * interested in them. */
-static void
-flush_queued_events_cb(mainloop_event_t *event, void *arg)
-{
-  (void) event;
-  (void) arg;
-  queued_events_flush_all(0);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is given by <b>msg</b>.
- *
- * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
- * respect to the EXTENDED_EVENTS feature. */
-MOCK_IMPL(STATIC void,
-send_control_event_string,(uint16_t event,
-                           const char *msg))
-{
-  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
-  queue_control_event_string(event, tor_strdup(msg));
-}
-
-/** Helper for send_control_event and control_event_status:
- * Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event_impl(uint16_t event,
-                        const char *format, va_list ap)
-{
-  char *buf = NULL;
-  int len;
-
-  len = tor_vasprintf(&buf, format, ap);
-  if (len < 0) {
-    log_warn(LD_BUG, "Unable to format event for controller.");
-    return;
-  }
-
-  queue_control_event_string(event, buf);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event(uint16_t event,
-                   const char *format, ...)
-{
-  va_list ap;
-  va_start(ap, format);
-  send_control_event_impl(event, format, ap);
-  va_end(ap);
-}
-
-/** Given a text circuit <b>id</b>, return the corresponding circuit. */
-static origin_circuit_t *
-get_circ(const char *id)
-{
-  uint32_t n_id;
-  int ok;
-  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  return circuit_get_by_global_id(n_id);
-}
-
-/** Given a text stream <b>id</b>, return the corresponding AP connection. */
-static entry_connection_t *
-get_stream(const char *id)
-{
-  uint64_t n_id;
-  int ok;
-  connection_t *conn;
-  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  conn = connection_get_by_global_id(n_id);
-  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
-    return NULL;
-  return TO_ENTRY_CONN(conn);
-}
-
-/** Helper for setconf and resetconf. Acts like setconf, except
- * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
- * contents of body.
- */
-static int
-control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
-                       int use_defaults)
-{
-  setopt_err_t opt_err;
-  config_line_t *lines=NULL;
-  char *start = body;
-  char *errstring = NULL;
-  const unsigned flags =
-    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
-
-  char *config;
-  smartlist_t *entries = smartlist_new();
-
-  /* We have a string, "body", of the format '(key(=val|="val")?)' entries
-   * separated by space.  break it into a list of configuration entries. */
-  while (*body) {
-    char *eq = body;
-    char *key;
-    char *entry;
-    while (!TOR_ISSPACE(*eq) && *eq != '=')
-      ++eq;
-    key = tor_strndup(body, eq-body);
-    body = eq+1;
-    if (*eq == '=') {
-      char *val=NULL;
-      size_t val_len=0;
-      if (*body != '\"') {
-        char *val_start = body;
-        while (!TOR_ISSPACE(*body))
-          body++;
-        val = tor_strndup(val_start, body-val_start);
-        val_len = strlen(val);
-      } else {
-        body = (char*)extract_escaped_string(body, (len - (body-start)),
-                                             &val, &val_len);
-        if (!body) {
-          connection_write_str_to_buf("551 Couldn't parse string\r\n", conn);
-          SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-          smartlist_free(entries);
-          tor_free(key);
-          return 0;
-        }
-      }
-      tor_asprintf(&entry, "%s %s", key, val);
-      tor_free(key);
-      tor_free(val);
-    } else {
-      entry = key;
-    }
-    smartlist_add(entries, entry);
-    while (TOR_ISSPACE(*body))
-      ++body;
-  }
-
-  smartlist_add_strdup(entries, "");
-  config = smartlist_join_strings(entries, "\n", 0, NULL);
-  SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-  smartlist_free(entries);
-
-  if (config_get_lines(config, &lines, 0) < 0) {
-    log_warn(LD_CONTROL,"Controller gave us config lines we can't parse.");
-    connection_write_str_to_buf("551 Couldn't parse configuration\r\n",
-                                conn);
-    tor_free(config);
-    return 0;
-  }
-  tor_free(config);
-
-  opt_err = options_trial_assign(lines, flags, &errstring);
-  {
-    const char *msg;
-    switch (opt_err) {
-      case SETOPT_ERR_MISC:
-        msg = "552 Unrecognized option";
-        break;
-      case SETOPT_ERR_PARSE:
-        msg = "513 Unacceptable option value";
-        break;
-      case SETOPT_ERR_TRANSITION:
-        msg = "553 Transition not allowed";
-        break;
-      case SETOPT_ERR_SETTING:
-      default:
-        msg = "553 Unable to set option";
-        break;
-      case SETOPT_OK:
-        config_free_lines(lines);
-        send_control_done(conn);
-        return 0;
-    }
-    log_warn(LD_CONTROL,
-             "Controller gave us config lines that didn't validate: %s",
-             errstring);
-    connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    config_free_lines(lines);
-    tor_free(errstring);
-    return 0;
-  }
-}
-
-/** Called when we receive a SETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body.*/
-static int
-handle_control_setconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 0);
-}
-
-/** Called when we receive a RESETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body. */
-static int
-handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 1);
-}
-
-/** Called when we receive a GETCONF message.  Parse the request, and
- * reply with a CONFVALUE or an ERROR message */
-static int
-handle_control_getconf(control_connection_t *conn, uint32_t body_len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *msg = NULL;
-  size_t msg_len;
-  const or_options_t *options = get_options();
-  int i, len;
-
-  (void) body_len; /* body is NUL-terminated; so we can ignore len. */
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    if (!option_is_recognized(q)) {
-      smartlist_add(unrecognized, (char*) q);
-    } else {
-      config_line_t *answer = option_get_assignment(options,q);
-      if (!answer) {
-        const char *name = option_get_canonical_name(q);
-        smartlist_add_asprintf(answers, "250-%s\r\n", name);
-      }
-
-      while (answer) {
-        config_line_t *next;
-        smartlist_add_asprintf(answers, "250-%s=%s\r\n",
-                     answer->key, answer->value);
-
-        next = answer->next;
-        tor_free(answer->key);
-        tor_free(answer->value);
-        tor_free(answer);
-        answer = next;
-      }
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if ((len = smartlist_len(unrecognized))) {
-    for (i=0; i < len-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-Unrecognized configuration key \"%s\"\r\n",
-                               (char*)smartlist_get(unrecognized, i));
-    connection_printf_to_buf(conn,
-                             "552 Unrecognized configuration key \"%s\"\r\n",
-                             (char*)smartlist_get(unrecognized, len-1));
-  } else if ((len = smartlist_len(answers))) {
-    char *tmp = smartlist_get(answers, len-1);
-    tor_assert(strlen(tmp)>4);
-    tmp[3] = ' ';
-    msg = smartlist_join_strings(answers, "", 0, &msg_len);
-    connection_buf_add(msg, msg_len, TO_CONN(conn));
-  } else {
-    connection_write_str_to_buf("250 OK\r\n", conn);
-  }
-
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  smartlist_free(unrecognized);
-
-  tor_free(msg);
-
-  return 0;
-}
-
-/** Called when we get a +LOADCONF message. */
-static int
-handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  setopt_err_t retval;
-  char *errstring = NULL;
-  const char *msg = NULL;
-  (void) len;
-
-  retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring);
-
-  if (retval != SETOPT_OK)
-    log_warn(LD_CONTROL,
-             "Controller gave us config file that didn't validate: %s",
-             errstring);
-
-  switch (retval) {
-  case SETOPT_ERR_PARSE:
-    msg = "552 Invalid config file";
-    break;
-  case SETOPT_ERR_TRANSITION:
-    msg = "553 Transition not allowed";
-    break;
-  case SETOPT_ERR_SETTING:
-    msg = "553 Unable to set option";
-    break;
-  case SETOPT_ERR_MISC:
-  default:
-    msg = "550 Unable to load config";
-    break;
-  case SETOPT_OK:
-    break;
-  }
-  if (msg) {
-    if (errstring)
-      connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    else
-      connection_printf_to_buf(conn, "%s\r\n", msg);
-  } else {
-    send_control_done(conn);
-  }
-  tor_free(errstring);
-  return 0;
-}
-
-/** Helper structure: maps event values to their names. */
-struct control_event_t {
-  uint16_t event_code;
-  const char *event_name;
-};
-/** Table mapping event values to their names.  Used to implement SETEVENTS
- * and GETINFO events/names, and to keep they in sync. */
-static const struct control_event_t control_event_table[] = {
-  { EVENT_CIRCUIT_STATUS, "CIRC" },
-  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
-  { EVENT_STREAM_STATUS, "STREAM" },
-  { EVENT_OR_CONN_STATUS, "ORCONN" },
-  { EVENT_BANDWIDTH_USED, "BW" },
-  { EVENT_DEBUG_MSG, "DEBUG" },
-  { EVENT_INFO_MSG, "INFO" },
-  { EVENT_NOTICE_MSG, "NOTICE" },
-  { EVENT_WARN_MSG, "WARN" },
-  { EVENT_ERR_MSG, "ERR" },
-  { EVENT_NEW_DESC, "NEWDESC" },
-  { EVENT_ADDRMAP, "ADDRMAP" },
-  { EVENT_DESCCHANGED, "DESCCHANGED" },
-  { EVENT_NS, "NS" },
-  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
-  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
-  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
-  { EVENT_GUARD, "GUARD" },
-  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
-  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
-  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
-  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
-  { EVENT_GOT_SIGNAL, "SIGNAL" },
-  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
-  { EVENT_CONN_BW, "CONN_BW" },
-  { EVENT_CELL_STATS, "CELL_STATS" },
-  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
-  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
-  { EVENT_HS_DESC, "HS_DESC" },
-  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
-  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
-  { 0, NULL },
-};
-
-/** Called when we get a SETEVENTS message: update conn->event_mask,
- * and reply with DONE or ERROR. */
-static int
-handle_control_setevents(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  int event_code;
-  event_mask_t event_mask = 0;
-  smartlist_t *events = smartlist_new();
-
-  (void) len;
-
-  smartlist_split_string(events, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
-    {
-      if (!strcasecmp(ev, "EXTENDED") ||
-          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
-        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
-                 "supported.", ev);
-        continue;
-      } else {
-        int i;
-        event_code = -1;
-
-        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-          if (!strcasecmp(ev, control_event_table[i].event_name)) {
-            event_code = control_event_table[i].event_code;
-            break;
-          }
-        }
-
-        if (event_code == -1) {
-          connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n",
-                                   ev);
-          SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-          smartlist_free(events);
-          return 0;
-        }
-      }
-      event_mask |= (((event_mask_t)1) << event_code);
-    }
-  SMARTLIST_FOREACH_END(ev);
-  SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-  smartlist_free(events);
-
-  conn->event_mask = event_mask;
-
-  control_update_global_event_mask();
-  send_control_done(conn);
-  return 0;
-}
-
-/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
- * Return a smartlist of acceptable passwords (unterminated strings of
- * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
- * failure.
- */
-smartlist_t *
-decode_hashed_passwords(config_line_t *passwords)
-{
-  char decoded[64];
-  config_line_t *cl;
-  smartlist_t *sl = smartlist_new();
-
-  tor_assert(passwords);
-
-  for (cl = passwords; cl; cl = cl->next) {
-    const char *hashed = cl->value;
-
-    if (!strcmpstart(hashed, "16:")) {
-      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
-                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
-          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
-        goto err;
-      }
-    } else {
-        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
-            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
-          goto err;
-        }
-    }
-    smartlist_add(sl,
-                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
-  }
-
-  return sl;
-
- err:
-  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
-  smartlist_free(sl);
-  return NULL;
-}
-
-/** Called when we get an AUTHENTICATE message.  Check whether the
- * authentication is valid, and if so, update the connection's state to
- * OPEN.  Reply with DONE or ERROR.
- */
-static int
-handle_control_authenticate(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  int used_quoted_string = 0;
-  const or_options_t *options = get_options();
-  const char *errstr = "Unknown error";
-  char *password;
-  size_t password_len;
-  const char *cp;
-  int i;
-  int bad_cookie=0, bad_password=0;
-  smartlist_t *sl = NULL;
-
-  if (!len) {
-    password = tor_strdup("");
-    password_len = 0;
-  } else if (TOR_ISXDIGIT(body[0])) {
-    cp = body;
-    while (TOR_ISXDIGIT(*cp))
-      ++cp;
-    i = (int)(cp - body);
-    tor_assert(i>0);
-    password_len = i/2;
-    password = tor_malloc(password_len + 1);
-    if (base16_decode(password, password_len+1, body, i)
-                      != (int) password_len) {
-      connection_write_str_to_buf(
-            "551 Invalid hexadecimal encoding.  Maybe you tried a plain text "
-            "password?  If so, the standard requires that you put it in "
-            "double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(password);
-      return 0;
-    }
-  } else {
-    if (!decode_escaped_string(body, len, &password, &password_len)) {
-      connection_write_str_to_buf("551 Invalid quoted string.  You need "
-            "to put the password in double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return 0;
-    }
-    used_quoted_string = 1;
-  }
-
-  if (conn->safecookie_client_hash != NULL) {
-    /* The controller has chosen safe cookie authentication; the only
-     * acceptable authentication value is the controller-to-server
-     * response. */
-
-    tor_assert(authentication_cookie_is_set);
-
-    if (password_len != DIGEST256_LEN) {
-      log_warn(LD_CONTROL,
-               "Got safe cookie authentication response with wrong length "
-               "(%d)", (int)password_len);
-      errstr = "Wrong length for safe cookie response.";
-      goto err;
-    }
-
-    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
-      log_warn(LD_CONTROL,
-               "Got incorrect safe cookie authentication response");
-      errstr = "Safe cookie response did not match expected value.";
-      goto err;
-    }
-
-    tor_free(conn->safecookie_client_hash);
-    goto ok;
-  }
-
-  if (!options->CookieAuthentication && !options->HashedControlPassword &&
-      !options->HashedControlSessionPassword) {
-    /* if Tor doesn't demand any stronger authentication, then
-     * the controller can get in with anything. */
-    goto ok;
-  }
-
-  if (options->CookieAuthentication) {
-    int also_password = options->HashedControlPassword != NULL ||
-      options->HashedControlSessionPassword != NULL;
-    if (password_len != AUTHENTICATION_COOKIE_LEN) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
-                 "(%d)", (int)password_len);
-        errstr = "Wrong length on authentication cookie.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else if (tor_memneq(authentication_cookie, password, password_len)) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
-        errstr = "Authentication cookie did not match expected value.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else {
-      goto ok;
-    }
-  }
-
-  if (options->HashedControlPassword ||
-      options->HashedControlSessionPassword) {
-    int bad = 0;
-    smartlist_t *sl_tmp;
-    char received[DIGEST_LEN];
-    int also_cookie = options->CookieAuthentication;
-    sl = smartlist_new();
-    if (options->HashedControlPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (options->HashedControlSessionPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (bad) {
-      if (!also_cookie) {
-        log_warn(LD_BUG,
-                 "Couldn't decode HashedControlPassword: invalid base16");
-        errstr="Couldn't decode HashedControlPassword value in configuration.";
-        goto err;
-      }
-      bad_password = 1;
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-    } else {
-      SMARTLIST_FOREACH(sl, char *, expected,
-      {
-        secret_to_key_rfc2440(received,DIGEST_LEN,
-                              password,password_len,expected);
-        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
-                      received, DIGEST_LEN))
-          goto ok;
-      });
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-
-      if (used_quoted_string)
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration";
-      else
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration. Maybe you tried a plain text password? "
-          "If so, the standard requires that you put it in double quotes.";
-      bad_password = 1;
-      if (!also_cookie)
-        goto err;
-    }
-  }
-
-  /** We only get here if both kinds of authentication failed. */
-  tor_assert(bad_password && bad_cookie);
-  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
-  errstr = "Password did not match HashedControlPassword *or* authentication "
-    "cookie.";
-
- err:
-  tor_free(password);
-  connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr);
-  connection_mark_for_close(TO_CONN(conn));
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
- ok:
-  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
-           ")", conn->base_.s);
-  send_control_done(conn);
-  conn->base_.state = CONTROL_CONN_STATE_OPEN;
-  tor_free(password);
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
-}
-
-/** Called when we get a SAVECONF command. Try to flush the current options to
- * disk, and report success or failure. */
-static int
-handle_control_saveconf(control_connection_t *conn, uint32_t len,
-                        const char *body)
-{
-  (void) len;
-
-  int force = !strcmpstart(body, "FORCE");
-  const or_options_t *options = get_options();
-  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
-    connection_write_str_to_buf(
-      "551 Unable to write configuration to disk.\r\n", conn);
-  } else {
-    send_control_done(conn);
-  }
-  return 0;
-}
-
-struct signal_t {
-  int sig;
-  const char *signal_name;
-};
-
-static const struct signal_t signal_table[] = {
+const struct signal_name_t signal_table[] = {
+  /* NOTE: this table is used for handling SIGNAL commands and generating
+   * SIGNAL events.  Order is significant: if there are two entries for the
+   * same numeric signal, the first one is the canonical name generated
+   * for the events. */
   { SIGHUP, "RELOAD" },
   { SIGHUP, "HUP" },
   { SIGINT, "SHUTDOWN" },
@@ -1680,3576 +175,11 @@ static const struct signal_t signal_table[] = {
   { SIGNEWNYM, "NEWNYM" },
   { SIGCLEARDNSCACHE, "CLEARDNSCACHE"},
   { SIGHEARTBEAT, "HEARTBEAT"},
+  { SIGACTIVE, "ACTIVE" },
+  { SIGDORMANT, "DORMANT" },
   { 0, NULL },
 };
 
-/** Called when we get a SIGNAL command. React to the provided signal, and
- * report success or failure. (If the signal results in a shutdown, success
- * may not be reported.) */
-static int
-handle_control_signal(control_connection_t *conn, uint32_t len,
-                      const char *body)
-{
-  int sig = -1;
-  int i;
-  int n = 0;
-  char *s;
-
-  (void) len;
-
-  while (body[n] && ! TOR_ISSPACE(body[n]))
-    ++n;
-  s = tor_strndup(body, n);
-
-  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
-    if (!strcasecmp(s, signal_table[i].signal_name)) {
-      sig = signal_table[i].sig;
-      break;
-    }
-  }
-
-  if (sig < 0)
-    connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n",
-                             s);
-  tor_free(s);
-  if (sig < 0)
-    return 0;
-
-  send_control_done(conn);
-  /* Flush the "done" first if the signal might make us shut down. */
-  if (sig == SIGTERM || sig == SIGINT)
-    connection_flush(TO_CONN(conn));
-
-  activate_signal(sig);
-
-  return 0;
-}
-
-/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
- * as an owning connection, so that we will exit if the connection
- * closes. */
-static int
-handle_control_takeownership(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  (void)len;
-  (void)body;
-
-  conn->is_owning_control_connection = 1;
-
-  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
-           "Tor instance.",
-           (int)(conn->base_.s));
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
- * containing funny characters. */
-static int
-address_is_invalid_mapaddress_target(const char *addr)
-{
-  if (!strcmpstart(addr, "*."))
-    return address_is_invalid_destination(addr+2, 1);
-  else
-    return address_is_invalid_destination(addr, 1);
-}
-
-/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
- * and report success or failure. */
-static int
-handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                          const char *body)
-{
-  smartlist_t *elts;
-  smartlist_t *lines;
-  smartlist_t *reply;
-  char *r;
-  size_t sz;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  lines = smartlist_new();
-  elts = smartlist_new();
-  reply = smartlist_new();
-  smartlist_split_string(lines, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(lines, char *, line) {
-    tor_strlower(line);
-    smartlist_split_string(elts, line, "=", 0, 2);
-    if (smartlist_len(elts) == 2) {
-      const char *from = smartlist_get(elts,0);
-      const char *to = smartlist_get(elts,1);
-      if (address_is_invalid_mapaddress_target(to)) {
-        smartlist_add_asprintf(reply,
-                     "512-syntax error: invalid address '%s'", to);
-        log_warn(LD_CONTROL,
-                 "Skipping invalid argument '%s' in MapAddress msg", to);
-      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
-                 !strcmp(from, "::")) {
-        const char type =
-          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
-          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
-        const char *address = addressmap_register_virtual_address(
-                                                     type, tor_strdup(to));
-        if (!address) {
-          smartlist_add_asprintf(reply,
-                       "451-resource exhausted: skipping '%s'", line);
-          log_warn(LD_CONTROL,
-                   "Unable to allocate address for '%s' in MapAddress msg",
-                   safe_str_client(line));
-        } else {
-          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
-        }
-      } else {
-        const char *msg;
-        if (addressmap_register_auto(from, to, 1,
-                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
-          smartlist_add_asprintf(reply,
-                                 "512-syntax error: invalid address mapping "
-                                 " '%s': %s", line, msg);
-          log_warn(LD_CONTROL,
-                   "Skipping invalid argument '%s' in MapAddress msg: %s",
-                   line, msg);
-        } else {
-          smartlist_add_asprintf(reply, "250-%s", line);
-        }
-      }
-    } else {
-      smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is "
-                   "not of expected form 'foo=bar'.", line);
-      log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong "
-                           "number of items.",
-                           safe_str_client(line));
-    }
-    SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
-    smartlist_clear(elts);
-  } SMARTLIST_FOREACH_END(line);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  smartlist_free(elts);
-
-  if (smartlist_len(reply)) {
-    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
-    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
-    connection_buf_add(r, sz, TO_CONN(conn));
-    tor_free(r);
-  } else {
-    const char *response =
-      "512 syntax error: not enough arguments to mapaddress.\r\n";
-    connection_buf_add(response, strlen(response), TO_CONN(conn));
-  }
-
-  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
-  smartlist_free(reply);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for various
- * trivial-to-implement questions. */
-static int
-getinfo_helper_misc(control_connection_t *conn, const char *question,
-                    char **answer, const char **errmsg)
-{
-  (void) conn;
-  if (!strcmp(question, "version")) {
-    *answer = tor_strdup(get_version());
-  } else if (!strcmp(question, "bw-event-cache")) {
-    *answer = get_bw_samples();
-  } else if (!strcmp(question, "config-file")) {
-    const char *a = get_torrc_fname(0);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-defaults-file")) {
-    const char *a = get_torrc_fname(1);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-text")) {
-    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
-  } else if (!strcmp(question, "config-can-saveconf")) {
-    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
-  } else if (!strcmp(question, "info/names")) {
-    *answer = list_getinfo_options();
-  } else if (!strcmp(question, "dormant")) {
-    int dormant = rep_hist_circbuilding_dormant(time(NULL));
-    *answer = tor_strdup(dormant ? "1" : "0");
-  } else if (!strcmp(question, "events/names")) {
-    int i;
-    smartlist_t *event_names = smartlist_new();
-
-    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-      smartlist_add(event_names, (char *)control_event_table[i].event_name);
-    }
-
-    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
-
-    smartlist_free(event_names);
-  } else if (!strcmp(question, "signal/names")) {
-    smartlist_t *signal_names = smartlist_new();
-    int j;
-    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
-      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
-    }
-
-    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
-
-    smartlist_free(signal_names);
-  } else if (!strcmp(question, "features/names")) {
-    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
-  } else if (!strcmp(question, "address")) {
-    uint32_t addr;
-    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
-      *errmsg = "Address unknown";
-      return -1;
-    }
-    *answer = tor_dup_ip(addr);
-  } else if (!strcmp(question, "traffic/read")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
-  } else if (!strcmp(question, "traffic/written")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
-  } else if (!strcmp(question, "uptime")) {
-    long uptime_secs = get_uptime();
-    tor_asprintf(answer, "%ld", uptime_secs);
-  } else if (!strcmp(question, "process/pid")) {
-    int myPid = -1;
-
-#ifdef _WIN32
-      myPid = _getpid();
-#else
-      myPid = getpid();
-#endif
-
-    tor_asprintf(answer, "%d", myPid);
-  } else if (!strcmp(question, "process/uid")) {
-#ifdef _WIN32
-      *answer = tor_strdup("-1");
-#else
-      int myUid = geteuid();
-      tor_asprintf(answer, "%d", myUid);
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/user")) {
-#ifdef _WIN32
-      *answer = tor_strdup("");
-#else
-      int myUid = geteuid();
-      const struct passwd *myPwEntry = tor_getpwuid(myUid);
-
-      if (myPwEntry) {
-        *answer = tor_strdup(myPwEntry->pw_name);
-      } else {
-        *answer = tor_strdup("");
-      }
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/descriptor-limit")) {
-    int max_fds = get_max_sockets();
-    tor_asprintf(answer, "%d", max_fds);
-  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
-    tor_asprintf(answer, "%"PRIu64,
-                 (get_options()->MaxMemInQueues));
-  } else if (!strcmp(question, "fingerprint")) {
-    crypto_pk_t *server_key;
-    if (!server_mode(get_options())) {
-      *errmsg = "Not running in server mode";
-      return -1;
-    }
-    server_key = get_server_identity_key();
-    *answer = tor_malloc(HEX_DIGEST_LEN+1);
-    crypto_pk_get_fingerprint(server_key, *answer, 0);
-  }
-  return 0;
-}
-
-/** Awful hack: return a newly allocated string based on a routerinfo and
- * (possibly) an extrainfo, sticking the read-history and write-history from
- * <b>ei</b> into the resulting string.  The thing you get back won't
- * necessarily have a valid signature.
- *
- * New code should never use this; it's for backward compatibility.
- *
- * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
- * not be NUL-terminated. */
-static char *
-munge_extrainfo_into_routerinfo(const char *ri_body,
-                                const signed_descriptor_t *ri,
-                                const signed_descriptor_t *ei)
-{
-  char *out = NULL, *outp;
-  int i;
-  const char *router_sig;
-  const char *ei_body = signed_descriptor_get_body(ei);
-  size_t ri_len = ri->signed_descriptor_len;
-  size_t ei_len = ei->signed_descriptor_len;
-  if (!ei_body)
-    goto bail;
-
-  outp = out = tor_malloc(ri_len+ei_len+1);
-  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
-    goto bail;
-  ++router_sig;
-  memcpy(out, ri_body, router_sig-ri_body);
-  outp += router_sig-ri_body;
-
-  for (i=0; i < 2; ++i) {
-    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
-    const char *cp, *eol;
-    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
-      continue;
-    ++cp;
-    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
-      continue;
-    memcpy(outp, cp, eol-cp+1);
-    outp += eol-cp+1;
-  }
-  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
-  *outp++ = '\0';
-  tor_assert(outp-out < (int)(ri_len+ei_len+1));
-
-  return out;
- bail:
-  tor_free(out);
-  return tor_strndup(ri_body, ri->signed_descriptor_len);
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * which ports are bound. */
-static int
-getinfo_helper_listeners(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  int type;
-  smartlist_t *res;
-
-  (void)control_conn;
-  (void)errmsg;
-
-  if (!strcmp(question, "net/listeners/or"))
-    type = CONN_TYPE_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/extor"))
-    type = CONN_TYPE_EXT_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/dir"))
-    type = CONN_TYPE_DIR_LISTENER;
-  else if (!strcmp(question, "net/listeners/socks"))
-    type = CONN_TYPE_AP_LISTENER;
-  else if (!strcmp(question, "net/listeners/trans"))
-    type = CONN_TYPE_AP_TRANS_LISTENER;
-  else if (!strcmp(question, "net/listeners/natd"))
-    type = CONN_TYPE_AP_NATD_LISTENER;
-  else if (!strcmp(question, "net/listeners/httptunnel"))
-    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
-  else if (!strcmp(question, "net/listeners/dns"))
-    type = CONN_TYPE_AP_DNS_LISTENER;
-  else if (!strcmp(question, "net/listeners/control"))
-    type = CONN_TYPE_CONTROL_LISTENER;
-  else
-    return 0; /* unknown key */
-
-  res = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
-    struct sockaddr_storage ss;
-    socklen_t ss_len = sizeof(ss);
-
-    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
-      continue;
-
-    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
-      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
-    } else {
-      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
-      smartlist_add(res, esc_for_log(tmp));
-      tor_free(tmp);
-    }
-
-  } SMARTLIST_FOREACH_END(conn);
-
-  *answer = smartlist_join_strings(res, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
-  smartlist_free(res);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * the current time in both local and UTC forms. */
-STATIC int
-getinfo_helper_current_time(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-
-  struct timeval now;
-  tor_gettimeofday(&now);
-  char timebuf[ISO_TIME_LEN+1];
-
-  if (!strcmp(question, "current-time/local"))
-    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else if (!strcmp(question, "current-time/utc"))
-    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else
-    return 0;
-
-  *answer = tor_strdup(timebuf);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * directory information. */
-STATIC int
-getinfo_helper_dir(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  (void) control_conn;
-  if (!strcmpstart(question, "desc/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/id/* "
-                "instead of desc/id/*.";
-      return 0;
-    }
-  } else if (!strcmpstart(question, "desc/name/")) {
-    const routerinfo_t *ri = NULL;
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node =
-      node_get_by_nickname(question+strlen("desc/name/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/name/* "
-                "instead of desc/name/*.";
-      return 0;
-    }
-  } else if (!strcmp(question, "desc/download-enabled")) {
-    int r = we_fetch_router_descriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmp(question, "desc/all-recent")) {
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
-      {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        if (body)
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-      });
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
-    /* XXXX Remove this once Torstat asks for extrainfos. */
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
-                                     ri->cache_info.extra_info_digest);
-        if (ei && body) {
-          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
-                                                        &ri->cache_info, ei));
-        } else if (body) {
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-        }
-      } SMARTLIST_FOREACH_END(ri);
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/client/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      rend_cache_entry_t *e = NULL;
-      if (!rend_cache_lookup_entry(question, -1, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      const char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_cache_lookup_encoded_as_client(&service_pk);
-      if (desc) {
-        *answer = tor_strdup(desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/service/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-    rend_cache_entry_t *e = NULL;
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_service_lookup_current_desc(&service_pk);
-      if (desc) {
-        /* Newly allocated string, we have ownership. */
-        *answer = desc;
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "md/all")) {
-    const smartlist_t *nodes = nodelist_get_list();
-    tor_assert(nodes);
-
-    if (smartlist_len(nodes) == 0) {
-      *answer = tor_strdup("");
-      return 0;
-    }
-
-    smartlist_t *microdescs = smartlist_new();
-
-    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
-      if (n->md && n->md->body) {
-        char *copy = tor_strndup(n->md->body, n->md->bodylen);
-        smartlist_add(microdescs, copy);
-      }
-    } SMARTLIST_FOREACH_END(n);
-
-    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
-    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
-    smartlist_free(microdescs);
-  } else if (!strcmpstart(question, "md/id/")) {
-    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmpstart(question, "md/name/")) {
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
-    /* XXXX duplicated code */
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmp(question, "md/download-enabled")) {
-    int r = we_fetch_microdescriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmpstart(question, "desc-annotations/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node =
-      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *annotations =
-        signed_descriptor_get_annotations(&ri->cache_info);
-      if (annotations)
-        *answer = tor_strndup(annotations,
-                              ri->cache_info.annotations_len);
-    }
-  } else if (!strcmpstart(question, "dir/server/")) {
-    size_t answer_len = 0;
-    char *url = NULL;
-    smartlist_t *descs = smartlist_new();
-    const char *msg;
-    int res;
-    char *cp;
-    tor_asprintf(&url, "/tor/%s", question+4);
-    res = dirserv_get_routerdescs(descs, url, &msg);
-    if (res) {
-      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
-      smartlist_free(descs);
-      tor_free(url);
-      *errmsg = msg;
-      return -1;
-    }
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      answer_len += sd->signed_descriptor_len);
-    cp = *answer = tor_malloc(answer_len+1);
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      {
-                        memcpy(cp, signed_descriptor_get_body(sd),
-                               sd->signed_descriptor_len);
-                        cp += sd->signed_descriptor_len;
-                      });
-    *cp = '\0';
-    tor_free(url);
-    smartlist_free(descs);
-  } else if (!strcmpstart(question, "dir/status/")) {
-    *answer = tor_strdup("");
-  } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
-    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
-      const cached_dir_t *consensus = dirserv_get_consensus("ns");
-      if (consensus)
-        *answer = tor_strdup(consensus->dir);
-    }
-    if (!*answer) { /* try loading it from disk */
-      *answer = networkstatus_read_cached_consensus("ns");
-      if (!*answer) { /* generate an error */
-        *errmsg = "Could not open cached consensus. "
-          "Make sure FetchUselessDescriptors is set to 1.";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "network-status")) { /* v1 */
-    static int network_status_warned = 0;
-    if (!network_status_warned) {
-      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
-               "go away in a future version of Tor.");
-      network_status_warned = 1;
-    }
-    routerlist_t *routerlist = router_get_routerlist();
-    if (!routerlist || !routerlist->routers ||
-        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
-      return -1;
-    }
-  } else if (!strcmpstart(question, "extra-info/digest/")) {
-    question += strlen("extra-info/digest/");
-    if (strlen(question) == HEX_DIGEST_LEN) {
-      char d[DIGEST_LEN];
-      signed_descriptor_t *sd = NULL;
-      if (base16_decode(d, sizeof(d), question, strlen(question))
-                        == sizeof(d)) {
-        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
-         * but I don't want to risk affecting other parts of the code,
-         * especially since the rules for using our own extrainfo (including
-         * when it might be freed) are different from those for using one
-         * we have downloaded. */
-        if (router_extrainfo_digest_is_me(d))
-          sd = &(router_get_my_extrainfo()->cache_info);
-        else
-          sd = extrainfo_get_by_descriptor_digest(d);
-      }
-      if (sd) {
-        const char *body = signed_descriptor_get_body(sd);
-        if (body)
-          *answer = tor_strndup(body, sd->signed_descriptor_len);
-      }
-    }
-  }
-
-  return 0;
-}
-
-/** Given a smartlist of 20-byte digests, return a newly allocated string
- * containing each of those digests in order, formatted in HEX, and terminated
- * with a newline. */
-static char *
-digest_list_to_string(const smartlist_t *sl)
-{
-  int len;
-  char *result, *s;
-
-  /* Allow for newlines, and a \0 at the end */
-  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
-  result = tor_malloc_zero(len);
-
-  s = result;
-  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
-    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
-    s[HEX_DIGEST_LEN] = '\n';
-    s += HEX_DIGEST_LEN + 1;
-  } SMARTLIST_FOREACH_END(digest);
-  *s = '\0';
-
-  return result;
-}
-
-/** Turn a download_status_t into a human-readable description in a newly
- * allocated string.  The format is specified in control-spec.txt, under
- * the documentation for "GETINFO download/..." .  */
-static char *
-download_status_to_string(const download_status_t *dl)
-{
-  char *rv = NULL;
-  char tbuf[ISO_TIME_LEN+1];
-  const char *schedule_str, *want_authority_str;
-  const char *increment_on_str, *backoff_str;
-
-  if (dl) {
-    /* Get some substrings of the eventual output ready */
-    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
-
-    switch (dl->schedule) {
-      case DL_SCHED_GENERIC:
-        schedule_str = "DL_SCHED_GENERIC";
-        break;
-      case DL_SCHED_CONSENSUS:
-        schedule_str = "DL_SCHED_CONSENSUS";
-        break;
-      case DL_SCHED_BRIDGE:
-        schedule_str = "DL_SCHED_BRIDGE";
-        break;
-      default:
-        schedule_str = "unknown";
-        break;
-    }
-
-    switch (dl->want_authority) {
-      case DL_WANT_ANY_DIRSERVER:
-        want_authority_str = "DL_WANT_ANY_DIRSERVER";
-        break;
-      case DL_WANT_AUTHORITY:
-        want_authority_str = "DL_WANT_AUTHORITY";
-        break;
-      default:
-        want_authority_str = "unknown";
-        break;
-    }
-
-    switch (dl->increment_on) {
-      case DL_SCHED_INCREMENT_FAILURE:
-        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
-        break;
-      case DL_SCHED_INCREMENT_ATTEMPT:
-        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
-        break;
-      default:
-        increment_on_str = "unknown";
-        break;
-    }
-
-    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
-
-    /* Now assemble them */
-    tor_asprintf(&rv,
-                 "next-attempt-at %s\n"
-                 "n-download-failures %u\n"
-                 "n-download-attempts %u\n"
-                 "schedule %s\n"
-                 "want-authority %s\n"
-                 "increment-on %s\n"
-                 "backoff %s\n"
-                 "last-backoff-position %u\n"
-                 "last-delay-used %d\n",
-                 tbuf,
-                 dl->n_download_failures,
-                 dl->n_download_attempts,
-                 schedule_str,
-                 want_authority_str,
-                 increment_on_str,
-                 backoff_str,
-                 dl->last_backoff_position,
-                 dl->last_delay_used);
-  }
-
-  return rv;
-}
-
-/** Handle the consensus download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_networkstatus(const char *flavor,
-                                       download_status_t **dl_to_emit,
-                                       const char **errmsg)
-{
-  /*
-   * We get the one for the current bootstrapped status by default, or
-   * take an extra /bootstrap or /running suffix
-   */
-  if (strcmp(flavor, "ns") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
-  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
-  } else if (strcmp(flavor, "ns/running") == 0 ) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
-  } else if (strcmp(flavor, "microdesc") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/running") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
-  } else {
-    *errmsg = "Unknown flavor";
-  }
-}
-
-/** Handle the cert download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_cert(const char *fp_sk_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  const char *sk_req;
-  char id_digest[DIGEST_LEN];
-  char sk_digest[DIGEST_LEN];
-
-  /*
-   * We have to handle four cases; fp_sk_req is the request with
-   * a prefix of "downloads/cert/" snipped off.
-   *
-   * Case 1: fp_sk_req = "fps"
-   *  - We should emit a digest_list with a list of all the identity
-   *    fingerprints that can be queried for certificate download status;
-   *    get it by calling list_authority_ids_with_downloads().
-   *
-   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
-   *  - We want the default certificate for this identity fingerprint's
-   *    download status; this is the download we get from URLs starting
-   *    in /fp/ on the directory server.  We can get it with
-   *    id_only_download_status_for_authority_id().
-   *
-   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
-   *  - We want a list of all signing key digests for this identity
-   *    fingerprint which can be queried for certificate download status.
-   *    Get it with list_sk_digests_for_authority_id().
-   *
-   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
-   *         signing key digest sk
-   *   - We want the download status for the certificate for this specific
-   *     signing key and fingerprint.  These correspond to the ones we get
-   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
-   *     list_sk_digests_for_authority_id().
-   */
-
-  if (strcmp(fp_sk_req, "fps") == 0) {
-    *digest_list = list_authority_ids_with_downloads();
-    if (!(*digest_list)) {
-      *errmsg = "Failed to get list of authority identity digests (!)";
-    }
-  } else if (!strcmpstart(fp_sk_req, "fp/")) {
-    fp_sk_req += strlen("fp/");
-    /* Okay, look for another / to tell the fp from fp-sk cases */
-    sk_req = strchr(fp_sk_req, '/');
-    if (sk_req) {
-      /* okay, split it here and try to parse <fp> */
-      if (base16_decode(id_digest, DIGEST_LEN,
-                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
-        /* Skip past the '/' */
-        ++sk_req;
-        if (strcmp(sk_req, "sks") == 0) {
-          /* We're asking for the list of signing key fingerprints */
-          *digest_list = list_sk_digests_for_authority_id(id_digest);
-          if (!(*digest_list)) {
-            *errmsg = "Failed to get list of signing key digests for this "
-                      "authority identity digest";
-          }
-        } else {
-          /* We've got a signing key digest */
-          if (base16_decode(sk_digest, DIGEST_LEN,
-                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
-            *dl_to_emit =
-              download_status_for_authority_id_and_sk(id_digest, sk_digest);
-            if (!(*dl_to_emit)) {
-              *errmsg = "Failed to get download status for this identity/"
-                        "signing key digest pair";
-            }
-          } else {
-            *errmsg = "That didn't look like a signing key digest";
-          }
-        }
-      } else {
-        *errmsg = "That didn't look like an identity digest";
-      }
-    } else {
-      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
-      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
-        if (base16_decode(id_digest, DIGEST_LEN,
-                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
-          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
-          if (!(*dl_to_emit)) {
-            *errmsg = "Failed to get download status for this authority "
-                      "identity digest";
-          }
-        } else {
-          *errmsg = "That didn't look like a digest";
-        }
-      } else {
-        *errmsg = "That didn't look like a digest";
-      }
-    }
-  } else {
-    *errmsg = "Unknown certificate download status query";
-  }
-}
-
-/** Handle the routerdesc download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_desc(const char *desc_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  char desc_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: desc_req = "descs"
-   *   - Emit a list of all router descriptor digests, which we get by
-   *     calling router_get_descriptor_digests(); this can return NULL
-   *     if we have no current ns-flavor consensus.
-   *
-   * Case 2: desc_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using router_get_dl_status_by_descriptor_digest().
-   */
-
-  if (strcmp(desc_req, "descs") == 0) {
-    *digest_list = router_get_descriptor_digests();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
-    }
-    /*
-     * Microdescs don't use the download_status_t mechanism, so we don't
-     * answer queries about their downloads here; see microdesc.c.
-     */
-  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(desc_digest, DIGEST_LEN,
-                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such descriptor digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown router descriptor download status query";
-  }
-}
-
-/** Handle the bridge download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_bridge(const char *bridge_req,
-                                download_status_t **dl_to_emit,
-                                smartlist_t **digest_list,
-                                const char **errmsg)
-{
-  char bridge_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: bridge_req = "bridges"
-   *   - Emit a list of all bridge identity digests, which we get by
-   *     calling list_bridge_identities(); this can return NULL if we are
-   *     not using bridges.
-   *
-   * Case 2: bridge_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using get_bridge_dl_status_by_id().
-   */
-
-  if (strcmp(bridge_req, "bridges") == 0) {
-    *digest_list = list_bridge_identities();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to be using bridges";
-    }
-  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(bridge_digest, DIGEST_LEN,
-                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such bridge identity digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown bridge descriptor download status query";
-  }
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * download status information. */
-STATIC int
-getinfo_helper_downloads(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  download_status_t *dl_to_emit = NULL;
-  smartlist_t *digest_list = NULL;
-
-  /* Assert args are sane */
-  tor_assert(control_conn != NULL);
-  tor_assert(question != NULL);
-  tor_assert(answer != NULL);
-  tor_assert(errmsg != NULL);
-
-  /* We check for this later to see if we should supply a default */
-  *errmsg = NULL;
-
-  /* Are we after networkstatus downloads? */
-  if (!strcmpstart(question, "downloads/networkstatus/")) {
-    getinfo_helper_downloads_networkstatus(
-        question + strlen("downloads/networkstatus/"),
-        &dl_to_emit, errmsg);
-  /* Certificates? */
-  } else if (!strcmpstart(question, "downloads/cert/")) {
-    getinfo_helper_downloads_cert(
-        question + strlen("downloads/cert/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Router descriptors? */
-  } else if (!strcmpstart(question, "downloads/desc/")) {
-    getinfo_helper_downloads_desc(
-        question + strlen("downloads/desc/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Bridge descriptors? */
-  } else if (!strcmpstart(question, "downloads/bridge/")) {
-    getinfo_helper_downloads_bridge(
-        question + strlen("downloads/bridge/"),
-        &dl_to_emit, &digest_list, errmsg);
-  } else {
-    *errmsg = "Unknown download status query";
-  }
-
-  if (dl_to_emit) {
-    *answer = download_status_to_string(dl_to_emit);
-
-    return 0;
-  } else if (digest_list) {
-    *answer = digest_list_to_string(digest_list);
-    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
-    smartlist_free(digest_list);
-
-    return 0;
-  } else {
-    if (!(*errmsg)) {
-      *errmsg = "Unknown error";
-    }
-
-    return -1;
-  }
-}
-
-/** Allocate and return a description of <b>circ</b>'s current status,
- * including its path (if any). */
-static char *
-circuit_describe_status_for_controller(origin_circuit_t *circ)
-{
-  char *rv;
-  smartlist_t *descparts = smartlist_new();
-
-  {
-    char *vpath = circuit_list_path_for_controller(circ);
-    if (*vpath) {
-      smartlist_add(descparts, vpath);
-    } else {
-      tor_free(vpath); /* empty path; don't put an extra space in the result */
-    }
-  }
-
-  {
-    cpath_build_state_t *build_state = circ->build_state;
-    smartlist_t *flaglist = smartlist_new();
-    char *flaglist_joined;
-
-    if (build_state->onehop_tunnel)
-      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
-    if (build_state->is_internal)
-      smartlist_add(flaglist, (void *)"IS_INTERNAL");
-    if (build_state->need_capacity)
-      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
-    if (build_state->need_uptime)
-      smartlist_add(flaglist, (void *)"NEED_UPTIME");
-
-    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
-    if (smartlist_len(flaglist)) {
-      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
-
-      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
-
-      tor_free(flaglist_joined);
-    }
-
-    smartlist_free(flaglist);
-  }
-
-  smartlist_add_asprintf(descparts, "PURPOSE=%s",
-                    circuit_purpose_to_controller_string(circ->base_.purpose));
-
-  {
-    const char *hs_state =
-      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
-
-    if (hs_state != NULL) {
-      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
-    }
-  }
-
-  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
-    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
-    const char *onion_address;
-    if (circ->rend_data) {
-      onion_address = rend_data_get_address(circ->rend_data);
-    } else {
-      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
-      onion_address = addr;
-    }
-    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
-  }
-
-  {
-    char tbuf[ISO_TIME_USEC_LEN+1];
-    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
-
-    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
-  }
-
-  // Show username and/or password if available.
-  if (circ->socks_username_len > 0) {
-    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
-                                     (size_t) circ->socks_username_len);
-    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
-                           socks_username_escaped);
-    tor_free(socks_username_escaped);
-  }
-  if (circ->socks_password_len > 0) {
-    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
-                                     (size_t) circ->socks_password_len);
-    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
-                           socks_password_escaped);
-    tor_free(socks_password_escaped);
-  }
-
-  rv = smartlist_join_strings(descparts, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
-  smartlist_free(descparts);
-
-  return rv;
-}
-
-/** Implementation helper for GETINFO: knows how to generate summaries of the
- * current states of things we send events about. */
-static int
-getinfo_helper_events(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  const or_options_t *options = get_options();
-  (void) control_conn;
-  if (!strcmp(question, "circuit-status")) {
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
-      origin_circuit_t *circ;
-      char *circdesc;
-      const char *state;
-      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
-        continue;
-      circ = TO_ORIGIN_CIRCUIT(circ_);
-
-      if (circ->base_.state == CIRCUIT_STATE_OPEN)
-        state = "BUILT";
-      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
-        state = "GUARD_WAIT";
-      else if (circ->cpath)
-        state = "EXTENDED";
-      else
-        state = "LAUNCHED";
-
-      circdesc = circuit_describe_status_for_controller(circ);
-
-      smartlist_add_asprintf(status, "%lu %s%s%s",
-                   (unsigned long)circ->global_identifier,
-                   state, *circdesc ? " " : "", circdesc);
-      tor_free(circdesc);
-    }
-    SMARTLIST_FOREACH_END(circ_);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "stream-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    char buf[256];
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      entry_connection_t *conn;
-      circuit_t *circ;
-      origin_circuit_t *origin_circ = NULL;
-      if (base_conn->type != CONN_TYPE_AP ||
-          base_conn->marked_for_close ||
-          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
-          base_conn->state == AP_CONN_STATE_NATD_WAIT)
-        continue;
-      conn = TO_ENTRY_CONN(base_conn);
-      switch (base_conn->state)
-        {
-        case AP_CONN_STATE_CONTROLLER_WAIT:
-        case AP_CONN_STATE_CIRCUIT_WAIT:
-          if (conn->socks_request &&
-              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
-            state = "NEWRESOLVE";
-          else
-            state = "NEW";
-          break;
-        case AP_CONN_STATE_RENDDESC_WAIT:
-        case AP_CONN_STATE_CONNECT_WAIT:
-          state = "SENTCONNECT"; break;
-        case AP_CONN_STATE_RESOLVE_WAIT:
-          state = "SENTRESOLVE"; break;
-        case AP_CONN_STATE_OPEN:
-          state = "SUCCEEDED"; break;
-        default:
-          log_warn(LD_BUG, "Asked for stream in unknown state %d",
-                   base_conn->state);
-          continue;
-        }
-      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-      if (circ && CIRCUIT_IS_ORIGIN(circ))
-        origin_circ = TO_ORIGIN_CIRCUIT(circ);
-      write_stream_target_to_buf(conn, buf, sizeof(buf));
-      smartlist_add_asprintf(status, "%lu %s %lu %s",
-                   (unsigned long) base_conn->global_identifier,state,
-                   origin_circ?
-                         (unsigned long)origin_circ->global_identifier : 0ul,
-                   buf);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "orconn-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      char name[128];
-      or_connection_t *conn;
-      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
-        continue;
-      conn = TO_OR_CONN(base_conn);
-      if (conn->base_.state == OR_CONN_STATE_OPEN)
-        state = "CONNECTED";
-      else if (conn->nickname)
-        state = "LAUNCHED";
-      else
-        state = "NEW";
-      orconn_target_get_name(name, sizeof(name), conn);
-      smartlist_add_asprintf(status, "%s %s", name, state);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmpstart(question, "address-mappings/")) {
-    time_t min_e, max_e;
-    smartlist_t *mappings;
-    question += strlen("address-mappings/");
-    if (!strcmp(question, "all")) {
-      min_e = 0; max_e = TIME_MAX;
-    } else if (!strcmp(question, "cache")) {
-      min_e = 2; max_e = TIME_MAX;
-    } else if (!strcmp(question, "config")) {
-      min_e = 0; max_e = 0;
-    } else if (!strcmp(question, "control")) {
-      min_e = 1; max_e = 1;
-    } else {
-      return 0;
-    }
-    mappings = smartlist_new();
-    addressmap_get_mappings(mappings, min_e, max_e, 1);
-    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
-    smartlist_free(mappings);
-  } else if (!strcmpstart(question, "status/")) {
-    /* Note that status/ is not a catch-all for events; there's only supposed
-     * to be a status GETINFO if there's a corresponding STATUS event. */
-    if (!strcmp(question, "status/circuit-established")) {
-      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
-    } else if (!strcmp(question, "status/enough-dir-info")) {
-      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
-    } else if (!strcmp(question, "status/good-server-descriptor") ||
-               !strcmp(question, "status/accepted-server-descriptor")) {
-      /* They're equivalent for now, until we can figure out how to make
-       * good-server-descriptor be what we want. See comment in
-       * control-spec.txt. */
-      *answer = tor_strdup(directories_have_accepted_server_descriptor()
-                           ? "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
-      *answer = tor_strdup(check_whether_orport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
-      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded")) {
-      tor_asprintf(answer, "OR=%d DIR=%d",
-                   check_whether_orport_reachable(options) ? 1 : 0,
-                   check_whether_dirport_reachable(options) ? 1 : 0);
-    } else if (!strcmp(question, "status/bootstrap-phase")) {
-      *answer = tor_strdup(last_sent_bootstrap_message);
-    } else if (!strcmpstart(question, "status/version/")) {
-      int is_server = server_mode(options);
-      networkstatus_t *c = networkstatus_get_latest_consensus();
-      version_status_t status;
-      const char *recommended;
-      if (c) {
-        recommended = is_server ? c->server_versions : c->client_versions;
-        status = tor_version_is_obsolete(VERSION, recommended);
-      } else {
-        recommended = "?";
-        status = VS_UNKNOWN;
-      }
-
-      if (!strcmp(question, "status/version/recommended")) {
-        *answer = tor_strdup(recommended);
-        return 0;
-      }
-      if (!strcmp(question, "status/version/current")) {
-        switch (status)
-          {
-          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
-          case VS_OLD: *answer = tor_strdup("obsolete"); break;
-          case VS_NEW: *answer = tor_strdup("new"); break;
-          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
-          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
-          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
-          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
-          default: tor_fragile_assert();
-          }
-      } else if (!strcmp(question, "status/version/num-versioning") ||
-                 !strcmp(question, "status/version/num-concurring")) {
-        tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO));
-        log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful "
-                 "information", question);
-      }
-    } else if (!strcmp(question, "status/clients-seen")) {
-      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
-      if (!bridge_stats) {
-        *errmsg = "No bridge-client stats available";
-        return -1;
-      }
-      *answer = bridge_stats;
-    } else if (!strcmp(question, "status/fresh-relay-descs")) {
-      if (!server_mode(options)) {
-        *errmsg = "Only relays have descriptors";
-        return -1;
-      }
-      routerinfo_t *r;
-      extrainfo_t *e;
-      if (router_build_fresh_descriptor(&r, &e) < 0) {
-        *errmsg = "Error generating descriptor";
-        return -1;
-      }
-      size_t size = r->cache_info.signed_descriptor_len + 1;
-      if (e) {
-        size += e->cache_info.signed_descriptor_len + 1;
-      }
-      tor_assert(r->cache_info.signed_descriptor_len);
-      char *descs = tor_malloc(size);
-      char *cp = descs;
-      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
-             r->cache_info.signed_descriptor_len);
-      cp += r->cache_info.signed_descriptor_len - 1;
-      if (e) {
-        if (cp[0] == '\0') {
-          cp[0] = '\n';
-        } else if (cp[0] != '\n') {
-          cp[1] = '\n';
-          cp++;
-        }
-        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
-               e->cache_info.signed_descriptor_len);
-        cp += e->cache_info.signed_descriptor_len - 1;
-      }
-      if (cp[0] == '\n') {
-        cp[0] = '\0';
-      } else if (cp[0] != '\0') {
-        cp[1] = '\0';
-      }
-      *answer = descs;
-      routerinfo_free(r);
-      extrainfo_free(e);
-    } else {
-      return 0;
-    }
-  }
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows how to enumerate hidden services
- * created via the control port. */
-STATIC int
-getinfo_helper_onions(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  smartlist_t *onion_list = NULL;
-  (void) errmsg;  /* no errors from this method */
-
-  if (control_conn && !strcmp(question, "onions/current")) {
-    onion_list = control_conn->ephemeral_onion_services;
-  } else if (!strcmp(question, "onions/detached")) {
-    onion_list = detached_onion_services;
-  } else {
-    return 0;
-  }
-  if (!onion_list || smartlist_len(onion_list) == 0) {
-    if (answer) {
-      *answer = tor_strdup("");
-    }
-  } else {
-    if (answer) {
-      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about network
- * liveness. */
-static int
-getinfo_helper_liveness(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-  if (strcmp(question, "network-liveness") == 0) {
-    if (get_cached_network_liveness()) {
-      *answer = tor_strdup("up");
-    } else {
-      *answer = tor_strdup("down");
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about shared random
- * value. */
-static int
-getinfo_helper_sr(control_connection_t *control_conn,
-                  const char *question, char **answer,
-                  const char **errmsg)
-{
-  (void) control_conn;
-  (void) errmsg;
-
-  if (!strcmp(question, "sr/current")) {
-    *answer = sr_get_current_for_control();
-  } else if (!strcmp(question, "sr/previous")) {
-    *answer = sr_get_previous_for_control();
-  }
-  /* Else statement here is unrecognized key so do nothing. */
-
-  return 0;
-}
-
-/** Callback function for GETINFO: on a given control connection, try to
- * answer the question <b>q</b> and store the newly-allocated answer in
- * *<b>a</b>. If an internal error occurs, return -1 and optionally set
- * *<b>error_out</b> to point to an error message to be delivered to the
- * controller. On success, _or if the key is not recognized_, return 0. Do not
- * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
- * to improve the error message.
- */
-typedef int (*getinfo_helper_t)(control_connection_t *,
-                                const char *q, char **a,
-                                const char **error_out);
-
-/** A single item for the GETINFO question-to-answer-function table. */
-typedef struct getinfo_item_t {
-  const char *varname; /**< The value (or prefix) of the question. */
-  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
-                        * this entry is documentation-only. */
-  const char *desc; /**< Description of the variable. */
-  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
-} getinfo_item_t;
-
-#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
-#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
-#define DOC(name, desc) { name, NULL, desc, 0 }
-
-/** Table mapping questions accepted by GETINFO to the functions that know how
- * to answer them. */
-static const getinfo_item_t getinfo_items[] = {
-  ITEM("version", misc, "The current version of Tor."),
-  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
-  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
-  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
-  ITEM("config-text", misc,
-       "Return the string that would be written by a saveconf command."),
-  ITEM("config-can-saveconf", misc,
-       "Is it possible to save the configuration to the \"torrc\" file?"),
-  ITEM("accounting/bytes", accounting,
-       "Number of bytes read/written so far in the accounting interval."),
-  ITEM("accounting/bytes-left", accounting,
-      "Number of bytes left to write/read so far in the accounting interval."),
-  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
-  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
-  ITEM("accounting/interval-start", accounting,
-       "Time when the accounting period starts."),
-  ITEM("accounting/interval-end", accounting,
-       "Time when the accounting period ends."),
-  ITEM("accounting/interval-wake", accounting,
-       "Time to wake up in this accounting period."),
-  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
-  ITEM("entry-guards", entry_guards,
-       "Which nodes are we using as entry guards?"),
-  ITEM("fingerprint", misc, NULL),
-  PREFIX("config/", config, "Current configuration values."),
-  DOC("config/names",
-      "List of configuration options, types, and documentation."),
-  DOC("config/defaults",
-      "List of default values for configuration options. "
-      "See also config/names"),
-  PREFIX("current-time/", current_time, "Current time."),
-  DOC("current-time/local", "Current time on the local system."),
-  DOC("current-time/utc", "Current UTC time."),
-  PREFIX("downloads/networkstatus/", downloads,
-         "Download statuses for networkstatus objects"),
-  DOC("downloads/networkstatus/ns",
-      "Download status for current-mode networkstatus download"),
-  DOC("downloads/networkstatus/ns/bootstrap",
-      "Download status for bootstrap-time networkstatus download"),
-  DOC("downloads/networkstatus/ns/running",
-      "Download status for run-time networkstatus download"),
-  DOC("downloads/networkstatus/microdesc",
-      "Download status for current-mode microdesc download"),
-  DOC("downloads/networkstatus/microdesc/bootstrap",
-      "Download status for bootstrap-time microdesc download"),
-  DOC("downloads/networkstatus/microdesc/running",
-      "Download status for run-time microdesc download"),
-  PREFIX("downloads/cert/", downloads,
-         "Download statuses for certificates, by id fingerprint and "
-         "signing key"),
-  DOC("downloads/cert/fps",
-      "List of authority fingerprints for which any download statuses "
-      "exist"),
-  DOC("downloads/cert/fp/<fp>",
-      "Download status for <fp> with the default signing key; corresponds "
-      "to /fp/ URLs on directory server."),
-  DOC("downloads/cert/fp/<fp>/sks",
-      "List of signing keys for which specific download statuses are "
-      "available for this id fingerprint"),
-  DOC("downloads/cert/fp/<fp>/<sk>",
-      "Download status for <fp> with signing key <sk>; corresponds "
-      "to /fp-sk/ URLs on directory server."),
-  PREFIX("downloads/desc/", downloads,
-         "Download statuses for router descriptors, by descriptor digest"),
-  DOC("downloads/desc/descs",
-      "Return a list of known router descriptor digests"),
-  DOC("downloads/desc/<desc>",
-      "Return a download status for a given descriptor digest"),
-  PREFIX("downloads/bridge/", downloads,
-         "Download statuses for bridge descriptors, by bridge identity "
-         "digest"),
-  DOC("downloads/bridge/bridges",
-      "Return a list of configured bridge identity digests with download "
-      "statuses"),
-  DOC("downloads/bridge/<desc>",
-      "Return a download status for a given bridge identity digest"),
-  ITEM("info/names", misc,
-       "List of GETINFO options, types, and documentation."),
-  ITEM("events/names", misc,
-       "Events that the controller can ask for with SETEVENTS."),
-  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
-  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
-  PREFIX("desc/id/", dir, "Router descriptors by ID."),
-  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
-  ITEM("desc/all-recent", dir,
-       "All non-expired, non-superseded router descriptors."),
-  ITEM("desc/download-enabled", dir,
-       "Do we try to download router descriptors?"),
-  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
-  ITEM("md/all", dir, "All known microdescriptors."),
-  PREFIX("md/id/", dir, "Microdescriptors by ID"),
-  PREFIX("md/name/", dir, "Microdescriptors by name"),
-  ITEM("md/download-enabled", dir,
-       "Do we try to download microdescriptors?"),
-  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
-  PREFIX("hs/client/desc/id", dir,
-         "Hidden Service descriptor in client's cache by onion."),
-  PREFIX("hs/service/desc/id/", dir,
-         "Hidden Service descriptor in services's cache by onion."),
-  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
-  ITEM("ns/all", networkstatus,
-       "Brief summary of router status (v2 directory format)"),
-  PREFIX("ns/id/", networkstatus,
-         "Brief summary of router status by ID (v2 directory format)."),
-  PREFIX("ns/name/", networkstatus,
-         "Brief summary of router status by nickname (v2 directory format)."),
-  PREFIX("ns/purpose/", networkstatus,
-         "Brief summary of router status by purpose (v2 directory format)."),
-  PREFIX("consensus/", networkstatus,
-         "Information about and from the ns consensus."),
-  ITEM("network-status", dir,
-       "Brief summary of router status (v1 directory format)"),
-  ITEM("network-liveness", liveness,
-       "Current opinion on whether the network is live"),
-  ITEM("circuit-status", events, "List of current circuits originating here."),
-  ITEM("stream-status", events,"List of current streams."),
-  ITEM("orconn-status", events, "A list of current OR connections."),
-  ITEM("dormant", misc,
-       "Is Tor dormant (not building circuits because it's idle)?"),
-  PREFIX("address-mappings/", events, NULL),
-  DOC("address-mappings/all", "Current address mappings."),
-  DOC("address-mappings/cache", "Current cached DNS replies."),
-  DOC("address-mappings/config",
-      "Current address mappings from configuration."),
-  DOC("address-mappings/control", "Current address mappings from controller."),
-  PREFIX("status/", events, NULL),
-  DOC("status/circuit-established",
-      "Whether we think client functionality is working."),
-  DOC("status/enough-dir-info",
-      "Whether we have enough up-to-date directory information to build "
-      "circuits."),
-  DOC("status/bootstrap-phase",
-      "The last bootstrap phase status event that Tor sent."),
-  DOC("status/clients-seen",
-      "Breakdown of client countries seen by a bridge."),
-  DOC("status/fresh-relay-descs",
-      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
-  DOC("status/version/recommended", "List of currently recommended versions."),
-  DOC("status/version/current", "Status of the current version."),
-  DOC("status/version/num-versioning", "Number of versioning authorities."),
-  DOC("status/version/num-concurring",
-      "Number of versioning authorities agreeing on the status of the "
-      "current version"),
-  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
-  ITEM("traffic/read", misc,"Bytes read since the process was started."),
-  ITEM("traffic/written", misc,
-       "Bytes written since the process was started."),
-  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
-  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
-  ITEM("process/uid", misc, "User id running the tor process."),
-  ITEM("process/user", misc,
-       "Username under which the tor process is running."),
-  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
-  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
-  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
-  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
-  PREFIX("dir/status/", dir,
-         "v2 networkstatus docs as retrieved from a DirPort."),
-  ITEM("dir/status-vote/current/consensus", dir,
-       "v3 Networkstatus consensus as retrieved from a DirPort."),
-  ITEM("exit-policy/default", policies,
-       "The default value appended to the configured exit policy."),
-  ITEM("exit-policy/reject-private/default", policies,
-       "The default rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate."),
-  ITEM("exit-policy/reject-private/relay", policies,
-       "The relay-specific rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
-  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
-  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
-  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
-  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
-  ITEM("onions/current", onions,
-       "Onion services owned by the current control connection."),
-  ITEM("onions/detached", onions,
-       "Onion services detached from the control connection."),
-  ITEM("sr/current", sr, "Get current shared random value."),
-  ITEM("sr/previous", sr, "Get previous shared random value."),
-  { NULL, NULL, NULL, 0 }
-};
-
-/** Allocate and return a list of recognized GETINFO options. */
-static char *
-list_getinfo_options(void)
-{
-  int i;
-  smartlist_t *lines = smartlist_new();
-  char *ans;
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    if (!getinfo_items[i].desc)
-      continue;
-
-    smartlist_add_asprintf(lines, "%s%s -- %s\n",
-                 getinfo_items[i].varname,
-                 getinfo_items[i].is_prefix ? "*" : "",
-                 getinfo_items[i].desc);
-  }
-  smartlist_sort_strings(lines);
-
-  ans = smartlist_join_strings(lines, "", 0, NULL);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-
-  return ans;
-}
-
-/** Lookup the 'getinfo' entry <b>question</b>, and return
- * the answer in <b>*answer</b> (or NULL if key not recognized).
- * Return 0 if success or unrecognized, or -1 if recognized but
- * internal error. */
-static int
-handle_getinfo_helper(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **err_out)
-{
-  int i;
-  *answer = NULL; /* unrecognized key by default */
-
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    int match;
-    if (getinfo_items[i].is_prefix)
-      match = !strcmpstart(question, getinfo_items[i].varname);
-    else
-      match = !strcmp(question, getinfo_items[i].varname);
-    if (match) {
-      tor_assert(getinfo_items[i].fn);
-      return getinfo_items[i].fn(control_conn, question, answer, err_out);
-    }
-  }
-
-  return 0; /* unrecognized */
-}
-
-/** Called when we receive a GETINFO command.  Try to fetch all requested
- * information, and reply with information or error message. */
-static int
-handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *ans = NULL;
-  int i;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    const char *errmsg = NULL;
-
-    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
-      if (!errmsg)
-        errmsg = "Internal error";
-      connection_printf_to_buf(conn, "551 %s\r\n", errmsg);
-      goto done;
-    }
-    if (!ans) {
-      if (errmsg) /* use provided error message */
-        smartlist_add_strdup(unrecognized, errmsg);
-      else /* use default error message */
-        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
-    } else {
-      smartlist_add_strdup(answers, q);
-      smartlist_add(answers, ans);
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if (smartlist_len(unrecognized)) {
-    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
-    for (i=0; i < smartlist_len(unrecognized)-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-%s\r\n",
-                               (char *)smartlist_get(unrecognized, i));
-
-    connection_printf_to_buf(conn,
-                             "552 %s\r\n",
-                             (char *)smartlist_get(unrecognized, i));
-    goto done;
-  }
-
-  for (i = 0; i < smartlist_len(answers); i += 2) {
-    char *k = smartlist_get(answers, i);
-    char *v = smartlist_get(answers, i+1);
-    if (!strchr(v, '\n') && !strchr(v, '\r')) {
-      connection_printf_to_buf(conn, "250-%s=", k);
-      connection_write_str_to_buf(v, conn);
-      connection_write_str_to_buf("\r\n", conn);
-    } else {
-      char *esc = NULL;
-      size_t esc_len;
-      esc_len = write_escaped_data(v, strlen(v), &esc);
-      connection_printf_to_buf(conn, "250+%s=\r\n", k);
-      connection_buf_add(esc, esc_len, TO_CONN(conn));
-      tor_free(esc);
-    }
-  }
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
-  smartlist_free(unrecognized);
-
-  return 0;
-}
-
-/** Given a string, convert it to a circuit purpose. */
-static uint8_t
-circuit_purpose_from_string(const char *string)
-{
-  if (!strcasecmpstart(string, "purpose="))
-    string += strlen("purpose=");
-
-  if (!strcasecmp(string, "general"))
-    return CIRCUIT_PURPOSE_C_GENERAL;
-  else if (!strcasecmp(string, "controller"))
-    return CIRCUIT_PURPOSE_CONTROLLER;
-  else
-    return CIRCUIT_PURPOSE_UNKNOWN;
-}
-
-/** Return a newly allocated smartlist containing the arguments to the command
- * waiting in <b>body</b>. If there are fewer than <b>min_args</b> arguments,
- * or if <b>max_args</b> is nonnegative and there are more than
- * <b>max_args</b> arguments, send a 512 error to the controller, using
- * <b>command</b> as the command name in the error message. */
-static smartlist_t *
-getargs_helper(const char *command, control_connection_t *conn,
-               const char *body, int min_args, int max_args)
-{
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(args) < min_args) {
-    connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command);
-    goto err;
-  } else if (max_args >= 0 && smartlist_len(args) > max_args) {
-    connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command);
-    goto err;
-  }
-  return args;
- err:
-  SMARTLIST_FOREACH(args, char *, s, tor_free(s));
-  smartlist_free(args);
-  return NULL;
-}
-
-/** Helper.  Return the first element of <b>sl</b> at index <b>start_at</b> or
- * higher that starts with <b>prefix</b>, case-insensitive.  Return NULL if no
- * such element exists. */
-static const char *
-find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix)
-{
-  int i;
-  for (i = start_at; i < smartlist_len(sl); ++i) {
-    const char *elt = smartlist_get(sl, i);
-    if (!strcasecmpstart(elt, prefix))
-      return elt;
-  }
-  return NULL;
-}
-
-/** Helper.  Return true iff s is an argument that we should treat as a
- * key-value pair. */
-static int
-is_keyval_pair(const char *s)
-{
-  /* An argument is a key-value pair if it has an =, and it isn't of the form
-   * $fingeprint=name */
-  return strchr(s, '=') && s[0] != '$';
-}
-
-/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
- * circuit, and report success or failure. */
-static int
-handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  smartlist_t *router_nicknames=NULL, *nodes=NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
-  smartlist_t *args;
-  (void) len;
-
-  router_nicknames = smartlist_new();
-
-  args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1);
-  if (!args)
-    goto done;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,0));
-
-  if (zero_circ) {
-    const char *purp = find_element_starting_with(args, 1, "PURPOSE=");
-
-    if (purp) {
-      intended_purpose = circuit_purpose_from_string(purp);
-      if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-        SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-        smartlist_free(args);
-        goto done;
-      }
-    }
-
-    if ((smartlist_len(args) == 1) ||
-        (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) {
-      // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar"
-      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
-      if (!circ) {
-        connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      } else {
-        connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                  (unsigned long)circ->global_identifier);
-      }
-      SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-      smartlist_free(args);
-      goto done;
-    }
-    // "EXTENDCIRCUIT 0 router1,router2" ||
-    // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo"
-  }
-
-  if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  if (smartlist_len(args) < 2) {
-    connection_printf_to_buf(conn,
-                             "512 syntax error: not enough arguments.\r\n");
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0);
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-
-  nodes = smartlist_new();
-  int first_node = zero_circ;
-  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
-    const node_t *node = node_get_by_nickname(n, 0);
-    if (!node) {
-      connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n);
-      goto done;
-    }
-    if (!node_has_preferred_descriptor(node, first_node)) {
-      connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n);
-      goto done;
-    }
-    smartlist_add(nodes, (void*)node);
-    first_node = 0;
-  } SMARTLIST_FOREACH_END(n);
-  if (!smartlist_len(nodes)) {
-    connection_write_str_to_buf("512 No router names provided\r\n", conn);
-    goto done;
-  }
-
-  if (zero_circ) {
-    /* start a new circuit */
-    circ = origin_circuit_init(intended_purpose, 0);
-  }
-
-  /* now circ refers to something that is ready to be extended */
-  first_node = zero_circ;
-  SMARTLIST_FOREACH(nodes, const node_t *, node,
-  {
-    extend_info_t *info = extend_info_from_node(node, first_node);
-    if (!info) {
-      tor_assert_nonfatal(first_node);
-      log_warn(LD_CONTROL,
-               "controller tried to connect to a node that lacks a suitable "
-               "descriptor, or which doesn't have any "
-               "addresses that are allowed by the firewall configuration; "
-               "circuit marked for closing.");
-      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-    circuit_append_new_exit(circ, info);
-    if (circ->build_state->desired_path_len > 1) {
-      circ->build_state->onehop_tunnel = 0;
-    }
-    extend_info_free(info);
-    first_node = 0;
-  });
-
-  /* now that we've populated the cpath, start extending */
-  if (zero_circ) {
-    int err_reason = 0;
-    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
-      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-  } else {
-    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
-        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
-      int err_reason = 0;
-      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
-      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
-        log_info(LD_CONTROL,
-                 "send_next_onion_skin failed; circuit marked for closing.");
-        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-        connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn);
-        goto done;
-      }
-    }
-  }
-
-  connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                             (unsigned long)circ->global_identifier);
-  if (zero_circ) /* send a 'launched' event, for completeness */
-    control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
- done:
-  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
-  smartlist_free(router_nicknames);
-  smartlist_free(nodes);
-  return 0;
-}
-
-/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
- * circuit and it's a valid purpose, change it. */
-static int
-handle_control_setcircuitpurpose(control_connection_t *conn,
-                                 uint32_t len, const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  uint8_t new_purpose;
-  smartlist_t *args;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1);
-  if (!args)
-    goto done;
-
-  if (!(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    goto done;
-  }
-
-  {
-    const char *purp = find_element_starting_with(args,1,"PURPOSE=");
-    if (!purp) {
-      connection_write_str_to_buf("552 No purpose given\r\n", conn);
-      goto done;
-    }
-    new_purpose = circuit_purpose_from_string(purp);
-    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-      connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-      goto done;
-    }
-  }
-
-  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  if (args) {
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-  }
-  return 0;
-}
-
-/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
- * stream, and report success or failure. */
-static int
-handle_control_attachstream(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  smartlist_t *args;
-  crypt_path_t *cpath=NULL;
-  int hop=0, hop_line_ok=1;
-  (void) len;
-
-  args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,1));
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 1));
-  } else if (circ) {
-    const char *hopstring = find_element_starting_with(args,2,"HOP=");
-    if (hopstring) {
-      hopstring += strlen("HOP=");
-      hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX,
-                                  &hop_line_ok, NULL);
-      if (!hop_line_ok) { /* broken hop line */
-        connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring);
-      }
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok)
-    return 0;
-
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
-    connection_write_str_to_buf(
-                    "555 Connection is not managed by controller.\r\n",
-                    conn);
-    return 0;
-  }
-
-  /* Do we need to detach it first? */
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
-    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
-    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
-    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
-    /* Un-mark it as ending, since we're going to reuse it. */
-    edge_conn->edge_has_sent_end = 0;
-    edge_conn->end_reason = 0;
-    if (tmpcirc)
-      circuit_detach_stream(tmpcirc, edge_conn);
-    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
-    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
-  }
-
-  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
-    connection_write_str_to_buf(
-                    "551 Can't attach stream to non-open origin circuit\r\n",
-                    conn);
-    return 0;
-  }
-  /* Is this a single hop circuit? */
-  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
-    connection_write_str_to_buf(
-               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
-    return 0;
-  }
-
-  if (circ && hop>0) {
-    /* find this hop in the circuit, and set cpath */
-    cpath = circuit_get_cpath_hop(circ, hop);
-    if (!cpath) {
-      connection_printf_to_buf(conn,
-                               "551 Circuit doesn't have %d hops.\r\n", hop);
-      return 0;
-    }
-  }
-  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
-    connection_write_str_to_buf("551 Unable to attach stream\r\n", conn);
-    return 0;
-  }
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
- * descriptor, and report success or failure. */
-static int
-handle_control_postdescriptor(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  char *desc;
-  const char *msg=NULL;
-  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
-  int cache = 0; /* eventually, we may switch this to 1 */
-
-  const char *cp = memchr(body, '\n', len);
-
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  ++cp;
-
-  char *cmdline = tor_memdup_nulterm(body, cp-body);
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, cmdline, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, char *, option) {
-    if (!strcasecmpstart(option, "purpose=")) {
-      option += strlen("purpose=");
-      purpose = router_purpose_from_string(option);
-      if (purpose == ROUTER_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else if (!strcasecmpstart(option, "cache=")) {
-      option += strlen("cache=");
-      if (!strcasecmp(option, "no"))
-        cache = 0;
-      else if (!strcasecmp(option, "yes"))
-        cache = 1;
-      else {
-        connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else { /* unrecognized argument? */
-      connection_printf_to_buf(conn,
-        "512 Unexpected argument \"%s\" to postdescriptor\r\n", option);
-      goto done;
-    }
-  } SMARTLIST_FOREACH_END(option);
-
-  read_escaped_data(cp, len-(cp-body), &desc);
-
-  switch (router_load_single_router(desc, purpose, cache, &msg)) {
-  case -1:
-    if (!msg) msg = "Could not parse descriptor";
-    connection_printf_to_buf(conn, "554 %s\r\n", msg);
-    break;
-  case 0:
-    if (!msg) msg = "Descriptor not added";
-    connection_printf_to_buf(conn, "251 %s\r\n",msg);
-    break;
-  case 1:
-    send_control_done(conn);
-    break;
-  }
-
-  tor_free(desc);
- done:
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  tor_free(cmdline);
-  return 0;
-}
-
-/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
- * address of the named AP stream, and report success or failure. */
-static int
-handle_control_redirectstream(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  char *new_addr = NULL;
-  uint16_t new_port = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
-           || !ap_conn->socks_request) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else {
-    int ok = 1;
-    if (smartlist_len(args) > 2) { /* they included a port too */
-      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
-                                            10, 1, 65535, &ok, NULL);
-    }
-    if (!ok) {
-      connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n",
-                               (char*)smartlist_get(args, 2));
-    } else {
-      new_addr = tor_strdup(smartlist_get(args, 1));
-    }
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!new_addr)
-    return 0;
-
-  strlcpy(ap_conn->socks_request->address, new_addr,
-          sizeof(ap_conn->socks_request->address));
-  if (new_port)
-    ap_conn->socks_request->port = new_port;
-  tor_free(new_addr);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSESTREAM command; try to close the named stream
- * and report success or failure. */
-static int
-handle_control_closestream(control_connection_t *conn, uint32_t len,
-                           const char *body)
-{
-  entry_connection_t *ap_conn=NULL;
-  uint8_t reason=0;
-  smartlist_t *args;
-  int ok;
-  (void) len;
-
-  args = getargs_helper("CLOSESTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  else if (!(ap_conn = get_stream(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
-                                       &ok, NULL);
-    if (!ok) {
-      connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n",
-                               (char*)smartlist_get(args, 1));
-      ap_conn = NULL;
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn)
-    return 0;
-
-  connection_mark_unattached_ap(ap_conn, reason);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
- * and report success or failure. */
-static int
-handle_control_closecircuit(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  int safe = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1);
-  if (!args)
-    return 0;
-
-  if (!(circ=get_circ(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    int i;
-    for (i=1; i < smartlist_len(args); ++i) {
-      if (!strcasecmp(smartlist_get(args, i), "IfUnused"))
-        safe = 1;
-      else
-        log_info(LD_CONTROL, "Skipping unknown option %s",
-                 (char*)smartlist_get(args,i));
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!circ)
-    return 0;
-
-  if (!safe || !circ->p_streams) {
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
-  }
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a RESOLVE command: start trying to resolve
- * the listed addresses. */
-static int
-handle_control_resolve(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *args, *failed;
-  int is_reverse = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-
-  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
-    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
-             "isn't listening for ADDRMAP events.  It probably won't see "
-             "the answer.");
-  }
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  {
-    const char *modearg = find_element_starting_with(args, 0, "mode=");
-    if (modearg && !strcasecmp(modearg, "mode=reverse"))
-      is_reverse = 1;
-  }
-  failed = smartlist_new();
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      if (!is_keyval_pair(arg)) {
-          if (dnsserv_launch_request(arg, is_reverse, conn)<0)
-            smartlist_add(failed, (char*)arg);
-      }
-  });
-
-  send_control_done(conn);
-  SMARTLIST_FOREACH(failed, const char *, arg, {
-      control_event_address_mapped(arg, arg, time(NULL),
-                                   "internal", 0);
-  });
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  smartlist_free(failed);
-  return 0;
-}
-
-/** Called when we get a PROTOCOLINFO command: send back a reply. */
-static int
-handle_control_protocolinfo(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  const char *bad_arg = NULL;
-  smartlist_t *args;
-  (void)len;
-
-  conn->have_sent_protocolinfo = 1;
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      int ok;
-      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
-      if (!ok) {
-        bad_arg = arg;
-        break;
-      }
-    });
-  if (bad_arg) {
-    connection_printf_to_buf(conn, "513 No such version %s\r\n",
-                             escaped(bad_arg));
-    /* Don't tolerate bad arguments when not authenticated. */
-    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
-      connection_mark_for_close(TO_CONN(conn));
-    goto done;
-  } else {
-    const or_options_t *options = get_options();
-    int cookies = options->CookieAuthentication;
-    char *cfile = get_controller_cookie_file_name();
-    char *abs_cfile;
-    char *esc_cfile;
-    char *methods;
-    abs_cfile = make_path_absolute(cfile);
-    esc_cfile = esc_for_log(abs_cfile);
-    {
-      int passwd = (options->HashedControlPassword != NULL ||
-                    options->HashedControlSessionPassword != NULL);
-      smartlist_t *mlist = smartlist_new();
-      if (cookies) {
-        smartlist_add(mlist, (char*)"COOKIE");
-        smartlist_add(mlist, (char*)"SAFECOOKIE");
-      }
-      if (passwd)
-        smartlist_add(mlist, (char*)"HASHEDPASSWORD");
-      if (!cookies && !passwd)
-        smartlist_add(mlist, (char*)"NULL");
-      methods = smartlist_join_strings(mlist, ",", 0, NULL);
-      smartlist_free(mlist);
-    }
-
-    connection_printf_to_buf(conn,
-                             "250-PROTOCOLINFO 1\r\n"
-                             "250-AUTH METHODS=%s%s%s\r\n"
-                             "250-VERSION Tor=%s\r\n"
-                             "250 OK\r\n",
-                             methods,
-                             cookies?" COOKIEFILE=":"",
-                             cookies?esc_cfile:"",
-                             escaped(VERSION));
-    tor_free(methods);
-    tor_free(cfile);
-    tor_free(abs_cfile);
-    tor_free(esc_cfile);
-  }
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Called when we get an AUTHCHALLENGE command. */
-static int
-handle_control_authchallenge(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  const char *cp = body;
-  char *client_nonce;
-  size_t client_nonce_len;
-  char server_hash[DIGEST256_LEN];
-  char server_hash_encoded[HEX_DIGEST256_LEN+1];
-  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
-  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
-
-  cp += strspn(cp, " \t\n\r");
-  if (!strcasecmpstart(cp, "SAFECOOKIE")) {
-    cp += strlen("SAFECOOKIE");
-  } else {
-    connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE "
-                                "authentication\r\n", conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  if (!authentication_cookie_is_set) {
-    connection_write_str_to_buf("515 Cookie authentication is disabled\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp == '"') {
-    const char *newcp =
-      decode_escaped_string(cp, len - (cp - body),
-                            &client_nonce, &client_nonce_len);
-    if (newcp == NULL) {
-      connection_write_str_to_buf("513 Invalid quoted client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return -1;
-    }
-    cp = newcp;
-  } else {
-    size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef");
-
-    client_nonce_len = client_nonce_encoded_len / 2;
-    client_nonce = tor_malloc_zero(client_nonce_len);
-
-    if (base16_decode(client_nonce, client_nonce_len,
-                      cp, client_nonce_encoded_len)
-                      != (int) client_nonce_len) {
-      connection_write_str_to_buf("513 Invalid base16 client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(client_nonce);
-      return -1;
-    }
-
-    cp += client_nonce_encoded_len;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp != '\0' ||
-      cp != body + len) {
-    connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    tor_free(client_nonce);
-    return -1;
-  }
-  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-  /* Now compute and send the server-to-controller response, and the
-   * server's nonce. */
-  tor_assert(authentication_cookie != NULL);
-
-  {
-    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
-                      client_nonce_len +
-                      SAFECOOKIE_SERVER_NONCE_LEN);
-    char *tmp = tor_malloc_zero(tmp_len);
-    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
-    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
-           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-    crypto_hmac_sha256(server_hash,
-                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
-                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    crypto_hmac_sha256(client_hash,
-                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
-                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    conn->safecookie_client_hash = client_hash;
-
-    tor_free(tmp);
-  }
-
-  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
-                server_hash, sizeof(server_hash));
-  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
-                server_nonce, sizeof(server_nonce));
-
-  connection_printf_to_buf(conn,
-                           "250 AUTHCHALLENGE SERVERHASH=%s "
-                           "SERVERNONCE=%s\r\n",
-                           server_hash_encoded,
-                           server_nonce_encoded);
-
-  tor_free(client_nonce);
-  return 0;
-}
-
-/** Called when we get a USEFEATURE command: parse the feature list, and
- * set up the control_connection's options properly. */
-static int
-handle_control_usefeature(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  int bad = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmp(arg, "VERBOSE_NAMES"))
-        ;
-      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
-        ;
-      else {
-        connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n",
-                                 arg);
-        bad = 1;
-        break;
-      }
-  } SMARTLIST_FOREACH_END(arg);
-
-  if (!bad) {
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the DROPGUARDS command. */
-static int
-handle_control_dropguards(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-
-  static int have_warned = 0;
-  if (! have_warned) {
-    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
-             "the risks before using it. It may be removed in a future "
-             "version of Tor.");
-    have_warned = 1;
-  }
-
-  if (smartlist_len(args)) {
-    connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
-  } else {
-    remove_all_entry_guards();
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the HSFETCH command. */
-static int
-handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  int i;
-  char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL;
-  smartlist_t *args = NULL, *hsdirs = NULL;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  static const char *hsfetch_command = "HSFETCH";
-  static const char *v2_str = "v2-";
-  const size_t v2_str_len = strlen(v2_str);
-  rend_data_t *rend_query = NULL;
-
-  /* Make sure we have at least one argument, the HSAddress. */
-  args = getargs_helper(hsfetch_command, conn, body, 1, -1);
-  if (!args) {
-    goto exit;
-  }
-
-  /* Extract the first argument (either HSAddress or DescID). */
-  arg1 = smartlist_get(args, 0);
-  /* Test if it's an HS address without the .onion part. */
-  if (rend_valid_v2_service_id(arg1)) {
-    hsaddress = arg1;
-  } else if (strcmpstart(arg1, v2_str) == 0 &&
-             rend_valid_descriptor_id(arg1 + v2_str_len) &&
-             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
-                           REND_DESC_ID_V2_LEN_BASE32) == 0) {
-    /* We have a well formed version 2 descriptor ID. Keep the decoded value
-     * of the id. */
-    desc_id = digest;
-  } else {
-    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
-                             arg1);
-    goto done;
-  }
-
-  static const char *opt_server = "SERVER=";
-
-  /* Skip first argument because it's the HSAddress or DescID. */
-  for (i = 1; i < smartlist_len(args); ++i) {
-    const char *arg = smartlist_get(args, i);
-    const node_t *node;
-
-    if (!strcasecmpstart(arg, opt_server)) {
-      const char *server;
-
-      server = arg + strlen(opt_server);
-      node = node_get_by_hex_id(server, 0);
-      if (!node) {
-        connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                 server);
-        goto done;
-      }
-      if (!hsdirs) {
-        /* Stores routerstatus_t object for each specified server. */
-        hsdirs = smartlist_new();
-      }
-      /* Valid server, add it to our local list. */
-      smartlist_add(hsdirs, node->rs);
-    } else {
-      connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n",
-                               arg);
-      goto done;
-    }
-  }
-
-  rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
-                                       REND_NO_AUTH);
-  if (rend_query == NULL) {
-    connection_printf_to_buf(conn, "551 Error creating the HS query\r\n");
-    goto done;
-  }
-
-  /* Using a descriptor ID, we force the user to provide at least one
-   * hsdir server using the SERVER= option. */
-  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
-      connection_printf_to_buf(conn, "512 %s option is required\r\n",
-                               opt_server);
-      goto done;
-  }
-
-  /* We are about to trigger HSDir fetch so send the OK now because after
-   * that 650 event(s) are possible so better to have the 250 OK before them
-   * to avoid out of order replies. */
-  send_control_done(conn);
-
-  /* Trigger the fetch using the built rend query and possibly a list of HS
-   * directory to use. This function ignores the client cache thus this will
-   * always send a fetch command. */
-  rend_client_fetch_v2_desc(rend_query, hsdirs);
-
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  /* Contains data pointer that we don't own thus no cleanup. */
-  smartlist_free(hsdirs);
-  rend_data_free(rend_query);
- exit:
-  return 0;
-}
-
-/** Implementation for the HSPOST command. */
-static int
-handle_control_hspost(control_connection_t *conn,
-                      uint32_t len,
-                      const char *body)
-{
-  static const char *opt_server = "SERVER=";
-  static const char *opt_hsaddress = "HSADDRESS=";
-  smartlist_t *hs_dirs = NULL;
-  const char *encoded_desc = body;
-  size_t encoded_desc_len = len;
-  const char *onion_address = NULL;
-
-  char *cp = memchr(body, '\n', len);
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  char *argline = tor_strndup(body, cp-body);
-
-  smartlist_t *args = smartlist_new();
-
-  /* If any SERVER= or HSADDRESS= options were specified, try to parse
-   * the options line. */
-  if (!strcasecmpstart(argline, opt_server) ||
-      !strcasecmpstart(argline, opt_hsaddress)) {
-    /* encoded_desc begins after a newline character */
-    cp = cp + 1;
-    encoded_desc = cp;
-    encoded_desc_len = len-(cp-body);
-
-    smartlist_split_string(args, argline, " ",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmpstart(arg, opt_server)) {
-        const char *server = arg + strlen(opt_server);
-        const node_t *node = node_get_by_hex_id(server, 0);
-
-        if (!node || !node->rs) {
-          connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                   server);
-          goto done;
-        }
-        /* Valid server, add it to our local list. */
-        if (!hs_dirs)
-          hs_dirs = smartlist_new();
-        smartlist_add(hs_dirs, node->rs);
-      } else if (!strcasecmpstart(arg, opt_hsaddress)) {
-        const char *address = arg + strlen(opt_hsaddress);
-        if (!hs_address_is_valid(address)) {
-          connection_printf_to_buf(conn, "512 Malformed onion address\r\n");
-          goto done;
-        }
-        onion_address = address;
-      } else {
-        connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n",
-                                 arg);
-        goto done;
-      }
-    } SMARTLIST_FOREACH_END(arg);
-  }
-
-  /* Handle the v3 case. */
-  if (onion_address) {
-    char *desc_str = NULL;
-    read_escaped_data(encoded_desc, encoded_desc_len, &desc_str);
-    if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) {
-      connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-    } else {
-      send_control_done(conn);
-    }
-    tor_free(desc_str);
-    goto done;
-  }
-
-  /* From this point on, it is only v2. */
-
-  /* Read the dot encoded descriptor, and parse it. */
-  rend_encoded_v2_service_descriptor_t *desc =
-      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
-  read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str);
-
-  rend_service_descriptor_t *parsed = NULL;
-  char *intro_content = NULL;
-  size_t intro_size;
-  size_t encoded_size;
-  const char *next_desc;
-  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
-                                        &intro_size, &encoded_size,
-                                        &next_desc, desc->desc_str, 1)) {
-    /* Post the descriptor. */
-    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
-    if (!rend_get_service_id(parsed->pk, serviceid)) {
-      smartlist_t *descs = smartlist_new();
-      smartlist_add(descs, desc);
-
-      /* We are about to trigger HS descriptor upload so send the OK now
-       * because after that 650 event(s) are possible so better to have the
-       * 250 OK before them to avoid out of order replies. */
-      send_control_done(conn);
-
-      /* Trigger the descriptor upload */
-      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
-      smartlist_free(descs);
-    }
-
-    rend_service_descriptor_free(parsed);
-  } else {
-    connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-  }
-
-  tor_free(intro_content);
-  rend_encoded_v2_service_descriptor_free(desc);
- done:
-  tor_free(argline);
-  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  return 0;
-}
-
-/* Helper function for ADD_ONION that adds an ephemeral service depending on
- * the given hs_version.
- *
- * The secret key in pk depends on the hs_version. The ownership of the key
- * used in pk is given to the HS subsystem so the caller must stop accessing
- * it after.
- *
- * The port_cfgs is a list of service port. Ownership transferred to service.
- * The max_streams refers to the MaxStreams= key.
- * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
- * The auth_type is the authentication type of the clients in auth_clients.
- * The ownership of that list is transferred to the service.
- *
- * On success (RSAE_OKAY), the address_out points to a newly allocated string
- * containing the onion address without the .onion part. On error, address_out
- * is untouched. */
-static hs_service_add_ephemeral_status_t
-add_onion_helper_add_service(int hs_version,
-                             add_onion_secret_key_t *pk,
-                             smartlist_t *port_cfgs, int max_streams,
-                             int max_streams_close_circuit, int auth_type,
-                             smartlist_t *auth_clients, char **address_out)
-{
-  hs_service_add_ephemeral_status_t ret;
-
-  tor_assert(pk);
-  tor_assert(port_cfgs);
-  tor_assert(address_out);
-
-  switch (hs_version) {
-  case HS_VERSION_TWO:
-    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
-                                     max_streams_close_circuit, auth_type,
-                                     auth_clients, address_out);
-    break;
-  case HS_VERSION_THREE:
-    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
-                                   max_streams_close_circuit, address_out);
-    break;
-  default:
-    tor_assert_unreached();
-  }
-
-  return ret;
-}
-
-/** Called when we get a ADD_ONION command; parse the body, and set up
- * the new ephemeral Onion Service. */
-static int
-handle_control_add_onion(control_connection_t *conn,
-                         uint32_t len,
-                         const char *body)
-{
-  smartlist_t *args;
-  int arg_len;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("ADD_ONION", conn, body, 2, -1);
-  if (!args)
-    return 0;
-  arg_len = smartlist_len(args);
-
-  /* Parse all of the arguments that do not involve handling cryptographic
-   * material first, since there's no reason to touch that at all if any of
-   * the other arguments are malformed.
-   */
-  smartlist_t *port_cfgs = smartlist_new();
-  smartlist_t *auth_clients = NULL;
-  smartlist_t *auth_created_clients = NULL;
-  int discard_pk = 0;
-  int detach = 0;
-  int max_streams = 0;
-  int max_streams_close_circuit = 0;
-  rend_auth_type_t auth_type = REND_NO_AUTH;
-  /* Default to adding an anonymous hidden service if no flag is given */
-  int non_anonymous = 0;
-  for (int i = 1; i < arg_len; i++) {
-    static const char *port_prefix = "Port=";
-    static const char *flags_prefix = "Flags=";
-    static const char *max_s_prefix = "MaxStreams=";
-    static const char *auth_prefix = "ClientAuth=";
-
-    const char *arg = smartlist_get(args, (int)i);
-    if (!strcasecmpstart(arg, port_prefix)) {
-      /* "Port=VIRTPORT[,TARGET]". */
-      const char *port_str = arg + strlen(port_prefix);
-
-      rend_service_port_config_t *cfg =
-          rend_service_parse_port_config(port_str, ",", NULL);
-      if (!cfg) {
-        connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-        goto out;
-      }
-      smartlist_add(port_cfgs, cfg);
-    } else if (!strcasecmpstart(arg, max_s_prefix)) {
-      /* "MaxStreams=[0..65535]". */
-      const char *max_s_str = arg + strlen(max_s_prefix);
-      int ok = 0;
-      max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL);
-      if (!ok) {
-        connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n");
-        goto out;
-      }
-    } else if (!strcasecmpstart(arg, flags_prefix)) {
-      /* "Flags=Flag[,Flag]", where Flag can be:
-       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
-       *                   the response.
-       *   * 'Detach' - Do not tie this onion service to any particular control
-       *                connection.
-       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
-       *                                exceeded.
-       *   * 'BasicAuth' - Client authorization using the 'basic' method.
-       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
-       *                      flag is present, tor must be in non-anonymous
-       *                      hidden service mode. If this flag is absent,
-       *                      tor must be in anonymous hidden service mode.
-       */
-      static const char *discard_flag = "DiscardPK";
-      static const char *detach_flag = "Detach";
-      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
-      static const char *basicauth_flag = "BasicAuth";
-      static const char *non_anonymous_flag = "NonAnonymous";
-
-      smartlist_t *flags = smartlist_new();
-      int bad = 0;
-
-      smartlist_split_string(flags, arg + strlen(flags_prefix), ",",
-                             SPLIT_IGNORE_BLANK, 0);
-      if (smartlist_len(flags) < 1) {
-        connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n");
-        bad = 1;
-      }
-      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
-      {
-        if (!strcasecmp(flag, discard_flag)) {
-          discard_pk = 1;
-        } else if (!strcasecmp(flag, detach_flag)) {
-          detach = 1;
-        } else if (!strcasecmp(flag, max_s_close_flag)) {
-          max_streams_close_circuit = 1;
-        } else if (!strcasecmp(flag, basicauth_flag)) {
-          auth_type = REND_BASIC_AUTH;
-        } else if (!strcasecmp(flag, non_anonymous_flag)) {
-          non_anonymous = 1;
-        } else {
-          connection_printf_to_buf(conn,
-                                   "512 Invalid 'Flags' argument: %s\r\n",
-                                   escaped(flag));
-          bad = 1;
-          break;
-        }
-      } SMARTLIST_FOREACH_END(flag);
-      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
-      smartlist_free(flags);
-      if (bad)
-        goto out;
-    } else if (!strcasecmpstart(arg, auth_prefix)) {
-      char *err_msg = NULL;
-      int created = 0;
-      rend_authorized_client_t *client =
-        add_onion_helper_clientauth(arg + strlen(auth_prefix),
-                                    &created, &err_msg);
-      if (!client) {
-        if (err_msg) {
-          connection_write_str_to_buf(err_msg, conn);
-          tor_free(err_msg);
-        }
-        goto out;
-      }
-
-      if (auth_clients != NULL) {
-        int bad = 0;
-        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
-          if (strcmp(ac->client_name, client->client_name) == 0) {
-            bad = 1;
-            break;
-          }
-        } SMARTLIST_FOREACH_END(ac);
-        if (bad) {
-          connection_printf_to_buf(conn,
-                                   "512 Duplicate name in ClientAuth\r\n");
-          rend_authorized_client_free(client);
-          goto out;
-        }
-      } else {
-        auth_clients = smartlist_new();
-        auth_created_clients = smartlist_new();
-      }
-      smartlist_add(auth_clients, client);
-      if (created) {
-        smartlist_add(auth_created_clients, client);
-      }
-    } else {
-      connection_printf_to_buf(conn, "513 Invalid argument\r\n");
-      goto out;
-    }
-  }
-  if (smartlist_len(port_cfgs) == 0) {
-    connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n");
-    goto out;
-  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
-    connection_printf_to_buf(conn, "512 No auth type specified\r\n");
-    goto out;
-  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
-    connection_printf_to_buf(conn, "512 No auth clients specified\r\n");
-    goto out;
-  } else if ((auth_type == REND_BASIC_AUTH &&
-              smartlist_len(auth_clients) > 512) ||
-             (auth_type == REND_STEALTH_AUTH &&
-              smartlist_len(auth_clients) > 16)) {
-    connection_printf_to_buf(conn, "512 Too many auth clients\r\n");
-    goto out;
-  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
-                                                              get_options())) {
-    /* If we failed, and the non-anonymous flag is set, Tor must be in
-     * anonymous hidden service mode.
-     * The error message changes based on the current Tor config:
-     * 512 Tor is in anonymous hidden service mode
-     * 512 Tor is in non-anonymous hidden service mode
-     * (I've deliberately written them out in full here to aid searchability.)
-     */
-    connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service "
-                             "mode\r\n",
-                             non_anonymous ? "" : "non-");
-    goto out;
-  }
-
-  /* Parse the "keytype:keyblob" argument. */
-  int hs_version = 0;
-  add_onion_secret_key_t pk = { NULL };
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-
-  if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
-                              &key_new_alg, &key_new_blob, &pk, &hs_version,
-                              &err_msg) < 0) {
-    if (err_msg) {
-      connection_write_str_to_buf(err_msg, conn);
-      tor_free(err_msg);
-    }
-    goto out;
-  }
-  tor_assert(!err_msg);
-
-  /* Hidden service version 3 don't have client authentication support so if
-   * ClientAuth was given, send back an error. */
-  if (hs_version == HS_VERSION_THREE && auth_clients) {
-    connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n");
-    goto out;
-  }
-
-  /* Create the HS, using private key pk, client authentication auth_type,
-   * the list of auth_clients, and port config port_cfg.
-   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
-   * regardless of success/failure.
-   */
-  char *service_id = NULL;
-  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
-                                         max_streams,
-                                         max_streams_close_circuit, auth_type,
-                                         auth_clients, &service_id);
-  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
-  auth_clients = NULL; /* so is auth_clients */
-  switch (ret) {
-  case RSAE_OKAY:
-  {
-    if (detach) {
-      if (!detached_onion_services)
-        detached_onion_services = smartlist_new();
-      smartlist_add(detached_onion_services, service_id);
-    } else {
-      if (!conn->ephemeral_onion_services)
-        conn->ephemeral_onion_services = smartlist_new();
-      smartlist_add(conn->ephemeral_onion_services, service_id);
-    }
-
-    tor_assert(service_id);
-    connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id);
-    if (key_new_alg) {
-      tor_assert(key_new_blob);
-      connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n",
-                               key_new_alg, key_new_blob);
-    }
-    if (auth_created_clients) {
-      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
-        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
-                                                auth_type);
-        tor_assert(encoded);
-        connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n",
-                                 ac->client_name, encoded);
-        memwipe(encoded, 0, strlen(encoded));
-        tor_free(encoded);
-      });
-    }
-
-    connection_printf_to_buf(conn, "250 OK\r\n");
-    break;
-  }
-  case RSAE_BADPRIVKEY:
-    connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n");
-    break;
-  case RSAE_ADDREXISTS:
-    connection_printf_to_buf(conn, "550 Onion address collision\r\n");
-    break;
-  case RSAE_BADVIRTPORT:
-    connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-    break;
-  case RSAE_BADAUTH:
-    connection_printf_to_buf(conn, "512 Invalid client authorization\r\n");
-    break;
-  case RSAE_INTERNAL: FALLTHROUGH;
-  default:
-    connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n");
-  }
-  if (key_new_blob) {
-    memwipe(key_new_blob, 0, strlen(key_new_blob));
-    tor_free(key_new_blob);
-  }
-
- out:
-  if (port_cfgs) {
-    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
-                      rend_service_port_config_free(p));
-    smartlist_free(port_cfgs);
-  }
-
-  if (auth_clients) {
-    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
-                      rend_authorized_client_free(ac));
-    smartlist_free(auth_clients);
-  }
-  if (auth_created_clients) {
-    // Do not free entries; they are the same as auth_clients
-    smartlist_free(auth_created_clients);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
-/** Helper function to handle parsing the KeyType:KeyBlob argument to the
- * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
- * and the private key not discarded, the algorithm and serialized private key,
- * or NULL and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned key_new_blob and err_msg.
- *
- * Note: The error messages returned are deliberately vague to avoid echoing
- * key material.
- */
-STATIC int
-add_onion_helper_keyarg(const char *arg, int discard_pk,
-                        const char **key_new_alg_out, char **key_new_blob_out,
-                        add_onion_secret_key_t *decoded_key, int *hs_version,
-                        char **err_msg_out)
-{
-  smartlist_t *key_args = smartlist_new();
-  crypto_pk_t *pk = NULL;
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-  int ret = -1;
-
-  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(key_args) != 2) {
-    err_msg = tor_strdup("512 Invalid key type/blob\r\n");
-    goto err;
-  }
-
-  /* The format is "KeyType:KeyBlob". */
-  static const char *key_type_new = "NEW";
-  static const char *key_type_best = "BEST";
-  static const char *key_type_rsa1024 = "RSA1024";
-  static const char *key_type_ed25519_v3 = "ED25519-V3";
-
-  const char *key_type = smartlist_get(key_args, 0);
-  const char *key_blob = smartlist_get(key_args, 1);
-
-  if (!strcasecmp(key_type_rsa1024, key_type)) {
-    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
-    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
-    if (!pk) {
-      err_msg = tor_strdup("512 Failed to decode RSA key\r\n");
-      goto err;
-    }
-    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
-      crypto_pk_free(pk);
-      err_msg = tor_strdup("512 Invalid RSA key size\r\n");
-      goto err;
-    }
-    decoded_key->v2 = pk;
-    *hs_version = HS_VERSION_TWO;
-  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
-    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
-    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
-                      strlen(key_blob)) != sizeof(sk->seckey)) {
-      tor_free(sk);
-      err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n");
-      goto err;
-    }
-    decoded_key->v3 = sk;
-    *hs_version = HS_VERSION_THREE;
-  } else if (!strcasecmp(key_type_new, key_type)) {
-    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
-    if (!strcasecmp(key_type_rsa1024, key_blob) ||
-        !strcasecmp(key_type_best, key_blob)) {
-      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
-      pk = crypto_pk_new();
-      if (crypto_pk_generate_key(pk)) {
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_rsa1024);
-        goto err;
-      }
-      if (!discard_pk) {
-        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
-          crypto_pk_free(pk);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_rsa1024);
-          goto err;
-        }
-        key_new_alg = key_type_rsa1024;
-      }
-      decoded_key->v2 = pk;
-      *hs_version = HS_VERSION_TWO;
-    } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) {
-      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-      if (ed25519_secret_key_generate(sk, 1) < 0) {
-        tor_free(sk);
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_ed25519_v3);
-        goto err;
-      }
-      if (!discard_pk) {
-        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
-        key_new_blob = tor_malloc_zero(len);
-        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
-                          sizeof(sk->seckey), 0) != (len - 1)) {
-          tor_free(sk);
-          tor_free(key_new_blob);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_ed25519_v3);
-          goto err;
-        }
-        key_new_alg = key_type_ed25519_v3;
-      }
-      decoded_key->v3 = sk;
-      *hs_version = HS_VERSION_THREE;
-    } else {
-      err_msg = tor_strdup("513 Invalid key type\r\n");
-      goto err;
-    }
-  } else {
-    err_msg = tor_strdup("513 Invalid key type\r\n");
-    goto err;
-  }
-
-  /* Succeeded in loading or generating a private key. */
-  ret = 0;
-
- err:
-  SMARTLIST_FOREACH(key_args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(key_args);
-
-  if (err_msg_out) {
-    *err_msg_out = err_msg;
-  } else {
-    tor_free(err_msg);
-  }
-  *key_new_alg_out = key_new_alg;
-  *key_new_blob_out = key_new_blob;
-
-  return ret;
-}
-
-/** Helper function to handle parsing a ClientAuth argument to the
- * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
- * and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned auth_client and err_msg.
- *
- * If 'created' is specified, it will be set to 1 when a new cookie has
- * been generated.
- */
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg)
-{
-  int ok = 0;
-
-  tor_assert(arg);
-  tor_assert(created);
-  tor_assert(err_msg);
-  *err_msg = NULL;
-
-  smartlist_t *auth_args = smartlist_new();
-  rend_authorized_client_t *client =
-    tor_malloc_zero(sizeof(rend_authorized_client_t));
-  smartlist_split_string(auth_args, arg, ":", 0, 0);
-  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
-    *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n");
-    goto err;
-  }
-  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
-  if (smartlist_len(auth_args) == 2) {
-    char *decode_err_msg = NULL;
-    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
-                                client->descriptor_cookie,
-                                NULL, &decode_err_msg) < 0) {
-      tor_assert(decode_err_msg);
-      tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg);
-      tor_free(decode_err_msg);
-      goto err;
-    }
-    *created = 0;
-  } else {
-    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
-    *created = 1;
-  }
-
-  if (!rend_valid_client_name(client->client_name)) {
-    *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n");
-    goto err;
-  }
-
-  ok = 1;
- err:
-  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
-  smartlist_free(auth_args);
-  if (!ok) {
-    rend_authorized_client_free(client);
-    client = NULL;
-  }
-  return client;
-}
-
-/** Called when we get a DEL_ONION command; parse the body, and remove
- * the existing ephemeral Onion Service. */
-static int
-handle_control_del_onion(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  int hs_version = 0;
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("DEL_ONION", conn, body, 1, 1);
-  if (!args)
-    return 0;
-
-  const char *service_id = smartlist_get(args, 0);
-  if (rend_valid_v2_service_id(service_id)) {
-    hs_version = HS_VERSION_TWO;
-  } else if (hs_address_is_valid(service_id)) {
-    hs_version = HS_VERSION_THREE;
-  } else {
-    connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n");
-    goto out;
-  }
-
-  /* Determine if the onion service belongs to this particular control
-   * connection, or if it is in the global list of detached services.  If it
-   * is in neither, either the service ID is invalid in some way, or it
-   * explicitly belongs to a different control connection, and an error
-   * should be returned.
-   */
-  smartlist_t *services[2] = {
-    conn->ephemeral_onion_services,
-    detached_onion_services
-  };
-  smartlist_t *onion_services = NULL;
-  int idx = -1;
-  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
-    idx = smartlist_string_pos(services[i], service_id);
-    if (idx != -1) {
-      onion_services = services[i];
-      break;
-    }
-  }
-  if (onion_services == NULL) {
-    connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n");
-  } else {
-    int ret = -1;
-    switch (hs_version) {
-    case HS_VERSION_TWO:
-      ret = rend_service_del_ephemeral(service_id);
-      break;
-    case HS_VERSION_THREE:
-      ret = hs_service_del_ephemeral(service_id);
-      break;
-    default:
-      /* The ret value will be -1 thus hitting the warning below. This should
-       * never happen because of the check at the start of the function. */
-      break;
-    }
-    if (ret < 0) {
-      /* This should *NEVER* fail, since the service is on either the
-       * per-control connection list, or the global one.
-       */
-      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
-               escaped(service_id));
-      tor_fragile_assert();
-    }
-
-    /* Remove/scrub the service_id from the appropriate list. */
-    char *cp = smartlist_get(onion_services, idx);
-    smartlist_del(onion_services, idx);
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-
-    send_control_done(conn);
-  }
-
- out:
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
 /** Called when <b>conn</b> has no more bytes left on its outbuf. */
 int
 connection_control_finished_flushing(control_connection_t *conn)
@@ -5349,6 +279,44 @@ peek_connection_has_http_command(connection_t *conn)
   return peek_buf_has_http_command(conn->inbuf);
 }
 
+/**
+ * Helper: take a nul-terminated command of given length, and find where the
+ * command starts and the arguments begin.  Separate them, allocate a new
+ * string in <b>current_cmd_out</b> for the command, and return a pointer
+ * to the arguments.
+ **/
+STATIC char *
+control_split_incoming_command(char *incoming_cmd,
+                               size_t *data_len,
+                               char **current_cmd_out)
+{
+  const bool is_multiline = *data_len && incoming_cmd[0] == '+';
+  size_t cmd_len = 0;
+  while (cmd_len < *data_len
+         && !TOR_ISSPACE(incoming_cmd[cmd_len]))
+    ++cmd_len;
+
+  *current_cmd_out = tor_memdup_nulterm(incoming_cmd, cmd_len);
+  char *args = incoming_cmd+cmd_len;
+  tor_assert(*data_len>=cmd_len);
+  *data_len -= cmd_len;
+  if (is_multiline) {
+    // Only match horizontal space: any line after the first is data,
+    // not arguments.
+    while ((*args == '\t' || *args == ' ') && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  } else {
+    while (TOR_ISSPACE(*args) && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  }
+
+  return args;
+}
+
 static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy"
   "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n"
@@ -5375,6 +343,60 @@ static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "</body>\n"
   "</html>\n";
 
+/** Return an error on a control connection that tried to use the v0 protocol.
+ */
+static void
+control_send_v0_reject(control_connection_t *conn)
+{
+  size_t body_len;
+  char buf[128];
+  set_uint16(buf+2, htons(0x0000)); /* type == error */
+  set_uint16(buf+4, htons(0x0001)); /* code == internal error */
+  strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
+          "and later; upgrade your controller.",
+          sizeof(buf)-6);
+  body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
+  set_uint16(buf+0, htons(body_len));
+  connection_buf_add(buf, 4+body_len, TO_CONN(conn));
+
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Return an error on a control connection that tried to use HTTP.
+ */
+static void
+control_send_http_reject(control_connection_t *conn)
+{
+  connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
+  log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Check if a control connection has tried to use a known invalid protocol.
+ * If it has, then:
+ *  - send a reject response,
+ *  - log a notice-level message, and
+ *  - return false. */
+static bool
+control_protocol_is_valid(control_connection_t *conn)
+{
+  /* Detect v0 commands and send a "no more v0" message. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_control0_command(TO_CONN(conn))) {
+    control_send_v0_reject(conn);
+    return 0;
+  }
+
+  /* If the user has the HTTP proxy port and the control port confused. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_http_command(TO_CONN(conn))) {
+    control_send_http_reject(conn);
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Called when data has arrived on a v1 control connection: Try to fetch
  * commands from conn->inbuf, and execute them.
  */
@@ -5383,7 +405,6 @@ connection_control_process_inbuf(control_connection_t *conn)
 {
   size_t data_len;
   uint32_t cmd_data_len;
-  int cmd_len;
   char *args;
 
   tor_assert(conn);
@@ -5396,30 +417,7 @@ connection_control_process_inbuf(control_connection_t *conn)
     conn->incoming_cmd_cur_len = 0;
   }
 
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_control0_command(TO_CONN(conn))) {
-    /* Detect v0 commands and send a "no more v0" message. */
-    size_t body_len;
-    char buf[128];
-    set_uint16(buf+2, htons(0x0000)); /* type == error */
-    set_uint16(buf+4, htons(0x0001)); /* code == internal error */
-    strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
-            "and later; upgrade your controller.",
-            sizeof(buf)-6);
-    body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
-    set_uint16(buf+0, htons(body_len));
-    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
-
-    connection_mark_and_flush(TO_CONN(conn));
-    return 0;
-  }
-
-  /* If the user has the HTTP proxy port and the control port confused. */
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_http_command(TO_CONN(conn))) {
-    connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
-    log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
-    connection_mark_and_flush(TO_CONN(conn));
+  if (!control_protocol_is_valid(conn)) {
     return 0;
   }
 
@@ -5438,7 +436,7 @@ connection_control_process_inbuf(control_connection_t *conn)
         return 0;
       else if (r == -1) {
         if (data_len + conn->incoming_cmd_cur_len > MAX_COMMAND_LINE_LENGTH) {
-          connection_write_str_to_buf("500 Line too long.\r\n", conn);
+          control_write_endreply(conn, 500, "Line too long.");
           connection_stop_reading(TO_CONN(conn));
           connection_mark_and_flush(TO_CONN(conn));
         }
@@ -5475,22 +473,15 @@ connection_control_process_inbuf(control_connection_t *conn)
     /* Otherwise, read another line. */
   }
   data_len = conn->incoming_cmd_cur_len;
+
   /* Okay, we now have a command sitting on conn->incoming_cmd. See if we
    * recognize it.
    */
-  cmd_len = 0;
-  while ((size_t)cmd_len < data_len
-         && !TOR_ISSPACE(conn->incoming_cmd[cmd_len]))
-    ++cmd_len;
-
-  conn->incoming_cmd[cmd_len]='\0';
-  args = conn->incoming_cmd+cmd_len+1;
-  tor_assert(data_len>(size_t)cmd_len);
-  data_len -= (cmd_len+1); /* skip the command and NUL we added after it */
-  while (TOR_ISSPACE(*args)) {
-    ++args;
-    --data_len;
-  }
+  tor_free(conn->current_cmd);
+  args = control_split_incoming_command(conn->incoming_cmd, &data_len,
+                                        &conn->current_cmd);
+  if (BUG(!conn->current_cmd))
+    return -1;
 
   /* If the connection is already closing, ignore further commands */
   if (TO_CONN(conn)->marked_for_close) {
@@ -5498,1448 +489,50 @@ connection_control_process_inbuf(control_connection_t *conn)
   }
 
   /* Otherwise, Quit is always valid. */
-  if (!strcasecmp(conn->incoming_cmd, "QUIT")) {
-    connection_write_str_to_buf("250 closing connection\r\n", conn);
+  if (!strcasecmp(conn->current_cmd, "QUIT")) {
+    control_write_endreply(conn, 250, "closing connection");
     connection_mark_and_flush(TO_CONN(conn));
     return 0;
   }
 
   if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      !is_valid_initial_command(conn, conn->incoming_cmd)) {
-    connection_write_str_to_buf("514 Authentication required.\r\n", conn);
+      !is_valid_initial_command(conn, conn->current_cmd)) {
+    control_write_endreply(conn, 514, "Authentication required.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
   if (data_len >= UINT32_MAX) {
-    connection_write_str_to_buf("500 A 4GB command? Nice try.\r\n", conn);
+    control_write_endreply(conn, 500, "A 4GB command? Nice try.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
-  /* XXXX Why is this not implemented as a table like the GETINFO
-   * items are?  Even handling the plus signs at the beginnings of
-   * commands wouldn't be very hard with proper macros. */
   cmd_data_len = (uint32_t)data_len;
-  if (!strcasecmp(conn->incoming_cmd, "SETCONF")) {
-    if (handle_control_setconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) {
-    if (handle_control_resetconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) {
-    if (handle_control_getconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) {
-    if (handle_control_loadconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) {
-    if (handle_control_setevents(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) {
-    if (handle_control_authenticate(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) {
-    if (handle_control_saveconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) {
-    if (handle_control_signal(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) {
-    if (handle_control_takeownership(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) {
-    if (handle_control_mapaddress(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) {
-    if (handle_control_getinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) {
-    if (handle_control_extendcircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) {
-    if (handle_control_setcircuitpurpose(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) {
-    connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn);
-  } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) {
-    if (handle_control_attachstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) {
-    if (handle_control_postdescriptor(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) {
-    if (handle_control_redirectstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) {
-    if (handle_control_closestream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) {
-    if (handle_control_closecircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) {
-    if (handle_control_usefeature(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) {
-    if (handle_control_resolve(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) {
-    if (handle_control_protocolinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) {
-    if (handle_control_authchallenge(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) {
-    if (handle_control_dropguards(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) {
-    if (handle_control_hsfetch(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) {
-    if (handle_control_hspost(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) {
-    int ret = handle_control_add_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the private key. */
-    if (ret)
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) {
-    int ret = handle_control_del_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */
-    if (ret)
-      return -1;
-  } else {
-    connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n",
-                             conn->incoming_cmd);
-  }
+  if (handle_control_command(conn, cmd_data_len, args) < 0)
+    return -1;
 
   conn->incoming_cmd_cur_len = 0;
   goto again;
 }
 
-/** Something major has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-int
-control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
-                             int reason_code)
-{
-  const char *status;
-  char reasons[64] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
-    return 0;
-  tor_assert(circ);
-
-  switch (tp)
-    {
-    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case CIRC_EVENT_BUILT: status = "BUILT"; break;
-    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
-    case CIRC_EVENT_FAILED: status = "FAILED"; break;
-    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
-    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
-    char unk_reason_buf[16];
-    if (!reason_str) {
-      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
-      reason_str = unk_reason_buf;
-    }
-    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
-    } else {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=%s", reason_str);
-    }
-  }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS,
-                                "650 CIRC %lu %s%s%s%s\r\n",
-                                (unsigned long)circ->global_identifier,
-                                status, sp,
-                                circdesc,
-                                reasons);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/** Something minor has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-static int
-control_event_circuit_status_minor(origin_circuit_t *circ,
-                                   circuit_status_minor_event_t e,
-                                   int purpose, const struct timeval *tv)
-{
-  const char *event_desc;
-  char event_tail[160] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
-    return 0;
-  tor_assert(circ);
-
-  switch (e)
-    {
-    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
-      event_desc = "PURPOSE_CHANGED";
-
-      {
-        /* event_tail can currently be up to 68 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "");
-      }
-
-      break;
-    case CIRC_MINOR_EVENT_CANNIBALIZED:
-      event_desc = "CANNIBALIZED";
-
-      {
-        /* event_tail can currently be up to 130 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        const struct timeval *old_timestamp_began = tv;
-        char tbuf[ISO_TIME_USEC_LEN+1];
-        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
-
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "",
-                     tbuf);
-      }
-
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
-                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
-                       (unsigned long)circ->global_identifier,
-                       event_desc, sp,
-                       circdesc,
-                       event_tail);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
- * interested controllers.
- */
-int
-control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                      int old_purpose)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-                                            old_purpose,
-                                            NULL);
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
- * created-time from <b>old_tv_created</b>: tell any interested controllers.
- */
-int
-control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                   int old_purpose,
-                                   const struct timeval *old_tv_created)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_CANNIBALIZED,
-                                            old_purpose,
-                                            old_tv_created);
-}
-
-/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
- * <b>buf</b>, determine the address:port combination requested on
- * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
- * failure. */
-static int
-write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
-{
-  char buf2[256];
-  if (conn->chosen_exit_name)
-    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
-      return -1;
-  if (!conn->socks_request)
-    return -1;
-  if (tor_snprintf(buf, len, "%s%s%s:%d",
-               conn->socks_request->address,
-               conn->chosen_exit_name ? buf2 : "",
-               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
-                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
-               conn->socks_request->port)<0)
-    return -1;
-  return 0;
-}
-
-/** Something has happened to the stream associated with AP connection
- * <b>conn</b>: tell any interested control connections. */
-int
-control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
-                            int reason_code)
-{
-  char reason_buf[64];
-  char addrport_buf[64];
-  const char *status;
-  circuit_t *circ;
-  origin_circuit_t *origin_circ = NULL;
-  char buf[256];
-  const char *purpose = "";
-  tor_assert(conn->socks_request);
-
-  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
-    return 0;
-
-  if (tp == STREAM_EVENT_CLOSED &&
-      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
-    return 0;
-
-  write_stream_target_to_buf(conn, buf, sizeof(buf));
-
-  reason_buf[0] = '\0';
-  switch (tp)
-    {
-    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
-    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
-    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
-    case STREAM_EVENT_FAILED: status = "FAILED"; break;
-    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
-    case STREAM_EVENT_NEW: status = "NEW"; break;
-    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
-    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
-    case STREAM_EVENT_REMAP: status = "REMAP"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (reason_code && (tp == STREAM_EVENT_FAILED ||
-                      tp == STREAM_EVENT_CLOSED ||
-                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
-    const char *reason_str = stream_end_reason_to_control_string(reason_code);
-    char *r = NULL;
-    if (!reason_str) {
-      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
-      reason_str = r;
-    }
-    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=END REMOTE_REASON=%s", reason_str);
-    else
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=%s", reason_str);
-    tor_free(r);
-  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
-    switch (reason_code) {
-    case REMAP_STREAM_SOURCE_CACHE:
-      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
-      break;
-    case REMAP_STREAM_SOURCE_EXIT:
-      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
-      break;
-    default:
-      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
-                   reason_code);
-      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
-      break;
-    }
-  }
-
-  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
-    /*
-     * When the control conn is an AF_UNIX socket and we have no address,
-     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
-     * dnsserv.c.
-     */
-    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
-      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
-                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
-    } else {
-      /*
-       * else leave it blank so control on AF_UNIX doesn't need to make
-       * something up.
-       */
-      addrport_buf[0] = '\0';
-    }
-  } else {
-    addrport_buf[0] = '\0';
-  }
-
-  if (tp == STREAM_EVENT_NEW_RESOLVE) {
-    purpose = " PURPOSE=DNS_REQUEST";
-  } else if (tp == STREAM_EVENT_NEW) {
-    if (conn->use_begindir) {
-      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
-      int linked_dir_purpose = -1;
-      if (linked && linked->type == CONN_TYPE_DIR)
-        linked_dir_purpose = linked->purpose;
-      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
-        purpose = " PURPOSE=DIR_UPLOAD";
-      else
-        purpose = " PURPOSE=DIR_FETCH";
-    } else
-      purpose = " PURPOSE=USER";
-  }
-
-  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-  if (circ && CIRCUIT_IS_ORIGIN(circ))
-    origin_circ = TO_ORIGIN_CIRCUIT(circ);
-  send_control_event(EVENT_STREAM_STATUS,
-                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
-                     (ENTRY_TO_CONN(conn)->global_identifier),
-                     status,
-                        origin_circ?
-                           (unsigned long)origin_circ->global_identifier : 0ul,
-                        buf, reason_buf, addrport_buf, purpose);
-
-  /* XXX need to specify its intended exit, etc? */
-
-  return 0;
-}
-
-/** Figure out the best name for the target router of an OR connection
- * <b>conn</b>, and write it into the <b>len</b>-character buffer
- * <b>name</b>. */
-static void
-orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
-{
-  const node_t *node = node_get_by_id(conn->identity_digest);
-  if (node) {
-    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
-    node_get_verbose_nickname(node, name);
-  } else if (! tor_digest_is_zero(conn->identity_digest)) {
-    name[0] = '$';
-    base16_encode(name+1, len-1, conn->identity_digest,
-                  DIGEST_LEN);
-  } else {
-    tor_snprintf(name, len, "%s:%d",
-                 conn->base_.address, conn->base_.port);
-  }
-}
-
-/** Called when the status of an OR connection <b>conn</b> changes: tell any
- * interested control connections. <b>tp</b> is the new status for the
- * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
- * may be the reason why.
- */
-int
-control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
-                             int reason)
-{
-  int ncircs = 0;
-  const char *status;
-  char name[128];
-  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
-
-  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
-    return 0;
-
-  switch (tp)
-    {
-    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
-    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
-    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
-    case OR_CONN_EVENT_NEW: status = "NEW"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (conn->chan) {
-    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
-  } else {
-    ncircs = 0;
-  }
-  ncircs += connection_or_get_num_circuits(conn);
-  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
-    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
-  }
-
-  orconn_target_get_name(name, sizeof(name), conn);
-  send_control_event(EVENT_OR_CONN_STATUS,
-                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
-                              name, status,
-                              reason ? " REASON=" : "",
-                              orconn_end_reason_to_control_string(reason),
-                              ncircs_buf,
-                              (conn->base_.global_identifier));
-
-  return 0;
-}
-
-/**
- * Print out STREAM_BW event for a single conn
- */
-int
-control_event_stream_bandwidth(edge_connection_t *edge_conn)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    if (!edge_conn->n_read && !edge_conn->n_written)
-      return 0;
-
-    tor_gettimeofday(&now);
-    format_iso_time_nospace_usec(tbuf, &now);
-    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                       (edge_conn->base_.global_identifier),
-                       (unsigned long)edge_conn->n_read,
-                       (unsigned long)edge_conn->n_written,
-                       tbuf);
-
-    edge_conn->n_written = edge_conn->n_read = 0;
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth streams have used. */
-int
-control_event_stream_bandwidth_used(void)
-{
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    smartlist_t *conns = get_connection_array();
-    edge_connection_t *edge_conn;
-    struct timeval now;
-    char tbuf[ISO_TIME_USEC_LEN+1];
-
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
-    {
-        if (conn->type != CONN_TYPE_AP)
-          continue;
-        edge_conn = TO_EDGE_CONN(conn);
-        if (!edge_conn->n_read && !edge_conn->n_written)
-          continue;
-
-        tor_gettimeofday(&now);
-        format_iso_time_nospace_usec(tbuf, &now);
-        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                           (edge_conn->base_.global_identifier),
-                           (unsigned long)edge_conn->n_read,
-                           (unsigned long)edge_conn->n_written,
-                           tbuf);
-
-        edge_conn->n_written = edge_conn->n_read = 0;
-    }
-    SMARTLIST_FOREACH_END(conn);
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control connections
- * how much bandwidth origin circuits have used. */
-int
-control_event_circ_bandwidth_used(void)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-
-    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
-  }
-  SMARTLIST_FOREACH_END(circ);
-
-  return 0;
-}
-
-/**
- * Emit a CIRC_BW event line for a specific circuit.
- *
- * This function sets the values it emits to 0, and does not emit
- * an event if there is no new data to report since the last call.
- *
- * Therefore, it may be called at any frequency.
- */
-int
-control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-
-  tor_assert(ocirc);
-
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  /* n_read_circ_bw and n_written_circ_bw are always updated
-   * when there is any new cell on a circuit, and set to 0 after
-   * the event, below.
-   *
-   * Therefore, checking them is sufficient to determine if there
-   * is new data to report. */
-  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
-    return 0;
-
-  tor_gettimeofday(&now);
-  format_iso_time_nospace_usec(tbuf, &now);
-  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
-                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
-                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
-                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
-                     ocirc->global_identifier,
-                     (unsigned long)ocirc->n_read_circ_bw,
-                     (unsigned long)ocirc->n_written_circ_bw,
-                     tbuf,
-                     (unsigned long)ocirc->n_delivered_read_circ_bw,
-                     (unsigned long)ocirc->n_overhead_read_circ_bw,
-                     (unsigned long)ocirc->n_delivered_written_circ_bw,
-                     (unsigned long)ocirc->n_overhead_written_circ_bw);
-  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-
-  return 0;
-}
-
-/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
-  * bandwidth counters. */
-int
-control_event_conn_bandwidth(connection_t *conn)
-{
-  const char *conn_type_str;
-  if (!get_options()->TestingEnableConnBwEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
-    return 0;
-  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
-    return 0;
-  switch (conn->type) {
-    case CONN_TYPE_OR:
-      conn_type_str = "OR";
-      break;
-    case CONN_TYPE_DIR:
-      conn_type_str = "DIR";
-      break;
-    case CONN_TYPE_EXIT:
-      conn_type_str = "EXIT";
-      break;
-    default:
-      return 0;
-  }
-  send_control_event(EVENT_CONN_BW,
-                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
-                     "READ=%lu WRITTEN=%lu\r\n",
-                     (conn->global_identifier),
-                     conn_type_str,
-                     (unsigned long)conn->n_read_conn_bw,
-                     (unsigned long)conn->n_written_conn_bw);
-  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth connections have used. */
-int
-control_event_conn_bandwidth_used(void)
-{
-  if (get_options()->TestingEnableConnBwEvent &&
-      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
-    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
-                      control_event_conn_bandwidth(conn));
-  }
-  return 0;
-}
-
-/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
- * cells, removed cells, and waiting times by cell command and direction.
- * Store results in <b>cell_stats</b>.  Free cell statistics of the
- * circuit afterwards. */
-void
-sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
-{
-  memset(cell_stats, 0, sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
-                          const testing_cell_stats_entry_t *, ent) {
-    tor_assert(ent->command <= CELL_COMMAND_MAX_);
-    if (!ent->removed && !ent->exitward) {
-      cell_stats->added_cells_appward[ent->command] += 1;
-    } else if (!ent->removed && ent->exitward) {
-      cell_stats->added_cells_exitward[ent->command] += 1;
-    } else if (!ent->exitward) {
-      cell_stats->removed_cells_appward[ent->command] += 1;
-      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
-    } else {
-      cell_stats->removed_cells_exitward[ent->command] += 1;
-      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
-    }
-  } SMARTLIST_FOREACH_END(ent);
-  circuit_clear_testing_cell_stats(circ);
-}
-
-/** Helper: append a cell statistics string to <code>event_parts</code>,
- * prefixed with <code>key</code>=.  Statistics consist of comma-separated
- * key:value pairs with lower-case command strings as keys and cell
- * numbers or total waiting times as values.  A key:value pair is included
- * if the entry in <code>include_if_non_zero</code> is not zero, but with
- * the (possibly zero) entry from <code>number_to_include</code>.  Both
- * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
- * entry in <code>include_if_non_zero</code> is positive, no string will
- * be added to <code>event_parts</code>. */
-void
-append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
-                             const uint64_t *include_if_non_zero,
-                             const uint64_t *number_to_include)
-{
-  smartlist_t *key_value_strings = smartlist_new();
-  int i;
-  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
-    if (include_if_non_zero[i] > 0) {
-      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
-                             cell_command_to_string(i),
-                             (number_to_include[i]));
-    }
-  }
-  if (smartlist_len(key_value_strings) > 0) {
-    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
-    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
-    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
-    tor_free(joined);
-  }
-  smartlist_free(key_value_strings);
-}
-
-/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
- * CELL_STATS event and write result string to <b>event_string</b>. */
-void
-format_cell_stats(char **event_string, circuit_t *circ,
-                  cell_stats_t *cell_stats)
-{
-  smartlist_t *event_parts = smartlist_new();
-  if (CIRCUIT_IS_ORIGIN(circ)) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "ID=%lu",
-                 (unsigned long)ocirc->global_identifier);
-  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
-    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
-                 (unsigned long)or_circ->p_circ_id);
-    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
-                 (or_circ->p_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "InboundAdded",
-                                 cell_stats->added_cells_appward,
-                                 cell_stats->added_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundRemoved",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->removed_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundTime",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->total_time_appward);
-  }
-  if (circ->n_chan) {
-    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
-                     (unsigned long)circ->n_circ_id);
-    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
-                 (circ->n_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "OutboundAdded",
-                                 cell_stats->added_cells_exitward,
-                                 cell_stats->added_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundRemoved",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->removed_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundTime",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->total_time_exitward);
-  }
-  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
-  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
-  smartlist_free(event_parts);
-}
-
-/** A second or more has elapsed: tell any interested control connection
- * how many cells have been processed for a given circuit. */
-int
-control_event_circuit_cell_stats(void)
-{
-  cell_stats_t *cell_stats;
-  char *event_string;
-  if (!get_options()->TestingEnableCellStatsEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
-    return 0;
-  cell_stats = tor_malloc(sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!circ->testing_cell_stats)
-      continue;
-    sum_up_cell_stats_by_command(circ, cell_stats);
-    format_cell_stats(&event_string, circ, cell_stats);
-    send_control_event(EVENT_CELL_STATS,
-                       "650 CELL_STATS %s\r\n", event_string);
-    tor_free(event_string);
-  }
-  SMARTLIST_FOREACH_END(circ);
-  tor_free(cell_stats);
-  return 0;
-}
-
-/* about 5 minutes worth. */
-#define N_BW_EVENTS_TO_CACHE 300
-/* Index into cached_bw_events to next write. */
-static int next_measurement_idx = 0;
-/* number of entries set in n_measurements */
-static int n_measurements = 0;
-static struct cached_bw_event_s {
-  uint32_t n_read;
-  uint32_t n_written;
-} cached_bw_events[N_BW_EVENTS_TO_CACHE];
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth we used. */
-int
-control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
-{
-  cached_bw_events[next_measurement_idx].n_read = n_read;
-  cached_bw_events[next_measurement_idx].n_written = n_written;
-  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
-    next_measurement_idx = 0;
-  if (n_measurements < N_BW_EVENTS_TO_CACHE)
-    ++n_measurements;
-
-  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
-    send_control_event(EVENT_BANDWIDTH_USED,
-                       "650 BW %lu %lu\r\n",
-                       (unsigned long)n_read,
-                       (unsigned long)n_written);
-  }
-
-  return 0;
-}
-
-STATIC char *
-get_bw_samples(void)
-{
-  int i;
-  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
-    % N_BW_EVENTS_TO_CACHE;
-  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-
-  smartlist_t *elements = smartlist_new();
-
-  for (i = 0; i < n_measurements; ++i) {
-    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-    const struct cached_bw_event_s *bwe = &cached_bw_events[idx];
-
-    smartlist_add_asprintf(elements, "%u,%u",
-                           (unsigned)bwe->n_read,
-                           (unsigned)bwe->n_written);
-
-    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
-  }
-
-  char *result = smartlist_join_strings(elements, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-
-  return result;
-}
-
-/** Called when we are sending a log message to the controllers: suspend
- * sending further log messages to the controllers until we're done.  Used by
- * CONN_LOG_PROTECT. */
-void
-disable_control_logging(void)
-{
-  ++disable_log_messages;
-}
-
-/** We're done sending a log message to the controllers: re-enable controller
- * logging.  Used by CONN_LOG_PROTECT. */
-void
-enable_control_logging(void)
-{
-  if (--disable_log_messages < 0)
-    tor_assert(0);
-}
-
-/** We got a log message: tell any interested control connections. */
-void
-control_event_logmsg(int severity, uint32_t domain, const char *msg)
-{
-  int event;
-
-  /* Don't even think of trying to add stuff to a buffer from a cpuworker
-   * thread. (See #25987 for plan to fix.) */
-  if (! in_main_thread())
-    return;
-
-  if (disable_log_messages)
-    return;
-
-  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
-      severity <= LOG_NOTICE) {
-    char *esc = esc_for_log(msg);
-    ++disable_log_messages;
-    control_event_general_status(severity, "BUG REASON=%s", esc);
-    --disable_log_messages;
-    tor_free(esc);
-  }
-
-  event = log_severity_to_event(severity);
-  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
-    char *b = NULL;
-    const char *s;
-    if (strchr(msg, '\n')) {
-      char *cp;
-      b = tor_strdup(msg);
-      for (cp = b; *cp; ++cp)
-        if (*cp == '\r' || *cp == '\n')
-          *cp = ' ';
-    }
-    switch (severity) {
-      case LOG_DEBUG: s = "DEBUG"; break;
-      case LOG_INFO: s = "INFO"; break;
-      case LOG_NOTICE: s = "NOTICE"; break;
-      case LOG_WARN: s = "WARN"; break;
-      case LOG_ERR: s = "ERR"; break;
-      default: s = "UnknownLogSeverity"; break;
-    }
-    ++disable_log_messages;
-    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
-    if (severity == LOG_ERR) {
-      /* Force a flush, since we may be about to die horribly */
-      queued_events_flush_all(1);
-    }
-    --disable_log_messages;
-    tor_free(b);
-  }
-}
-
-/**
- * Logging callback: called when there is a queued pending log callback.
- */
-void
-control_event_logmsg_pending(void)
-{
-  if (! in_main_thread()) {
-    /* We can't handle this case yet, since we're using a
-     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
-     * use a different mechanism instead: see #25987.
-     **/
-    return;
-  }
-  tor_assert(flush_queued_events_event);
-  mainloop_event_activate(flush_queued_events_event);
-}
-
-/** Called whenever we receive new router descriptors: tell any
- * interested control connections.  <b>routers</b> is a list of
- * routerinfo_t's.
- */
-int
-control_event_descriptors_changed(smartlist_t *routers)
-{
-  char *msg;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
-    return 0;
-
-  {
-    smartlist_t *names = smartlist_new();
-    char *ids;
-    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
-        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
-        router_get_verbose_nickname(b, ri);
-        smartlist_add(names, b);
-      });
-    ids = smartlist_join_strings(names, " ", 0, NULL);
-    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
-    send_control_event_string(EVENT_NEW_DESC,  msg);
-    tor_free(ids);
-    tor_free(msg);
-    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
-    smartlist_free(names);
-  }
-  return 0;
-}
-
-/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
- * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
- * <b>error</b> is non-NULL, it is an error code describing the failure
- * mode of the mapping.
- */
-int
-control_event_address_mapped(const char *from, const char *to, time_t expires,
-                             const char *error, const int cached)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
-    return 0;
-
-  if (expires < 3 || expires == TIME_MAX)
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s NEVER %s%s"
-                                "CACHED=\"%s\"\r\n",
-                                  from, to, error?error:"", error?" ":"",
-                                cached?"YES":"NO");
-  else {
-    char buf[ISO_TIME_LEN+1];
-    char buf2[ISO_TIME_LEN+1];
-    format_local_iso_time(buf,expires);
-    format_iso_time(buf2,expires);
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s \"%s\""
-                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
-                                from, to, buf,
-                                error?error:"", error?" ":"",
-                                buf2, cached?"YES":"NO");
-  }
-
-  return 0;
-}
-
 /** Cached liveness for network liveness events and GETINFO
  */
 
 static int network_is_live = 0;
 
-static int
+int
 get_cached_network_liveness(void)
 {
   return network_is_live;
 }
 
-static void
+void
 set_cached_network_liveness(int liveness)
 {
   network_is_live = liveness;
 }
 
-/** The network liveness has changed; this is called from circuitstats.c
- * whenever we receive a cell, or when timeout expires and we assume the
- * network is down. */
-int
-control_event_network_liveness_update(int liveness)
-{
-  if (liveness > 0) {
-    if (get_cached_network_liveness() <= 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(1);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS UP\r\n");
-    }
-    /* else was already live, no-op */
-  } else {
-    if (get_cached_network_liveness() > 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(0);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS DOWN\r\n");
-    }
-    /* else was already dead, no-op */
-  }
-
-  return 0;
-}
-
-/** Helper function for NS-style events. Constructs and sends an event
- * of type <b>event</b> with string <b>event_string</b> out of the set of
- * networkstatuses <b>statuses</b>. Currently it is used for NS events
- * and NEWCONSENSUS events. */
-static int
-control_event_networkstatus_changed_helper(smartlist_t *statuses,
-                                           uint16_t event,
-                                           const char *event_string)
-{
-  smartlist_t *strs;
-  char *s, *esc = NULL;
-  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
-    return 0;
-
-  strs = smartlist_new();
-  smartlist_add_strdup(strs, "650+");
-  smartlist_add_strdup(strs, event_string);
-  smartlist_add_strdup(strs, "\r\n");
-  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
-    {
-      s = networkstatus_getinfo_helper_single(rs);
-      if (!s) continue;
-      smartlist_add(strs, s);
-    });
-
-  s = smartlist_join_strings(strs, "", 0, NULL);
-  write_escaped_data(s, strlen(s), &esc);
-  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
-  smartlist_free(strs);
-  tor_free(s);
-  send_control_event_string(event,  esc);
-  send_control_event_string(event,
-                            "650 OK\r\n");
-
-  tor_free(esc);
-  return 0;
-}
-
-/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
- * an NS event to any controller that cares. */
-int
-control_event_networkstatus_changed(smartlist_t *statuses)
-{
-  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
-}
-
-/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
- * event consisting of an NS-style line for each relay in the consensus. */
-int
-control_event_newconsensus(const networkstatus_t *consensus)
-{
-  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
-    return 0;
-  return control_event_networkstatus_changed_helper(
-           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
-}
-
-/** Called when we compute a new circuitbuildtimeout */
-int
-control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                               const char *args)
-{
-  const char *type_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
-    return 0;
-
-  switch (type) {
-    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
-      type_string = "COMPUTED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESET:
-      type_string = "RESET";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
-      type_string = "SUSPENDED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_DISCARD:
-      type_string = "DISCARD";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESUME:
-      type_string = "RESUME";
-      break;
-    default:
-      type_string = "UNKNOWN";
-      break;
-  }
-
-  send_control_event(EVENT_BUILDTIMEOUT_SET,
-                     "650 BUILDTIMEOUT_SET %s %s\r\n",
-                     type_string, args);
-
-  return 0;
-}
-
-/** Called when a signal has been processed from signal_callback */
-int
-control_event_signal(uintptr_t signal_num)
-{
-  const char *signal_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
-    return 0;
-
-  switch (signal_num) {
-    case SIGHUP:
-      signal_string = "RELOAD";
-      break;
-    case SIGUSR1:
-      signal_string = "DUMP";
-      break;
-    case SIGUSR2:
-      signal_string = "DEBUG";
-      break;
-    case SIGNEWNYM:
-      signal_string = "NEWNYM";
-      break;
-    case SIGCLEARDNSCACHE:
-      signal_string = "CLEARDNSCACHE";
-      break;
-    case SIGHEARTBEAT:
-      signal_string = "HEARTBEAT";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
-               (unsigned long)signal_num);
-      return -1;
-  }
-
-  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
-                     signal_string);
-  return 0;
-}
-
-/** Called when a single local_routerstatus_t has changed: Sends an NS event
- * to any controller that cares. */
-int
-control_event_networkstatus_changed_single(const routerstatus_t *rs)
-{
-  smartlist_t *statuses;
-  int r;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NS))
-    return 0;
-
-  statuses = smartlist_new();
-  smartlist_add(statuses, (void*)rs);
-  r = control_event_networkstatus_changed(statuses);
-  smartlist_free(statuses);
-  return r;
-}
-
-/** Our own router descriptor has changed; tell any controllers that care.
- */
-int
-control_event_my_descriptor_changed(void)
-{
-  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
-  return 0;
-}
-
-/** Helper: sends a status event where <b>type</b> is one of
- * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
- * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
- * string corresponding to <b>args</b>. */
-static int
-control_event_status(int type, int severity, const char *format, va_list args)
-{
-  char *user_buf = NULL;
-  char format_buf[160];
-  const char *status, *sev;
-
-  switch (type) {
-    case EVENT_STATUS_GENERAL:
-      status = "STATUS_GENERAL";
-      break;
-    case EVENT_STATUS_CLIENT:
-      status = "STATUS_CLIENT";
-      break;
-    case EVENT_STATUS_SERVER:
-      status = "STATUS_SERVER";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status type %d", type);
-      return -1;
-  }
-  switch (severity) {
-    case LOG_NOTICE:
-      sev = "NOTICE";
-      break;
-    case LOG_WARN:
-      sev = "WARN";
-      break;
-    case LOG_ERR:
-      sev = "ERR";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
-      return -1;
-  }
-  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
-                   status, sev)<0) {
-    log_warn(LD_BUG, "Format string too long.");
-    return -1;
-  }
-  tor_vasprintf(&user_buf, format, args);
-
-  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
-  tor_free(user_buf);
-  return 0;
-}
-
-#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
-  int r;                                                        \
-  do {                                                          \
-    va_list ap;                                                 \
-    if (!EVENT_IS_INTERESTING(event))                           \
-      return 0;                                                 \
-                                                                \
-    va_start(ap, format);                                       \
-    r = control_event_status((event), (sev), format, ap);       \
-    va_end(ap);                                                 \
-  } while (0)
-
-/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_general_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_general_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_client_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_client_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_server_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_server_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Called when the status of an entry guard with the given <b>nickname</b>
- * and identity <b>digest</b> has changed to <b>status</b>: tells any
- * controllers that care. */
-int
-control_event_guard(const char *nickname, const char *digest,
-                    const char *status)
-{
-  char hbuf[HEX_DIGEST_LEN+1];
-  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
-  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
-    return 0;
-
-  {
-    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
-    const node_t *node = node_get_by_id(digest);
-    if (node) {
-      node_get_verbose_nickname(node, buf);
-    } else {
-      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
-    }
-    send_control_event(EVENT_GUARD,
-                       "650 GUARD ENTRY %s %s\r\n", buf, status);
-  }
-  return 0;
-}
-
-/** Called when a configuration option changes. This is generally triggered
- * by SETCONF requests and RELOAD/SIGHUP signals. The <b>elements</b> is
- * a smartlist_t containing (key, value, ...) pairs in sequence.
- * <b>value</b> can be NULL. */
-int
-control_event_conf_changed(const smartlist_t *elements)
-{
-  int i;
-  char *result;
-  smartlist_t *lines;
-  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) ||
-      smartlist_len(elements) == 0) {
-    return 0;
-  }
-  lines = smartlist_new();
-  for (i = 0; i < smartlist_len(elements); i += 2) {
-    char *k = smartlist_get(elements, i);
-    char *v = smartlist_get(elements, i+1);
-    if (v == NULL) {
-      smartlist_add_asprintf(lines, "650-%s", k);
-    } else {
-      smartlist_add_asprintf(lines, "650-%s=%s", k, v);
-    }
-  }
-  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
-  send_control_event(EVENT_CONF_CHANGED,
-    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
-  tor_free(result);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  return 0;
-}
-
-/** Helper: Return a newly allocated string containing a path to the
- * file where we store our authentication cookie. */
-char *
-get_controller_cookie_file_name(void)
-{
-  const or_options_t *options = get_options();
-  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
-    return tor_strdup(options->CookieAuthFile);
-  } else {
-    return get_datadir_fname("control_auth_cookie");
-  }
-}
-
-/* Initialize the cookie-based authentication system of the
- * ControlPort. If <b>enabled</b> is 0, then disable the cookie
- * authentication system.  */
-int
-init_control_cookie_authentication(int enabled)
-{
-  char *fname = NULL;
-  int retval;
-
-  if (!enabled) {
-    authentication_cookie_is_set = 0;
-    return 0;
-  }
-
-  fname = get_controller_cookie_file_name();
-  retval = init_cookie_authentication(fname, "", /* no header */
-                                      AUTHENTICATION_COOKIE_LEN,
-                                   get_options()->CookieAuthFileGroupReadable,
-                                      &authentication_cookie,
-                                      &authentication_cookie_is_set);
-  tor_free(fname);
-  return retval;
-}
-
 /** A copy of the process specifier of Tor's owning controller, or
  * NULL if this Tor instance is not currently owned by a process. */
 static char *owning_controller_process_spec = NULL;
@@ -7008,895 +601,12 @@ monitor_owning_controller_process(const char *process_spec)
   }
 }
 
-/** Convert the name of a bootstrapping phase <b>s</b> into strings
- * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
-static int
-bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
-                           const char **summary)
-{
-  switch (s) {
-    case BOOTSTRAP_STATUS_UNDEF:
-      *tag = "undef";
-      *summary = "Undefined";
-      break;
-    case BOOTSTRAP_STATUS_STARTING:
-      *tag = "starting";
-      *summary = "Starting";
-      break;
-    case BOOTSTRAP_STATUS_CONN_DIR:
-      *tag = "conn_dir";
-      *summary = "Connecting to directory server";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE:
-      *tag = "status_handshake";
-      *summary = "Finishing handshake";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_DIR:
-      *tag = "handshake_dir";
-      *summary = "Finishing handshake with directory server";
-      break;
-    case BOOTSTRAP_STATUS_ONEHOP_CREATE:
-      *tag = "onehop_create";
-      *summary = "Establishing an encrypted directory connection";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_STATUS:
-      *tag = "requesting_status";
-      *summary = "Asking for networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_STATUS:
-      *tag = "loading_status";
-      *summary = "Loading networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_KEYS:
-      *tag = "loading_keys";
-      *summary = "Loading authority key certs";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS:
-      *tag = "requesting_descriptors";
-      /* XXXX this appears to incorrectly report internal on most loads */
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Asking for relay descriptors for internal paths" :
-        "Asking for relay descriptors";
-      break;
-    /* If we're sure there are no exits in the consensus,
-     * inform the controller by adding "internal"
-     * to the status summaries.
-     * (We only check this while loading descriptors,
-     * so we may not know in the earlier stages.)
-     * But if there are exits, we can't be sure whether
-     * we're creating internal or exit paths/circuits.
-     * XXXX Or should be use different tags or statuses
-     * for internal and exit/all? */
-    case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS:
-      *tag = "loading_descriptors";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Loading relay descriptors for internal paths" :
-        "Loading relay descriptors";
-      break;
-    case BOOTSTRAP_STATUS_CONN_OR:
-      *tag = "conn_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Connecting to the Tor network internally" :
-        "Connecting to the Tor network";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_OR:
-      *tag = "handshake_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Finishing handshake with first hop of internal circuit" :
-        "Finishing handshake with first hop";
-      break;
-    case BOOTSTRAP_STATUS_CIRCUIT_CREATE:
-      *tag = "circuit_create";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Establishing an internal Tor circuit" :
-        "Establishing a Tor circuit";
-      break;
-    case BOOTSTRAP_STATUS_DONE:
-      *tag = "done";
-      *summary = "Done";
-      break;
-    default:
-//      log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s);
-      *tag = *summary = "unknown";
-      return -1;
-  }
-  return 0;
-}
-
-/** What percentage through the bootstrap process are we? We remember
- * this so we can avoid sending redundant bootstrap status events, and
- * so we can guess context for the bootstrap messages which are
- * ambiguous. It starts at 'undef', but gets set to 'starting' while
- * Tor initializes. */
-static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_percent, but only takes on the enumerated values in
- * bootstrap_status_t.
- */
-static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** As bootstrap_percent, but holds the bootstrapping level at which we last
- * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
- * to avoid flooding the log with a new message every time we get a few more
- * microdescriptors */
-static int notice_bootstrap_percent = 0;
-
-/** How many problems have we had getting to the next bootstrapping phase?
- * These include failure to establish a connection to a Tor relay,
- * failures to finish the TLS handshake, failures to validate the
- * consensus document, etc. */
-static int bootstrap_problems = 0;
-
-/** We only tell the controller once we've hit a threshold of problems
- * for the current phase. */
-#define BOOTSTRAP_PROBLEM_THRESHOLD 10
-
-/** When our bootstrapping progress level changes, but our bootstrapping
- * status has not advanced, we only log at NOTICE when we have made at least
- * this much progress.
- */
-#define BOOTSTRAP_PCT_INCREMENT 5
-
-/** Do the actual logging and notifications for
- * control_event_bootstrap().  Doesn't change any state beyond that.
- */
-static void
-control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
-                             int progress)
-{
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *tag, *summary;
-
-  bootstrap_status_to_string(status, &tag, &summary);
-  /* Locally reset status if there's incremental progress */
-  if (progress)
-    status = progress;
-
-  tor_log(loglevel, LD_CONTROL,
-          "Bootstrapped %d%%: %s", status, summary);
-  tor_snprintf(buf, sizeof(buf),
-               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
-               status, tag, summary);
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "NOTICE %s", buf);
-  control_event_client_status(LOG_NOTICE, "%s", buf);
-}
-
-/** Called when Tor has made progress at bootstrapping its directory
- * information and initial circuits.
- *
- * <b>status</b> is the new status, that is, what task we will be doing
- * next. <b>progress</b> is zero if we just started this task, else it
- * represents progress on the task.
- */
-void
-control_event_bootstrap(bootstrap_status_t status, int progress)
-{
-  int loglevel = LOG_NOTICE;
-
-  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  /* special case for handshaking status, since our TLS handshaking code
-   * can't distinguish what the connection is going to be for. */
-  if (status == BOOTSTRAP_STATUS_HANDSHAKE) {
-    if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_DIR;
-    } else {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_OR;
-    }
-  }
-
-  if (status <= bootstrap_percent) {
-    /* If there's no new progress, return early. */
-    if (!progress || progress <= bootstrap_percent)
-      return;
-    /* Log at INFO if not enough progress happened. */
-    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
-      loglevel = LOG_INFO;
-  }
-
-  control_event_bootstrap_core(loglevel, status, progress);
-
-  if (status > bootstrap_percent) {
-    bootstrap_phase = status; /* new milestone reached */
-    bootstrap_percent = status;
-  }
-  if (progress > bootstrap_percent) {
-    /* incremental progress within a milestone */
-    bootstrap_percent = progress;
-    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
-  }
-  if (loglevel == LOG_NOTICE &&
-      bootstrap_percent > notice_bootstrap_percent) {
-    /* Remember that we gave a notice at this level. */
-    notice_bootstrap_percent = bootstrap_percent;
-  }
-}
-
-/** Flag whether we've opened an OR_CONN yet  */
-static int bootstrap_first_orconn = 0;
-
-/** Like bootstrap_phase, but for (possibly deferred) directory progress */
-static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
-static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-
-/** Defer directory info bootstrap events until we have successfully
- * completed our first connection to a router.  */
-void
-control_event_boot_dir(bootstrap_status_t status, int progress)
-{
-  if (status > bootstrap_dir_progress) {
-    bootstrap_dir_progress = status;
-    bootstrap_dir_phase = status;
-  }
-  if (progress && progress >= bootstrap_dir_progress) {
-    bootstrap_dir_progress = progress;
-  }
-
-  /* Don't report unless we have successfully opened at least one OR_CONN */
-  if (!bootstrap_first_orconn)
-    return;
-
-  control_event_bootstrap(status, progress);
-}
-
-/** Set a flag to allow reporting of directory bootstrap progress.
- * (Code that reports completion of an OR_CONN calls this.)  Also,
- * report directory progress so far. */
-void
-control_event_boot_first_orconn(void)
-{
-  bootstrap_first_orconn = 1;
-  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a human-readable hint
- * as to why, and <b>reason</b> provides a controller-facing short
- * tag.  <b>conn</b> is the connection that caused this problem and
- * can be NULL if a connection cannot be easily identified.
- */
-void
-control_event_bootstrap_problem(const char *warn, const char *reason,
-                                const connection_t *conn, int dowarn)
-{
-  int status = bootstrap_percent;
-  const char *tag = "", *summary = "";
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *recommendation = "ignore";
-  int severity;
-  char *or_id = NULL, *hostaddr = NULL;
-  or_connection_t *or_conn = NULL;
-
-  /* bootstrap_percent must not be in "undefined" state here. */
-  tor_assert(status >= 0);
-
-  if (bootstrap_percent == 100)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  bootstrap_problems++;
-
-  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
-    dowarn = 1;
-
-  /* Don't warn about our bootstrapping status if we are hibernating or
-   * shutting down. */
-  if (we_are_hibernating())
-    dowarn = 0;
-
-  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
-
-  severity = dowarn ? LOG_WARN : LOG_INFO;
-
-  if (dowarn)
-    recommendation = "warn";
-
-  if (conn && conn->type == CONN_TYPE_OR) {
-    /* XXX TO_OR_CONN can't deal with const */
-    or_conn = TO_OR_CONN((connection_t *)conn);
-    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
-  } else {
-    or_id = tor_strdup("?");
-  }
-
-  if (conn)
-    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
-  else
-    hostaddr = tor_strdup("?");
-
-  log_fn(severity,
-         LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; "
-         "count %d; recommendation %s; host %s at %s)",
-         status, summary, warn, reason,
-         bootstrap_problems, recommendation,
-         or_id, hostaddr);
-
-  connection_or_report_broken_states(severity, LD_HANDSHAKE);
-
-  tor_snprintf(buf, sizeof(buf),
-      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
-      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
-      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
-      recommendation,
-      or_id, hostaddr);
-
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "WARN %s", buf);
-  control_event_client_status(LOG_WARN, "%s", buf);
-
-  tor_free(hostaddr);
-  tor_free(or_id);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a hint as to why, and
- * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
- * is the connection that caused this problem.
- */
-MOCK_IMPL(void,
-control_event_bootstrap_prob_or, (const char *warn, int reason,
-                                  or_connection_t *or_conn))
-{
-  int dowarn = 0;
-
-  if (or_conn->have_noted_bootstrap_problem)
-    return;
-
-  or_conn->have_noted_bootstrap_problem = 1;
-
-  if (reason == END_OR_CONN_REASON_NO_ROUTE)
-    dowarn = 1;
-
-  /* If we are using bridges and all our OR connections are now
-     closed, it means that we totally failed to connect to our
-     bridges. Throw a warning. */
-  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
-    dowarn = 1;
-
-  control_event_bootstrap_problem(warn,
-                                  orconn_end_reason_to_control_string(reason),
-                                  TO_CONN(or_conn), dowarn);
-}
-
-/** We just generated a new summary of which countries we've seen clients
- * from recently. Send a copy to the controller in case it wants to
- * display it for the user. */
-void
-control_event_clients_seen(const char *controller_str)
-{
-  send_control_event(EVENT_CLIENTS_SEEN,
-    "650 CLIENTS_SEEN %s\r\n", controller_str);
-}
-
-/** A new pluggable transport called <b>transport_name</b> was
- *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
- *  "server" or "client" depending on the mode of the pluggable
- *  transport.
- *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
- */
-void
-control_event_transport_launched(const char *mode, const char *transport_name,
-                                 tor_addr_t *addr, uint16_t port)
-{
-  send_control_event(EVENT_TRANSPORT_LAUNCHED,
-                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
-                     mode, transport_name, fmt_addr(addr), port);
-}
-
-/** Convert rendezvous auth type to string for HS_DESC control events
- */
-const char *
-rend_auth_type_to_string(rend_auth_type_t auth_type)
-{
-  const char *str;
-
-  switch (auth_type) {
-    case REND_NO_AUTH:
-      str = "NO_AUTH";
-      break;
-    case REND_BASIC_AUTH:
-      str = "BASIC_AUTH";
-      break;
-    case REND_STEALTH_AUTH:
-      str = "STEALTH_AUTH";
-      break;
-    default:
-      str = "UNKNOWN";
-  }
-
-  return str;
-}
-
-/** Return a longname the node whose identity is <b>id_digest</b>. If
- * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
- * returned instead.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-MOCK_IMPL(const char *,
-node_describe_longname_by_id,(const char *id_digest))
-{
-  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
-  node_get_verbose_nickname_by_id(id_digest, longname);
-  return longname;
-}
-
-/** Return either the onion address if the given pointer is a non empty
- * string else the unknown string. */
-static const char *
-rend_hsaddress_str_or_unknown(const char *onion_address)
-{
-  static const char *str_unknown = "UNKNOWN";
-  const char *str_ret = str_unknown;
-
-  /* No valid pointer, unknown it is. */
-  if (!onion_address) {
-    goto end;
-  }
-  /* Empty onion address thus we don't know, unknown it is. */
-  if (onion_address[0] == '\0') {
-    goto end;
-  }
-  /* All checks are good so return the given onion address. */
-  str_ret = onion_address;
-
- end:
-  return str_ret;
-}
-
-/** send HS_DESC requested event.
- *
- * <b>rend_query</b> is used to fetch requested onion address and auth type.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id_base32</b> is the ID of requested hs descriptor.
- * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
- */
-void
-control_event_hs_descriptor_requested(const char *onion_address,
-                                      rend_auth_type_t auth_type,
-                                      const char *id_digest,
-                                      const char *desc_id,
-                                      const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** For an HS descriptor query <b>rend_data</b>, using the
- * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
- * which descriptor ID in the query is the right one.
- *
- * Return a pointer of the binary descriptor ID found in the query's object
- * or NULL if not found. */
-static const char *
-get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
-{
-  int replica;
-  const char *desc_id = NULL;
-  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
-
-  /* Possible if the fetch was done using a descriptor ID. This means that
-   * the HSFETCH command was used. */
-  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
-    desc_id = rend_data_v2->desc_id_fetch;
-    goto end;
-  }
-
-  /* Without a directory fingerprint at this stage, we can't do much. */
-  if (hsdir_fp == NULL) {
-     goto end;
-  }
-
-  /* OK, we have an onion address so now let's find which descriptor ID
-   * is the one associated with the HSDir fingerprint. */
-  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
-       replica++) {
-    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
-
-    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
-      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
-        /* Found it! This descriptor ID is the right one. */
-        desc_id = digest;
-        goto end;
-      }
-    } SMARTLIST_FOREACH_END(fingerprint);
-  }
-
- end:
-  return desc_id;
-}
-
-/** send HS_DESC CREATED event when a local service generates a descriptor.
- *
- * <b>onion_address</b> is service address.
- * <b>desc_id</b> is the descriptor ID.
- * <b>replica</b> is the the descriptor replica number. If it is negative, it
- * is ignored.
- */
-void
-control_event_hs_descriptor_created(const char *onion_address,
-                                    const char *desc_id,
-                                    int replica)
-{
-  char *replica_field = NULL;
-
-  if (BUG(!onion_address || !desc_id)) {
-    return;
-  }
-
-  if (replica >= 0) {
-    tor_asprintf(&replica_field, " REPLICA=%d", replica);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
-                     onion_address, desc_id,
-                     replica_field ? replica_field : "");
-  tor_free(replica_field);
-}
-
-/** send HS_DESC upload event.
- *
- * <b>onion_address</b> is service address.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id</b> is the ID of requested hs descriptor.
- */
-void
-control_event_hs_descriptor_upload(const char *onion_address,
-                                   const char *id_digest,
-                                   const char *desc_id,
-                                   const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!onion_address || !id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
-                     onion_address,
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hsv2_descriptor_received
- * control_event_hsv2_descriptor_failed
- * control_event_hsv3_descriptor_failed
- *
- * So do not call this function directly.
- */
-static void
-event_hs_descriptor_receive_end(const char *action,
-                                const char *onion_address,
-                                const char *desc_id,
-                                rend_auth_type_t auth_type,
-                                const char *hsdir_id_digest,
-                                const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !onion_address)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s %s %s%s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     desc_id ? desc_id : "",
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hs_descriptor_uploaded
- * control_event_hs_descriptor_upload_failed
- *
- * So do not call this function directly.
- */
-void
-control_event_hs_descriptor_upload_end(const char *action,
-                                       const char *onion_address,
-                                       const char *id_digest,
-                                       const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !id_digest)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     node_describe_longname_by_id(id_digest),
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC RECEIVED event
- *
- * called when we successfully received a hidden service descriptor.
- */
-void
-control_event_hsv2_descriptor_received(const char *onion_address,
-                                       const rend_data_t *rend_data,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/* Send HS_DESC RECEIVED event
- *
- * Called when we successfully received a hidden service descriptor. */
-void
-control_event_hsv3_descriptor_received(const char *onion_address,
-                                       const char *desc_id,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/** send HS_DESC UPLOADED event
- *
- * called when we successfully uploaded a hidden service descriptor.
- */
-void
-control_event_hs_descriptor_uploaded(const char *id_digest,
-                                     const char *onion_address)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-
-  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
-                                         id_digest, NULL);
-}
-
-/** Send HS_DESC event to inform controller that query <b>rend_data</b>
- * failed to retrieve hidden service descriptor from directory identified by
- * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
- * add it to REASON= field.
- */
-void
-control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
-                                  desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC event to inform controller that the query to
- * <b>onion_address</b> failed to retrieve hidden service descriptor
- * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
- * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
- * field. */
-void
-control_event_hsv3_descriptor_failed(const char *onion_address,
-                                     const char *desc_id,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !reason)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC_CONTENT event after completion of a successful fetch
- * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
- * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
- * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
- * not trigger the control event. */
-void
-control_event_hs_descriptor_content(const char *onion_address,
-                                    const char *desc_id,
-                                    const char *hsdir_id_digest,
-                                    const char *content)
-{
-  static const char *event_name = "HS_DESC_CONTENT";
-  char *esc_content = NULL;
-
-  if (!onion_address || !desc_id) {
-    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
-             onion_address, desc_id);
-    return;
-  }
-
-  if (content == NULL) {
-    /* Point it to empty content so it can still be escaped. */
-    content = "";
-  }
-  write_escaped_data(content, strlen(content), &esc_content);
-
-  send_control_event(EVENT_HS_DESC_CONTENT,
-                     "650+%s %s %s %s\r\n%s650 OK\r\n",
-                     event_name,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     desc_id,
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     esc_content);
-  tor_free(esc_content);
-}
-
-/** Send HS_DESC event to inform controller upload of hidden service
- * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
- * is not NULL, add it to REASON= field.
- */
-void
-control_event_hs_descriptor_upload_failed(const char *id_digest,
-                                          const char *onion_address,
-                                          const char *reason)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-  control_event_hs_descriptor_upload_end("FAILED", onion_address,
-                                         id_digest, reason);
-}
-
 /** Free any leftover allocated memory of the control.c subsystem. */
 void
 control_free_all(void)
 {
-  smartlist_t *queued_events = NULL;
-
-  stats_prev_n_read = stats_prev_n_written = 0;
-
-  if (authentication_cookie) /* Free the auth cookie */
-    tor_free(authentication_cookie);
-  if (detached_onion_services) { /* Free the detached onion services */
-    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
-    smartlist_free(detached_onion_services);
-  }
-
-  if (queued_control_events_lock) {
-    tor_mutex_acquire(queued_control_events_lock);
-    flush_queued_event_pending = 0;
-    queued_events = queued_control_events;
-    queued_control_events = NULL;
-    tor_mutex_release(queued_control_events_lock);
-  }
-  if (queued_events) {
-    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
-                      queued_event_free(ev));
-    smartlist_free(queued_events);
-  }
-  if (flush_queued_events_event) {
-    mainloop_event_free(flush_queued_events_event);
-    flush_queued_events_event = NULL;
-  }
-  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-  notice_bootstrap_percent = 0;
-  bootstrap_problems = 0;
-  bootstrap_first_orconn = 0;
-  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-  authentication_cookie_is_set = 0;
-  global_event_mask = 0;
-  disable_log_messages = 0;
-  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
-}
-
-#ifdef TOR_UNIT_TESTS
-/* For testing: change the value of global_event_mask */
-void
-control_testing_set_global_event_mask(uint64_t mask)
-{
-  global_event_mask = mask;
+  control_auth_free_all();
+  control_events_free_all();
+  control_cmd_free_all();
+  control_event_bootstrap_reset();
 }
-#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control.h b/src/feature/control/control.h
index a09c1cd11b..8d3595d2ed 100644
--- a/src/feature/control/control.h
+++ b/src/feature/control/control.h
@@ -12,76 +12,6 @@
 #ifndef TOR_CONTROL_H
 #define TOR_CONTROL_H
 
-/** Used to indicate the type of a circuit event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum circuit_status_event_t {
-  CIRC_EVENT_LAUNCHED = 0,
-  CIRC_EVENT_BUILT    = 1,
-  CIRC_EVENT_EXTENDED = 2,
-  CIRC_EVENT_FAILED   = 3,
-  CIRC_EVENT_CLOSED   = 4,
-} circuit_status_event_t;
-
-/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
- * The various types are defined in control-spec.txt . */
-typedef enum circuit_status_minor_event_t {
-  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-  CIRC_MINOR_EVENT_CANNIBALIZED,
-} circuit_status_minor_event_t;
-
-/** Used to indicate the type of a stream event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum stream_status_event_t {
-  STREAM_EVENT_SENT_CONNECT = 0,
-  STREAM_EVENT_SENT_RESOLVE = 1,
-  STREAM_EVENT_SUCCEEDED    = 2,
-  STREAM_EVENT_FAILED       = 3,
-  STREAM_EVENT_CLOSED       = 4,
-  STREAM_EVENT_NEW          = 5,
-  STREAM_EVENT_NEW_RESOLVE  = 6,
-  STREAM_EVENT_FAILED_RETRIABLE = 7,
-  STREAM_EVENT_REMAP        = 8
-} stream_status_event_t;
-
-/** Used to indicate the type of an OR connection event passed to the
- * controller.  The various types are defined in control-spec.txt */
-typedef enum or_conn_status_event_t {
-  OR_CONN_EVENT_LAUNCHED     = 0,
-  OR_CONN_EVENT_CONNECTED    = 1,
-  OR_CONN_EVENT_FAILED       = 2,
-  OR_CONN_EVENT_CLOSED       = 3,
-  OR_CONN_EVENT_NEW          = 4,
-} or_conn_status_event_t;
-
-/** Used to indicate the type of a buildtime event */
-typedef enum buildtimeout_set_event_t {
-  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
-  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
-  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
-  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
-  BUILDTIMEOUT_SET_EVENT_RESUME = 4
-} buildtimeout_set_event_t;
-
-/** Enum describing various stages of bootstrapping, for use with controller
- * bootstrap status events. The values range from 0 to 100. */
-typedef enum {
-  BOOTSTRAP_STATUS_UNDEF=-1,
-  BOOTSTRAP_STATUS_STARTING=0,
-  BOOTSTRAP_STATUS_CONN_DIR=5,
-  BOOTSTRAP_STATUS_HANDSHAKE=-2,
-  BOOTSTRAP_STATUS_HANDSHAKE_DIR=10,
-  BOOTSTRAP_STATUS_ONEHOP_CREATE=15,
-  BOOTSTRAP_STATUS_REQUESTING_STATUS=20,
-  BOOTSTRAP_STATUS_LOADING_STATUS=25,
-  BOOTSTRAP_STATUS_LOADING_KEYS=40,
-  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
-  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
-  BOOTSTRAP_STATUS_CONN_OR=80,
-  BOOTSTRAP_STATUS_HANDSHAKE_OR=85,
-  BOOTSTRAP_STATUS_CIRCUIT_CREATE=90,
-  BOOTSTRAP_STATUS_DONE=100
-} bootstrap_status_t;
-
 control_connection_t *TO_CONTROL_CONN(connection_t *);
 
 #define CONTROL_CONN_STATE_MIN_ 1
@@ -92,18 +22,6 @@ control_connection_t *TO_CONTROL_CONN(connection_t *);
 #define CONTROL_CONN_STATE_NEEDAUTH 2
 #define CONTROL_CONN_STATE_MAX_ 2
 
-/** Reason for remapping an AP connection's address: we have a cached
- * answer. */
-#define REMAP_STREAM_SOURCE_CACHE 1
-/** Reason for remapping an AP connection's address: the exit node told us an
- * answer. */
-#define REMAP_STREAM_SOURCE_EXIT 2
-
-void control_initialize_event_queue(void);
-
-void control_update_global_event_mask(void);
-void control_adjust_event_log_severity(void);
-
 void control_ports_write_to_file(void);
 
 /** Log information about the connection <b>conn</b>, protecting it as with
@@ -124,294 +42,28 @@ void connection_control_closed(control_connection_t *conn);
 
 int connection_control_process_inbuf(control_connection_t *conn);
 
-#define EVENT_NS 0x000F
-int control_event_is_interesting(int event);
-
-void control_per_second_events(void);
-int control_any_per_second_event_enabled(void);
-
-int control_event_circuit_status(origin_circuit_t *circ,
-                                 circuit_status_event_t e, int reason);
-int control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                          int old_purpose);
-int control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                       int old_purpose,
-                                       const struct timeval *old_tv_created);
-int control_event_stream_status(entry_connection_t *conn,
-                                stream_status_event_t e,
-                                int reason);
-int control_event_or_conn_status(or_connection_t *conn,
-                                 or_conn_status_event_t e, int reason);
-int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
-int control_event_stream_bandwidth(edge_connection_t *edge_conn);
-int control_event_stream_bandwidth_used(void);
-int control_event_circ_bandwidth_used(void);
-int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
-int control_event_conn_bandwidth(connection_t *conn);
-int control_event_conn_bandwidth_used(void);
-int control_event_circuit_cell_stats(void);
-void control_event_logmsg(int severity, uint32_t domain, const char *msg);
-void control_event_logmsg_pending(void);
-int control_event_descriptors_changed(smartlist_t *routers);
-int control_event_address_mapped(const char *from, const char *to,
-                                 time_t expires, const char *error,
-                                 const int cached);
-int control_event_my_descriptor_changed(void);
-int control_event_network_liveness_update(int liveness);
-int control_event_networkstatus_changed(smartlist_t *statuses);
-
-int control_event_newconsensus(const networkstatus_t *consensus);
-int control_event_networkstatus_changed_single(const routerstatus_t *rs);
-int control_event_general_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_client_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_server_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-
-int control_event_general_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_client_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_server_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-
-int control_event_guard(const char *nickname, const char *digest,
-                        const char *status);
-int control_event_conf_changed(const smartlist_t *elements);
-int control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                                   const char *args);
-int control_event_signal(uintptr_t signal);
-
-int init_control_cookie_authentication(int enabled);
-char *get_controller_cookie_file_name(void);
-struct config_line_t;
-smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
 void disable_control_logging(void);
 void enable_control_logging(void);
 
 void monitor_owning_controller_process(const char *process_spec);
 
-void control_event_bootstrap(bootstrap_status_t status, int progress);
-MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
-                                                 int reason,
-                                                 or_connection_t *or_conn));
-void control_event_boot_dir(bootstrap_status_t status, int progress);
-void control_event_boot_first_orconn(void);
-void control_event_bootstrap_problem(const char *warn, const char *reason,
-                                     const connection_t *conn, int dowarn);
-
-void control_event_clients_seen(const char *controller_str);
-void control_event_transport_launched(const char *mode,
-                                      const char *transport_name,
-                                      tor_addr_t *addr, uint16_t port);
 const char *rend_auth_type_to_string(rend_auth_type_t auth_type);
-MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
-void control_event_hs_descriptor_requested(const char *onion_address,
-                                           rend_auth_type_t auth_type,
-                                           const char *id_digest,
-                                           const char *desc_id,
-                                           const char *hsdir_index);
-void control_event_hs_descriptor_created(const char *onion_address,
-                                         const char *desc_id,
-                                         int replica);
-void control_event_hs_descriptor_upload(const char *onion_address,
-                                        const char *desc_id,
-                                        const char *hs_dir,
-                                        const char *hsdir_index);
-void control_event_hs_descriptor_upload_end(const char *action,
-                                            const char *onion_address,
-                                            const char *hs_dir,
-                                            const char *reason);
-void control_event_hs_descriptor_uploaded(const char *hs_dir,
-                                          const char *onion_address);
-/* Hidden service v2 HS_DESC specific. */
-void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                          const char *id_digest,
-                                          const char *reason);
-void control_event_hsv2_descriptor_received(const char *onion_address,
-                                            const rend_data_t *rend_data,
-                                            const char *id_digest);
-/* Hidden service v3 HS_DESC specific. */
-void control_event_hsv3_descriptor_failed(const char *onion_address,
-                                          const char *desc_id,
-                                          const char *hsdir_id_digest,
-                                          const char *reason);
-void control_event_hsv3_descriptor_received(const char *onion_address,
-                                            const char *desc_id,
-                                            const char *hsdir_id_digest);
-void control_event_hs_descriptor_upload_failed(const char *hs_dir,
-                                               const char *onion_address,
-                                               const char *reason);
-void control_event_hs_descriptor_content(const char *onion_address,
-                                         const char *desc_id,
-                                         const char *hsdir_fp,
-                                         const char *content);
 void control_free_all(void);
 
-#ifdef CONTROL_PRIVATE
-#include "lib/crypt_ops/crypto_ed25519.h"
-
-/* Recognized asynchronous event types.  It's okay to expand this list
- * because it is used both as a list of v0 event types, and as indices
- * into the bitfield to determine which controllers want which events.
- */
-/* This bitfield has no event zero    0x0000 */
-#define EVENT_MIN_                    0x0001
-#define EVENT_CIRCUIT_STATUS          0x0001
-#define EVENT_STREAM_STATUS           0x0002
-#define EVENT_OR_CONN_STATUS          0x0003
-#define EVENT_BANDWIDTH_USED          0x0004
-#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
-#define EVENT_NEW_DESC                0x0006
-#define EVENT_DEBUG_MSG               0x0007
-#define EVENT_INFO_MSG                0x0008
-#define EVENT_NOTICE_MSG              0x0009
-#define EVENT_WARN_MSG                0x000A
-#define EVENT_ERR_MSG                 0x000B
-#define EVENT_ADDRMAP                 0x000C
-/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
-   can reclaim 0x000D. */
-#define EVENT_DESCCHANGED             0x000E
-/* Exposed above */
-// #define EVENT_NS                   0x000F
-#define EVENT_STATUS_CLIENT           0x0010
-#define EVENT_STATUS_SERVER           0x0011
-#define EVENT_STATUS_GENERAL          0x0012
-#define EVENT_GUARD                   0x0013
-#define EVENT_STREAM_BANDWIDTH_USED   0x0014
-#define EVENT_CLIENTS_SEEN            0x0015
-#define EVENT_NEWCONSENSUS            0x0016
-#define EVENT_BUILDTIMEOUT_SET        0x0017
-#define EVENT_GOT_SIGNAL              0x0018
-#define EVENT_CONF_CHANGED            0x0019
-#define EVENT_CONN_BW                 0x001A
-#define EVENT_CELL_STATS              0x001B
-/* UNUSED :                           0x001C */
-#define EVENT_CIRC_BANDWIDTH_USED     0x001D
-#define EVENT_TRANSPORT_LAUNCHED      0x0020
-#define EVENT_HS_DESC                 0x0021
-#define EVENT_HS_DESC_CONTENT         0x0022
-#define EVENT_NETWORK_LIVENESS        0x0023
-#define EVENT_MAX_                    0x0023
+#ifdef CONTROL_MODULE_PRIVATE
+struct signal_name_t {
+  int sig;
+  const char *signal_name;
+};
+extern const struct signal_name_t signal_table[];
+int get_cached_network_liveness(void);
+void set_cached_network_liveness(int liveness);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
 
-/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
-#define EVENT_CAPACITY_               0x0040
-
-/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
- * different structure, as it can only handle a maximum left shift of 1<<63. */
-
-#if EVENT_MAX_ >= EVENT_CAPACITY_
-#error control_connection_t.event_mask has an event greater than its capacity
+#ifdef CONTROL_PRIVATE
+STATIC char *control_split_incoming_command(char *incoming_cmd,
+                                            size_t *data_len,
+                                            char **current_cmd_out);
 #endif
 
-#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
-
-#define EVENT_MASK_NONE_             ((uint64_t)0x0)
-
-#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
-#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
-                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
-                                          - EVENT_MIN_))
-
-#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
-                                      & EVENT_MASK_BELOW_MAX_)
-
-/* Used only by control.c and test.c */
-STATIC size_t write_escaped_data(const char *data, size_t len, char **out);
-STATIC size_t read_escaped_data(const char *data, size_t len, char **out);
-
-#ifdef TOR_UNIT_TESTS
-MOCK_DECL(STATIC void,
-          send_control_event_string,(uint16_t event, const char *msg));
-
-MOCK_DECL(STATIC void,
-          queue_control_event_string,(uint16_t event, char *msg));
-
-void control_testing_set_global_event_mask(uint64_t mask);
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Helper structure: temporarily stores cell statistics for a circuit. */
-typedef struct cell_stats_t {
-  /** Number of cells added in app-ward direction by command. */
-  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells added in exit-ward direction by command. */
-  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in app-ward direction by command. */
-  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in exit-ward direction by command. */
-  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in app-ward direction by command. */
-  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in exit-ward direction by command. */
-  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
-} cell_stats_t;
-void sum_up_cell_stats_by_command(circuit_t *circ,
-                                  cell_stats_t *cell_stats);
-void append_cell_stats_by_command(smartlist_t *event_parts,
-                                  const char *key,
-                                  const uint64_t *include_if_non_zero,
-                                  const uint64_t *number_to_include);
-void format_cell_stats(char **event_string, circuit_t *circ,
-                       cell_stats_t *cell_stats);
-STATIC char *get_bw_samples(void);
-
-/* ADD_ONION secret key to create an ephemeral service. The command supports
- * multiple versions so this union stores the key and passes it to the HS
- * subsystem depending on the requested version. */
-typedef union add_onion_secret_key_t {
-  /* Hidden service v2 secret key. */
-  crypto_pk_t *v2;
-  /* Hidden service v3 secret key. */
-  ed25519_secret_key_t *v3;
-} add_onion_secret_key_t;
-
-STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
-                                   const char **key_new_alg_out,
-                                   char **key_new_blob_out,
-                                   add_onion_secret_key_t *decoded_key,
-                                   int *hs_version, char **err_msg_out);
-
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
-
-STATIC int getinfo_helper_onions(
-    control_connection_t *control_conn,
-    const char *question,
-    char **answer,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_networkstatus(
-    const char *flavor,
-    download_status_t **dl_to_emit,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_cert(
-    const char *fp_sk_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_desc(
-    const char *desc_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_bridge(
-    const char *bridge_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC int getinfo_helper_downloads(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_dir(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_current_time(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-
-#endif /* defined(CONTROL_PRIVATE) */
-
 #endif /* !defined(TOR_CONTROL_H) */
diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c
new file mode 100644
index 0000000000..a574d07b33
--- /dev/null
+++ b/src/feature/control/control_auth.c
@@ -0,0 +1,441 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.c
+ * \brief Authentication for Tor's control-socket interface.
+ **/
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_proto.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+
+#include "lib/crypt_ops/crypto_s2k.h"
+
+/** If we're using cookie-type authentication, how long should our cookies be?
+ */
+#define AUTHENTICATION_COOKIE_LEN 32
+
+/** If true, we've set authentication_cookie to a secret code and
+ * stored it to disk. */
+static int authentication_cookie_is_set = 0;
+/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
+ * and which we're using to authenticate controllers.  (If the controller can
+ * read it off disk, it has permission to connect.) */
+static uint8_t *authentication_cookie = NULL;
+
+#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
+  "Tor safe cookie authentication server-to-controller hash"
+#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
+  "Tor safe cookie authentication controller-to-server hash"
+#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
+
+/** Helper: Return a newly allocated string containing a path to the
+ * file where we store our authentication cookie. */
+char *
+get_controller_cookie_file_name(void)
+{
+  const or_options_t *options = get_options();
+  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
+    return tor_strdup(options->CookieAuthFile);
+  } else {
+    return get_datadir_fname("control_auth_cookie");
+  }
+}
+
+/* Initialize the cookie-based authentication system of the
+ * ControlPort. If <b>enabled</b> is 0, then disable the cookie
+ * authentication system.  */
+int
+init_control_cookie_authentication(int enabled)
+{
+  char *fname = NULL;
+  int retval;
+
+  if (!enabled) {
+    authentication_cookie_is_set = 0;
+    return 0;
+  }
+
+  fname = get_controller_cookie_file_name();
+  retval = init_cookie_authentication(fname, "", /* no header */
+                                      AUTHENTICATION_COOKIE_LEN,
+                                   get_options()->CookieAuthFileGroupReadable,
+                                      &authentication_cookie,
+                                      &authentication_cookie_is_set);
+  tor_free(fname);
+  return retval;
+}
+
+/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
+ * Return a smartlist of acceptable passwords (unterminated strings of
+ * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
+ * failure.
+ */
+smartlist_t *
+decode_hashed_passwords(config_line_t *passwords)
+{
+  char decoded[64];
+  config_line_t *cl;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(passwords);
+
+  for (cl = passwords; cl; cl = cl->next) {
+    const char *hashed = cl->value;
+
+    if (!strcmpstart(hashed, "16:")) {
+      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
+                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
+          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
+        goto err;
+      }
+    } else {
+        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
+            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
+          goto err;
+        }
+    }
+    smartlist_add(sl,
+                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
+  }
+
+  return sl;
+
+ err:
+  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
+  smartlist_free(sl);
+  return NULL;
+}
+
+const control_cmd_syntax_t authchallenge_syntax = {
+   .min_args = 1,
+   .max_args = 1,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHCHALLENGE command. */
+int
+handle_control_authchallenge(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  char *client_nonce;
+  size_t client_nonce_len;
+  char server_hash[DIGEST256_LEN];
+  char server_hash_encoded[HEX_DIGEST256_LEN+1];
+  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
+  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
+
+  if (strcasecmp(smartlist_get(args->args, 0), "SAFECOOKIE")) {
+    control_write_endreply(conn, 513,
+                           "AUTHCHALLENGE only supports SAFECOOKIE "
+                           "authentication");
+    goto fail;
+  }
+  if (!authentication_cookie_is_set) {
+    control_write_endreply(conn, 515, "Cookie authentication is disabled");
+    goto fail;
+  }
+  if (args->kwargs == NULL || args->kwargs->next != NULL) {
+    control_write_endreply(conn, 512,
+                           "Wrong number of arguments for AUTHCHALLENGE");
+    goto fail;
+  }
+  if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHCHALLENGE does not accept keyword "
+                           "arguments.");
+    goto fail;
+  }
+
+  bool contains_quote = strchr(args->raw_body, '\"');
+  if (contains_quote) {
+    /* The nonce was quoted */
+    client_nonce = tor_strdup(args->kwargs->value);
+    client_nonce_len = strlen(client_nonce);
+  } else {
+    /* The nonce was should be in hex. */
+    const char *hex_nonce = args->kwargs->value;
+    client_nonce_len = strlen(hex_nonce) / 2;
+    client_nonce = tor_malloc(client_nonce_len);
+    if (base16_decode(client_nonce, client_nonce_len, hex_nonce,
+                      strlen(hex_nonce)) != (int)client_nonce_len) {
+      control_write_endreply(conn, 513, "Invalid base16 client nonce");
+      tor_free(client_nonce);
+      goto fail;
+    }
+  }
+
+  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+  /* Now compute and send the server-to-controller response, and the
+   * server's nonce. */
+  tor_assert(authentication_cookie != NULL);
+
+  {
+    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
+                      client_nonce_len +
+                      SAFECOOKIE_SERVER_NONCE_LEN);
+    char *tmp = tor_malloc_zero(tmp_len);
+    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
+    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
+           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+    crypto_hmac_sha256(server_hash,
+                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
+                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    crypto_hmac_sha256(client_hash,
+                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
+                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    conn->safecookie_client_hash = client_hash;
+
+    tor_free(tmp);
+  }
+
+  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
+                server_hash, sizeof(server_hash));
+  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
+                server_nonce, sizeof(server_nonce));
+
+  control_printf_endreply(conn, 250,
+                          "AUTHCHALLENGE SERVERHASH=%s SERVERNONCE=%s",
+                          server_hash_encoded,
+                          server_nonce_encoded);
+
+  tor_free(client_nonce);
+  return 0;
+ fail:
+  connection_mark_for_close(TO_CONN(conn));
+  return -1;
+}
+
+const control_cmd_syntax_t authenticate_syntax = {
+   .max_args = 0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHENTICATE message.  Check whether the
+ * authentication is valid, and if so, update the connection's state to
+ * OPEN.  Reply with DONE or ERROR.
+ */
+int
+handle_control_authenticate(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  bool used_quoted_string = false;
+  const or_options_t *options = get_options();
+  const char *errstr = "Unknown error";
+  char *password;
+  size_t password_len;
+  int bad_cookie=0, bad_password=0;
+  smartlist_t *sl = NULL;
+
+  if (args->kwargs == NULL) {
+    password = tor_strdup("");
+    password_len = 0;
+  } else if (args->kwargs->next) {
+    control_write_endreply(conn, 512, "Too many arguments to AUTHENTICATE.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHENTICATE does not accept keyword arguments.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strchr(args->raw_body, '\"')) {
+    used_quoted_string = true;
+    password = tor_strdup(args->kwargs->value);
+    password_len = strlen(password);
+  } else {
+    const char *hex_passwd = args->kwargs->value;
+    password_len = strlen(hex_passwd) / 2;
+    password = tor_malloc(password_len+1);
+    if (base16_decode(password, password_len+1, hex_passwd, strlen(hex_passwd))
+                      != (int) password_len) {
+      control_write_endreply(conn, 551,
+            "Invalid hexadecimal encoding.  Maybe you tried a plain text "
+            "password?  If so, the standard requires that you put it in "
+            "double quotes.");
+      connection_mark_for_close(TO_CONN(conn));
+      tor_free(password);
+      return 0;
+    }
+  }
+
+  if (conn->safecookie_client_hash != NULL) {
+    /* The controller has chosen safe cookie authentication; the only
+     * acceptable authentication value is the controller-to-server
+     * response. */
+
+    tor_assert(authentication_cookie_is_set);
+
+    if (password_len != DIGEST256_LEN) {
+      log_warn(LD_CONTROL,
+               "Got safe cookie authentication response with wrong length "
+               "(%d)", (int)password_len);
+      errstr = "Wrong length for safe cookie response.";
+      goto err;
+    }
+
+    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
+      log_warn(LD_CONTROL,
+               "Got incorrect safe cookie authentication response");
+      errstr = "Safe cookie response did not match expected value.";
+      goto err;
+    }
+
+    tor_free(conn->safecookie_client_hash);
+    goto ok;
+  }
+
+  if (!options->CookieAuthentication && !options->HashedControlPassword &&
+      !options->HashedControlSessionPassword) {
+    /* if Tor doesn't demand any stronger authentication, then
+     * the controller can get in with anything. */
+    goto ok;
+  }
+
+  if (options->CookieAuthentication) {
+    int also_password = options->HashedControlPassword != NULL ||
+      options->HashedControlSessionPassword != NULL;
+    if (password_len != AUTHENTICATION_COOKIE_LEN) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
+                 "(%d)", (int)password_len);
+        errstr = "Wrong length on authentication cookie.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else if (tor_memneq(authentication_cookie, password, password_len)) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
+        errstr = "Authentication cookie did not match expected value.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else {
+      goto ok;
+    }
+  }
+
+  if (options->HashedControlPassword ||
+      options->HashedControlSessionPassword) {
+    int bad = 0;
+    smartlist_t *sl_tmp;
+    char received[DIGEST_LEN];
+    int also_cookie = options->CookieAuthentication;
+    sl = smartlist_new();
+    if (options->HashedControlPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (options->HashedControlSessionPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (bad) {
+      if (!also_cookie) {
+        log_warn(LD_BUG,
+                 "Couldn't decode HashedControlPassword: invalid base16");
+        errstr="Couldn't decode HashedControlPassword value in configuration.";
+        goto err;
+      }
+      bad_password = 1;
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+    } else {
+      SMARTLIST_FOREACH(sl, char *, expected,
+      {
+        secret_to_key_rfc2440(received,DIGEST_LEN,
+                              password,password_len,expected);
+        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
+                      received, DIGEST_LEN))
+          goto ok;
+      });
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+
+      if (used_quoted_string)
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration";
+      else
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration. Maybe you tried a plain text password? "
+          "If so, the standard requires that you put it in double quotes.";
+      bad_password = 1;
+      if (!also_cookie)
+        goto err;
+    }
+  }
+
+  /** We only get here if both kinds of authentication failed. */
+  tor_assert(bad_password && bad_cookie);
+  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
+  errstr = "Password did not match HashedControlPassword *or* authentication "
+    "cookie.";
+
+ err:
+  tor_free(password);
+  control_printf_endreply(conn, 515, "Authentication failed: %s", errstr);
+  connection_mark_for_close(TO_CONN(conn));
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+ ok:
+  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
+           ")", conn->base_.s);
+  send_control_done(conn);
+  conn->base_.state = CONTROL_CONN_STATE_OPEN;
+  tor_free(password);
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+}
+
+void
+control_auth_free_all(void)
+{
+  if (authentication_cookie) /* Free the auth cookie */
+    tor_free(authentication_cookie);
+  authentication_cookie_is_set = 0;
+}
diff --git a/src/feature/control/control_auth.h b/src/feature/control/control_auth.h
new file mode 100644
index 0000000000..246e18ccbc
--- /dev/null
+++ b/src/feature/control/control_auth.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.h
+ * \brief Header file for control_auth.c.
+ **/
+
+#ifndef TOR_CONTROL_AUTH_H
+#define TOR_CONTROL_AUTH_H
+
+struct control_cmd_args_t;
+struct control_cmd_syntax_t;
+
+int init_control_cookie_authentication(int enabled);
+char *get_controller_cookie_file_name(void);
+struct config_line_t;
+smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
+
+int handle_control_authchallenge(control_connection_t *conn,
+                                 const struct control_cmd_args_t *args);
+int handle_control_authenticate(control_connection_t *conn,
+                                const struct control_cmd_args_t *args);
+void control_auth_free_all(void);
+
+extern const struct control_cmd_syntax_t authchallenge_syntax;
+extern const struct control_cmd_syntax_t authenticate_syntax;
+
+#endif /* !defined(TOR_CONTROL_AUTH_H) */
diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c
new file mode 100644
index 0000000000..098e24682e
--- /dev/null
+++ b/src/feature/control/control_bootstrap.c
@@ -0,0 +1,383 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_bootstrap.c
+ * \brief Provide bootstrap progress events for the control port.
+ */
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "core/or/connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/reasons.h"
+#include "feature/control/control_events.h"
+#include "feature/hibernate/hibernate.h"
+#include "lib/malloc/malloc.h"
+
+/** A sufficiently large size to record the last bootstrap phase string. */
+#define BOOTSTRAP_MSG_LEN 1024
+
+/** What was the last bootstrap phase message we sent? We keep track
+ * of this so we can respond to getinfo status/bootstrap-phase queries. */
+static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
+
+/** Table to convert bootstrap statuses to strings.  */
+static const struct {
+  bootstrap_status_t status;
+  const char *tag;
+  const char *summary;
+} boot_to_str_tab[] = {
+  { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" },
+  { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" },
+
+  /* Initial connection to any relay */
+
+  { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt",
+    "Connected to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy",
+    "Connected to proxy" },
+  { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" },
+  { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE, "handshake",
+    "Handshaking with a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done",
+    "Handshake with a relay done" },
+
+  /* Loading directory info */
+
+  { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create",
+    "Establishing an encrypted directory connection" },
+  { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status",
+    "Asking for networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status",
+    "Loading networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys",
+    "Loading authority key certs" },
+  { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors",
+    "Asking for relay descriptors" },
+  { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors",
+    "Loading relay descriptors" },
+  { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo",
+    "Loaded enough directory info to build circuits" },
+
+  /* Connecting to a relay for AP circuits */
+
+  { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt",
+    "Connecting to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt",
+    "Connected to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy",
+    "Connecting to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy",
+    "Connected to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN, "ap_conn",
+    "Connecting to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done",
+    "Connected to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake",
+    "Finishing handshake with a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done",
+    "Handshake finished with a relay to build circuits" },
+
+  /* Creating AP circuits */
+
+  { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create",
+    "Establishing a Tor circuit" },
+  { BOOTSTRAP_STATUS_DONE, "done", "Done" },
+};
+#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0]))
+
+/** Convert the name of a bootstrapping phase <b>s</b> into strings
+ * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
+static int
+bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
+                           const char **summary)
+{
+  for (size_t i = 0; i < N_BOOT_TO_STR; i++) {
+    if (s == boot_to_str_tab[i].status) {
+      *tag = boot_to_str_tab[i].tag;
+      *summary = boot_to_str_tab[i].summary;
+      return 0;
+    }
+  }
+
+  *tag = *summary = "unknown";
+  return -1;
+}
+
+/** What percentage through the bootstrap process are we? We remember
+ * this so we can avoid sending redundant bootstrap status events, and
+ * so we can guess context for the bootstrap messages which are
+ * ambiguous. It starts at 'undef', but gets set to 'starting' while
+ * Tor initializes. */
+static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_percent, but only takes on the enumerated values in
+ * bootstrap_status_t.
+ */
+static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** As bootstrap_percent, but holds the bootstrapping level at which we last
+ * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
+ * to avoid flooding the log with a new message every time we get a few more
+ * microdescriptors */
+static int notice_bootstrap_percent = 0;
+
+/** How many problems have we had getting to the next bootstrapping phase?
+ * These include failure to establish a connection to a Tor relay,
+ * failures to finish the TLS handshake, failures to validate the
+ * consensus document, etc. */
+static int bootstrap_problems = 0;
+
+/** We only tell the controller once we've hit a threshold of problems
+ * for the current phase. */
+#define BOOTSTRAP_PROBLEM_THRESHOLD 10
+
+/** When our bootstrapping progress level changes, but our bootstrapping
+ * status has not advanced, we only log at NOTICE when we have made at least
+ * this much progress.
+ */
+#define BOOTSTRAP_PCT_INCREMENT 5
+
+/** Do the actual logging and notifications for
+ * control_event_bootstrap().  Doesn't change any state beyond that.
+ */
+static void
+control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
+                             int progress)
+{
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *tag, *summary;
+
+  bootstrap_status_to_string(status, &tag, &summary);
+  /* Locally reset status if there's incremental progress */
+  if (progress)
+    status = progress;
+
+  tor_log(loglevel, LD_CONTROL,
+          "Bootstrapped %d%% (%s): %s", status, tag, summary);
+  tor_snprintf(buf, sizeof(buf),
+               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
+               status, tag, summary);
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "NOTICE %s", buf);
+  control_event_client_status(LOG_NOTICE, "%s", buf);
+}
+
+/** Called when Tor has made progress at bootstrapping its directory
+ * information and initial circuits.
+ *
+ * <b>status</b> is the new status, that is, what task we will be doing
+ * next. <b>progress</b> is zero if we just started this task, else it
+ * represents progress on the task.
+ */
+void
+control_event_bootstrap(bootstrap_status_t status, int progress)
+{
+  int loglevel = LOG_NOTICE;
+
+  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  if (status <= bootstrap_percent) {
+    /* If there's no new progress, return early. */
+    if (!progress || progress <= bootstrap_percent)
+      return;
+    /* Log at INFO if not enough progress happened. */
+    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
+      loglevel = LOG_INFO;
+  }
+
+  control_event_bootstrap_core(loglevel, status, progress);
+
+  if (status > bootstrap_percent) {
+    bootstrap_phase = status; /* new milestone reached */
+    bootstrap_percent = status;
+  }
+  if (progress > bootstrap_percent) {
+    /* incremental progress within a milestone */
+    bootstrap_percent = progress;
+    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
+  }
+  if (loglevel == LOG_NOTICE &&
+      bootstrap_percent > notice_bootstrap_percent) {
+    /* Remember that we gave a notice at this level. */
+    notice_bootstrap_percent = bootstrap_percent;
+  }
+}
+
+/** Flag whether we've opened an OR_CONN yet  */
+static int bootstrap_first_orconn = 0;
+
+/** Like bootstrap_phase, but for (possibly deferred) directory progress */
+static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
+static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+
+/** Defer directory info bootstrap events until we have successfully
+ * completed our first connection to a router.  */
+void
+control_event_boot_dir(bootstrap_status_t status, int progress)
+{
+  if (status > bootstrap_dir_progress) {
+    bootstrap_dir_progress = status;
+    bootstrap_dir_phase = status;
+  }
+  if (progress && progress >= bootstrap_dir_progress) {
+    bootstrap_dir_progress = progress;
+  }
+
+  /* Don't report unless we have successfully opened at least one OR_CONN */
+  if (!bootstrap_first_orconn)
+    return;
+
+  control_event_bootstrap(status, progress);
+}
+
+/** Set a flag to allow reporting of directory bootstrap progress.
+ * (Code that reports completion of an OR_CONN calls this.)  Also,
+ * report directory progress so far. */
+void
+control_event_boot_first_orconn(void)
+{
+  bootstrap_first_orconn = 1;
+  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a human-readable hint
+ * as to why, and <b>reason</b> provides a controller-facing short
+ * tag.  <b>conn</b> is the connection that caused this problem and
+ * can be NULL if a connection cannot be easily identified.
+ */
+void
+control_event_bootstrap_problem(const char *warn, const char *reason,
+                                const connection_t *conn, int dowarn)
+{
+  int status = bootstrap_percent;
+  const char *tag = "", *summary = "";
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *recommendation = "ignore";
+  int severity;
+  char *or_id = NULL, *hostaddr = NULL;
+  or_connection_t *or_conn = NULL;
+
+  /* bootstrap_percent must not be in "undefined" state here. */
+  tor_assert(status >= 0);
+
+  if (bootstrap_percent == 100)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  bootstrap_problems++;
+
+  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
+    dowarn = 1;
+
+  /* Don't warn about our bootstrapping status if we are hibernating or
+   * shutting down. */
+  if (we_are_hibernating())
+    dowarn = 0;
+
+  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
+
+  severity = dowarn ? LOG_WARN : LOG_INFO;
+
+  if (dowarn)
+    recommendation = "warn";
+
+  if (conn && conn->type == CONN_TYPE_OR) {
+    /* XXX TO_OR_CONN can't deal with const */
+    or_conn = TO_OR_CONN((connection_t *)conn);
+    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
+  } else {
+    or_id = tor_strdup("?");
+  }
+
+  if (conn)
+    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
+  else
+    hostaddr = tor_strdup("?");
+
+  log_fn(severity,
+         LD_CONTROL, "Problem bootstrapping. Stuck at %d%% (%s): %s. (%s; %s; "
+         "count %d; recommendation %s; host %s at %s)",
+         status, tag, summary, warn, reason,
+         bootstrap_problems, recommendation,
+         or_id, hostaddr);
+
+  connection_or_report_broken_states(severity, LD_HANDSHAKE);
+
+  tor_snprintf(buf, sizeof(buf),
+      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
+      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
+      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
+      recommendation,
+      or_id, hostaddr);
+
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "WARN %s", buf);
+  control_event_client_status(LOG_WARN, "%s", buf);
+
+  tor_free(hostaddr);
+  tor_free(or_id);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a hint as to why, and
+ * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
+ * is the connection that caused this problem.
+ */
+MOCK_IMPL(void,
+control_event_bootstrap_prob_or, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
+{
+  int dowarn = 0;
+
+  if (or_conn->have_noted_bootstrap_problem)
+    return;
+
+  or_conn->have_noted_bootstrap_problem = 1;
+
+  if (reason == END_OR_CONN_REASON_NO_ROUTE)
+    dowarn = 1;
+
+  /* If we are using bridges and all our OR connections are now
+     closed, it means that we totally failed to connect to our
+     bridges. Throw a warning. */
+  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
+    dowarn = 1;
+
+  control_event_bootstrap_problem(warn,
+                                  orconn_end_reason_to_control_string(reason),
+                                  TO_CONN(or_conn), dowarn);
+}
+
+/** Return a copy of the last sent bootstrap message. */
+char *
+control_event_boot_last_msg(void)
+{
+  return tor_strdup(last_sent_bootstrap_message);
+}
+
+/** Reset bootstrap tracking state. */
+void
+control_event_bootstrap_reset(void)
+{
+  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+  notice_bootstrap_percent = 0;
+  bootstrap_problems = 0;
+  bootstrap_first_orconn = 0;
+  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
+}
diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c
new file mode 100644
index 0000000000..68d05abb73
--- /dev/null
+++ b/src/feature/control/control_cmd.c
@@ -0,0 +1,2399 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.c
+ * \brief Implement various commands for Tor's control-socket interface.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/confmgt/confparse.h"
+#include "app/main/main.h"
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/dnsserv.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/hs/hs_control.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "feature/rend/rendservice.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/rend/rend_authorized_client_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+
+static int control_setconf_helper(control_connection_t *conn,
+                                  const control_cmd_args_t *args,
+                                  int use_defaults);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/**
+ * Release all storage held in <b>args</b>
+ **/
+void
+control_cmd_args_free_(control_cmd_args_t *args)
+{
+  if (! args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, tor_free(c));
+    smartlist_free(args->args);
+  }
+  config_free_lines(args->kwargs);
+  tor_free(args->cmddata);
+
+  tor_free(args);
+}
+
+/** Erase all memory held in <b>args</b>. */
+void
+control_cmd_args_wipe(control_cmd_args_t *args)
+{
+  if (!args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, memwipe(c, 0, strlen(c)));
+  }
+  for (config_line_t *line = args->kwargs; line; line = line->next) {
+    memwipe(line->key, 0, strlen(line->key));
+    memwipe(line->value, 0, strlen(line->value));
+  }
+  if (args->cmddata)
+    memwipe(args->cmddata, 0, args->cmddata_len);
+}
+
+/**
+ * Return true iff any element of the NULL-terminated <b>array</b> matches
+ * <b>kwd</b>. Case-insensitive.
+ **/
+static bool
+string_array_contains_keyword(const char **array, const char *kwd)
+{
+  for (unsigned i = 0; array[i]; ++i) {
+    if (! strcasecmp(array[i], kwd))
+      return true;
+  }
+  return false;
+}
+
+/** Helper for argument parsing: check whether the keyword arguments just
+ * parsed in <b>result</b> were well-formed according to <b>syntax</b>.
+ *
+ * On success, return 0.  On failure, return -1 and set *<b>error_out</b>
+ * to a newly allocated error string.
+ **/
+static int
+kvline_check_keyword_args(const control_cmd_args_t *result,
+                          const control_cmd_syntax_t *syntax,
+                          char **error_out)
+{
+  if (result->kwargs == NULL) {
+    tor_asprintf(error_out, "Cannot parse keyword argument(s)");
+    return -1;
+  }
+
+  if (! syntax->allowed_keywords) {
+    /* All keywords are permitted. */
+    return 0;
+  }
+
+  /* Check for unpermitted arguments */
+  const config_line_t *line;
+  for (line = result->kwargs; line; line = line->next) {
+    if (! string_array_contains_keyword(syntax->allowed_keywords,
+                                        line->key)) {
+      tor_asprintf(error_out, "Unrecognized keyword argument %s",
+                   escaped(line->key));
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Helper: parse the arguments to a command according to <b>syntax</b>.  On
+ * success, set *<b>error_out</b> to NULL and return a newly allocated
+ * control_cmd_args_t.  On failure, set *<b>error_out</b> to newly allocated
+ * error string, and return NULL.
+ **/
+STATIC control_cmd_args_t *
+control_cmd_parse_args(const char *command,
+                       const control_cmd_syntax_t *syntax,
+                       size_t body_len,
+                       const char *body,
+                       char **error_out)
+{
+  *error_out = NULL;
+  control_cmd_args_t *result = tor_malloc_zero(sizeof(control_cmd_args_t));
+  const char *cmdline;
+  char *cmdline_alloc = NULL;
+  tor_assert(syntax->max_args < INT_MAX || syntax->max_args == UINT_MAX);
+
+  result->command = command;
+
+  if (syntax->store_raw_body) {
+    tor_assert(body[body_len] == 0);
+    result->raw_body = body;
+  }
+
+  const char *eol = memchr(body, '\n', body_len);
+  if (syntax->want_cmddata) {
+    if (! eol || (eol+1) == body+body_len) {
+      *error_out = tor_strdup("Empty body");
+      goto err;
+    }
+    cmdline_alloc = tor_memdup_nulterm(body, eol-body);
+    cmdline = cmdline_alloc;
+    ++eol;
+    result->cmddata_len = read_escaped_data(eol, (body+body_len)-eol,
+                                           &result->cmddata);
+  } else {
+    if (eol && (eol+1) != body+body_len) {
+      *error_out = tor_strdup("Unexpected body");
+      goto err;
+    }
+    cmdline = body;
+  }
+
+  result->args = smartlist_new();
+  smartlist_split_string(result->args, cmdline, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK,
+                         (int)(syntax->max_args+1));
+  size_t n_args = smartlist_len(result->args);
+  if (n_args < syntax->min_args) {
+    tor_asprintf(error_out, "Need at least %u argument(s)",
+                 syntax->min_args);
+    goto err;
+  } else if (n_args > syntax->max_args && ! syntax->accept_keywords) {
+    tor_asprintf(error_out, "Cannot accept more than %u argument(s)",
+                 syntax->max_args);
+    goto err;
+  }
+
+  if (n_args > syntax->max_args) {
+    /* We have extra arguments after the positional arguments, and we didn't
+       treat them as an error, so they must count as keyword arguments: Either
+       K=V pairs, or flags, or both. */
+    tor_assert(n_args == syntax->max_args + 1);
+    tor_assert(syntax->accept_keywords);
+    char *remainder = smartlist_pop_last(result->args);
+    result->kwargs = kvline_parse(remainder, syntax->kvline_flags);
+    tor_free(remainder);
+    if (kvline_check_keyword_args(result, syntax, error_out) < 0) {
+      goto err;
+    }
+  }
+
+  tor_assert_nonfatal(*error_out == NULL);
+  goto done;
+ err:
+  tor_assert_nonfatal(*error_out != NULL);
+  control_cmd_args_free(result);
+ done:
+  tor_free(cmdline_alloc);
+  return result;
+}
+
+/**
+ * Return true iff <b>lines</b> contains <b>flags</b> as a no-value
+ * (keyword-only) entry.
+ **/
+static bool
+config_lines_contain_flag(const config_line_t *lines, const char *flag)
+{
+  const config_line_t *line = config_line_find_case(lines, flag);
+  return line && !strcmp(line->value, "");
+}
+
+static const control_cmd_syntax_t setconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a SETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body.*/
+static int
+handle_control_setconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 0);
+}
+
+static const control_cmd_syntax_t resetconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a RESETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body. */
+static int
+handle_control_resetconf(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 1);
+}
+
+static const control_cmd_syntax_t getconf_syntax = {
+  .max_args=UINT_MAX
+};
+
+/** Called when we receive a GETCONF message.  Parse the request, and
+ * reply with a CONFVALUE or an ERROR message */
+static int
+handle_control_getconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *msg = NULL;
+  size_t msg_len;
+  const or_options_t *options = get_options();
+  int i, len;
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    if (!option_is_recognized(q)) {
+      smartlist_add(unrecognized, (char*) q);
+    } else {
+      config_line_t *answer = option_get_assignment(options,q);
+      if (!answer) {
+        const char *name = option_get_canonical_name(q);
+        smartlist_add_asprintf(answers, "250-%s\r\n", name);
+      }
+
+      while (answer) {
+        config_line_t *next;
+        smartlist_add_asprintf(answers, "250-%s=%s\r\n",
+                     answer->key, answer->value);
+
+        next = answer->next;
+        tor_free(answer->key);
+        tor_free(answer->value);
+        tor_free(answer);
+        answer = next;
+      }
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if ((len = smartlist_len(unrecognized))) {
+    for (i=0; i < len-1; ++i)
+      control_printf_midreply(conn, 552,
+                              "Unrecognized configuration key \"%s\"",
+                              (char*)smartlist_get(unrecognized, i));
+    control_printf_endreply(conn, 552,
+                            "Unrecognized configuration key \"%s\"",
+                            (char*)smartlist_get(unrecognized, len-1));
+  } else if ((len = smartlist_len(answers))) {
+    char *tmp = smartlist_get(answers, len-1);
+    tor_assert(strlen(tmp)>4);
+    tmp[3] = ' ';
+    msg = smartlist_join_strings(answers, "", 0, &msg_len);
+    connection_buf_add(msg, msg_len, TO_CONN(conn));
+  } else {
+    send_control_done(conn);
+  }
+
+  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
+  smartlist_free(answers);
+  smartlist_free(unrecognized);
+
+  tor_free(msg);
+
+  return 0;
+}
+
+static const control_cmd_syntax_t loadconf_syntax = {
+  .want_cmddata = true
+};
+
+/** Called when we get a +LOADCONF message. */
+static int
+handle_control_loadconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  setopt_err_t retval;
+  char *errstring = NULL;
+
+  retval = options_init_from_string(NULL, args->cmddata,
+                                    CMD_RUN_TOR, NULL, &errstring);
+
+  if (retval != SETOPT_OK)
+    log_warn(LD_CONTROL,
+             "Controller gave us config file that didn't validate: %s",
+             errstring);
+
+#define SEND_ERRMSG(code, msg)                          \
+  control_printf_endreply(conn, code, msg "%s%s",       \
+                          errstring ? ": " : "",        \
+                          errstring ? errstring : "")
+  switch (retval) {
+  case SETOPT_ERR_PARSE:
+    SEND_ERRMSG(552, "Invalid config file");
+    break;
+  case SETOPT_ERR_TRANSITION:
+    SEND_ERRMSG(553, "Transition not allowed");
+    break;
+  case SETOPT_ERR_SETTING:
+    SEND_ERRMSG(553, "Unable to set option");
+    break;
+  case SETOPT_ERR_MISC:
+  default:
+    SEND_ERRMSG(550, "Unable to load config");
+    break;
+  case SETOPT_OK:
+    send_control_done(conn);
+    break;
+  }
+#undef SEND_ERRMSG
+  tor_free(errstring);
+  return 0;
+}
+
+static const control_cmd_syntax_t setevents_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a SETEVENTS message: update conn->event_mask,
+ * and reply with DONE or ERROR. */
+static int
+handle_control_setevents(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  int event_code;
+  event_mask_t event_mask = 0;
+  const smartlist_t *events = args->args;
+
+  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
+    {
+      if (!strcasecmp(ev, "EXTENDED") ||
+          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
+        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
+                 "supported.", ev);
+        continue;
+      } else {
+        int i;
+        event_code = -1;
+
+        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+          if (!strcasecmp(ev, control_event_table[i].event_name)) {
+            event_code = control_event_table[i].event_code;
+            break;
+          }
+        }
+
+        if (event_code == -1) {
+          control_printf_endreply(conn, 552, "Unrecognized event \"%s\"", ev);
+          return 0;
+        }
+      }
+      event_mask |= (((event_mask_t)1) << event_code);
+    }
+  SMARTLIST_FOREACH_END(ev);
+
+  conn->event_mask = event_mask;
+
+  control_update_global_event_mask();
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t saveconf_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a SAVECONF command. Try to flush the current options to
+ * disk, and report success or failure. */
+static int
+handle_control_saveconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  bool force = config_lines_contain_flag(args->kwargs, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
+    control_write_endreply(conn, 551,
+                           "Unable to write configuration to disk.");
+  } else {
+    send_control_done(conn);
+  }
+  return 0;
+}
+
+static const control_cmd_syntax_t signal_syntax = {
+  .min_args = 1,
+  .max_args = 1,
+};
+
+/** Called when we get a SIGNAL command. React to the provided signal, and
+ * report success or failure. (If the signal results in a shutdown, success
+ * may not be reported.) */
+static int
+handle_control_signal(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  int sig = -1;
+  int i;
+
+  tor_assert(smartlist_len(args->args) == 1);
+  const char *s = smartlist_get(args->args, 0);
+
+  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if (!strcasecmp(s, signal_table[i].signal_name)) {
+      sig = signal_table[i].sig;
+      break;
+    }
+  }
+
+  if (sig < 0)
+    control_printf_endreply(conn, 552, "Unrecognized signal code \"%s\"", s);
+  if (sig < 0)
+    return 0;
+
+  send_control_done(conn);
+  /* Flush the "done" first if the signal might make us shut down. */
+  if (sig == SIGTERM || sig == SIGINT)
+    connection_flush(TO_CONN(conn));
+
+  activate_signal(sig);
+
+  return 0;
+}
+
+static const control_cmd_syntax_t takeownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
+ * as an owning connection, so that we will exit if the connection
+ * closes. */
+static int
+handle_control_takeownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 1;
+
+  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t dropownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a DROPOWNERSHIP command.  Mark this connection
+ * as a non-owning connection, so that we will not exit if the connection
+ * closes. */
+static int
+handle_control_dropownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 0;
+
+  log_info(LD_CONTROL, "Control connection %d has dropped ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+/** Given a text circuit <b>id</b>, return the corresponding circuit. */
+static origin_circuit_t *
+get_circ(const char *id)
+{
+  uint32_t n_id;
+  int ok;
+  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  return circuit_get_by_global_id(n_id);
+}
+
+/** Given a text stream <b>id</b>, return the corresponding AP connection. */
+static entry_connection_t *
+get_stream(const char *id)
+{
+  uint64_t n_id;
+  int ok;
+  connection_t *conn;
+  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  conn = connection_get_by_global_id(n_id);
+  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
+    return NULL;
+  return TO_ENTRY_CONN(conn);
+}
+
+/** Helper for setconf and resetconf. Acts like setconf, except
+ * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
+ * contents of body.
+ */
+static int
+control_setconf_helper(control_connection_t *conn,
+                       const control_cmd_args_t *args,
+                       int use_defaults)
+{
+  setopt_err_t opt_err;
+  char *errstring = NULL;
+  const unsigned flags =
+    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
+
+  // We need a copy here, since confparse.c wants to canonicalize cases.
+  config_line_t *lines = config_lines_dup(args->kwargs);
+
+  opt_err = options_trial_assign(lines, flags, &errstring);
+  {
+#define SEND_ERRMSG(code, msg)                                  \
+    control_printf_endreply(conn, code, msg ": %s", errstring);
+
+    switch (opt_err) {
+      case SETOPT_ERR_MISC:
+        SEND_ERRMSG(552, "Unrecognized option");
+        break;
+      case SETOPT_ERR_PARSE:
+        SEND_ERRMSG(513, "Unacceptable option value");
+        break;
+      case SETOPT_ERR_TRANSITION:
+        SEND_ERRMSG(553, "Transition not allowed");
+        break;
+      case SETOPT_ERR_SETTING:
+      default:
+        SEND_ERRMSG(553, "Unable to set option");
+        break;
+      case SETOPT_OK:
+        config_free_lines(lines);
+        send_control_done(conn);
+        return 0;
+    }
+#undef SEND_ERRMSG
+    log_warn(LD_CONTROL,
+             "Controller gave us config lines that didn't validate: %s",
+             errstring);
+    config_free_lines(lines);
+    tor_free(errstring);
+    return 0;
+  }
+}
+
+/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
+ * containing funny characters. */
+static int
+address_is_invalid_mapaddress_target(const char *addr)
+{
+  if (!strcmpstart(addr, "*."))
+    return address_is_invalid_destination(addr+2, 1);
+  else
+    return address_is_invalid_destination(addr, 1);
+}
+
+static const control_cmd_syntax_t mapaddress_syntax = {
+  // no positional arguments are expected
+  .max_args=0,
+  // an arbitrary number of K=V entries are supported.
+  .accept_keywords=true,
+};
+
+/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
+ * and report success or failure. */
+static int
+handle_control_mapaddress(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  smartlist_t *reply;
+  char *r;
+  size_t sz;
+
+  reply = smartlist_new();
+  const config_line_t *line;
+  for (line = args->kwargs; line; line = line->next) {
+    const char *from = line->key;
+    const char *to = line->value;
+    {
+      if (address_is_invalid_mapaddress_target(to)) {
+        smartlist_add_asprintf(reply,
+                     "512-syntax error: invalid address '%s'", to);
+        log_warn(LD_CONTROL,
+                 "Skipping invalid argument '%s' in MapAddress msg", to);
+      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
+                 !strcmp(from, "::")) {
+        const char type =
+          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
+          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
+        const char *address = addressmap_register_virtual_address(
+                                                     type, tor_strdup(to));
+        if (!address) {
+          smartlist_add_asprintf(reply,
+                   "451-resource exhausted: skipping '%s=%s'", from,to);
+          log_warn(LD_CONTROL,
+                   "Unable to allocate address for '%s' in MapAddress msg",
+                   safe_str_client(to));
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
+        }
+      } else {
+        const char *msg;
+        if (addressmap_register_auto(from, to, 1,
+                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
+          smartlist_add_asprintf(reply,
+                                 "512-syntax error: invalid address mapping "
+                                 " '%s=%s': %s", from, to, msg);
+          log_warn(LD_CONTROL,
+                   "Skipping invalid argument '%s=%s' in MapAddress msg: %s",
+                   from, to, msg);
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", from, to);
+        }
+      }
+    }
+  }
+
+  if (smartlist_len(reply)) {
+    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
+    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
+    connection_buf_add(r, sz, TO_CONN(conn));
+    tor_free(r);
+  } else {
+    control_write_endreply(conn, 512, "syntax error: "
+                           "not enough arguments to mapaddress.");
+  }
+
+  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
+  smartlist_free(reply);
+  return 0;
+}
+
+/** Given a string, convert it to a circuit purpose. */
+static uint8_t
+circuit_purpose_from_string(const char *string)
+{
+  if (!strcasecmpstart(string, "purpose="))
+    string += strlen("purpose=");
+
+  if (!strcasecmp(string, "general"))
+    return CIRCUIT_PURPOSE_C_GENERAL;
+  else if (!strcasecmp(string, "controller"))
+    return CIRCUIT_PURPOSE_CONTROLLER;
+  else
+    return CIRCUIT_PURPOSE_UNKNOWN;
+}
+
+static const control_cmd_syntax_t extendcircuit_syntax = {
+  .min_args=1,
+  .max_args=1, // see note in function
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS
+};
+
+/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
+ * circuit, and report success or failure. */
+static int
+handle_control_extendcircuit(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  smartlist_t *router_nicknames=smartlist_new(), *nodes=NULL;
+  origin_circuit_t *circ = NULL;
+  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  const config_line_t *kwargs = args->kwargs;
+  const char *circ_id = smartlist_get(args->args, 0);
+  const char *path_str = NULL;
+  char *path_str_alloc = NULL;
+
+  /* The syntax for this command is unfortunate. The second argument is
+     optional, and is a comma-separated list long-format fingerprints, which
+     can (historically!) contain an equals sign.
+
+     Here we check the second argument to see if it's a path, and if so we
+     remove it from the kwargs list and put it in path_str.
+  */
+  if (kwargs) {
+    const config_line_t *arg1 = kwargs;
+    if (!strcmp(arg1->value, "")) {
+      path_str = arg1->key;
+      kwargs = kwargs->next;
+    } else if (arg1->key[0] == '$') {
+      tor_asprintf(&path_str_alloc, "%s=%s", arg1->key, arg1->value);
+      path_str = path_str_alloc;
+      kwargs = kwargs->next;
+    }
+  }
+
+  const config_line_t *purpose_line = config_line_find_case(kwargs, "PURPOSE");
+  bool zero_circ = !strcmp("0", circ_id);
+
+  if (purpose_line) {
+    intended_purpose = circuit_purpose_from_string(purpose_line->value);
+    if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purpose_line->value);
+      goto done;
+    }
+  }
+
+  if (zero_circ) {
+    if (!path_str) {
+      // "EXTENDCIRCUIT 0" with no path.
+      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
+      if (!circ) {
+        control_write_endreply(conn, 551, "Couldn't start circuit");
+      } else {
+        control_printf_endreply(conn, 250, "EXTENDED %lu",
+                                (unsigned long)circ->global_identifier);
+      }
+      goto done;
+    }
+  }
+
+  if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  if (!path_str) {
+    control_write_endreply(conn, 512, "syntax error: path required.");
+    goto done;
+  }
+
+  smartlist_split_string(router_nicknames, path_str, ",", 0, 0);
+
+  nodes = smartlist_new();
+  bool first_node = zero_circ;
+  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
+    const node_t *node = node_get_by_nickname(n, 0);
+    if (!node) {
+      control_printf_endreply(conn, 552, "No such router \"%s\"", n);
+      goto done;
+    }
+    if (!node_has_preferred_descriptor(node, first_node)) {
+      control_printf_endreply(conn, 552, "No descriptor for \"%s\"", n);
+      goto done;
+    }
+    smartlist_add(nodes, (void*)node);
+    first_node = false;
+  } SMARTLIST_FOREACH_END(n);
+
+  if (!smartlist_len(nodes)) {
+    control_write_endreply(conn, 512, "No router names provided");
+    goto done;
+  }
+
+  if (zero_circ) {
+    /* start a new circuit */
+    circ = origin_circuit_init(intended_purpose, 0);
+  }
+
+  /* now circ refers to something that is ready to be extended */
+  first_node = zero_circ;
+  SMARTLIST_FOREACH(nodes, const node_t *, node,
+  {
+    extend_info_t *info = extend_info_from_node(node, first_node);
+    if (!info) {
+      tor_assert_nonfatal(first_node);
+      log_warn(LD_CONTROL,
+               "controller tried to connect to a node that lacks a suitable "
+               "descriptor, or which doesn't have any "
+               "addresses that are allowed by the firewall configuration; "
+               "circuit marked for closing.");
+      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+    circuit_append_new_exit(circ, info);
+    if (circ->build_state->desired_path_len > 1) {
+      circ->build_state->onehop_tunnel = 0;
+    }
+    extend_info_free(info);
+    first_node = 0;
+  });
+
+  /* now that we've populated the cpath, start extending */
+  if (zero_circ) {
+    int err_reason = 0;
+    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
+      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+  } else {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
+      int err_reason = 0;
+      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
+      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
+        log_info(LD_CONTROL,
+                 "send_next_onion_skin failed; circuit marked for closing.");
+        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+        control_write_endreply(conn, 551, "Couldn't send onion skin");
+        goto done;
+      }
+    }
+  }
+
+  control_printf_endreply(conn, 250, "EXTENDED %lu",
+                          (unsigned long)circ->global_identifier);
+  if (zero_circ) /* send a 'launched' event, for completeness */
+    circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
+ done:
+  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
+  smartlist_free(router_nicknames);
+  smartlist_free(nodes);
+  tor_free(path_str_alloc);
+  return 0;
+}
+
+static const control_cmd_syntax_t setcircuitpurpose_syntax = {
+  .max_args=1,
+  .accept_keywords=true,
+};
+
+/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
+ * circuit and it's a valid purpose, change it. */
+static int
+handle_control_setcircuitpurpose(control_connection_t *conn,
+                                 const control_cmd_args_t *args)
+{
+  origin_circuit_t *circ = NULL;
+  uint8_t new_purpose;
+  const char *circ_id = smartlist_get(args->args,0);
+
+  if (!(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  {
+    const config_line_t *purp = config_line_find_case(args->kwargs, "PURPOSE");
+    if (!purp) {
+      control_write_endreply(conn, 552, "No purpose given");
+      goto done;
+    }
+    new_purpose = circuit_purpose_from_string(purp->value);
+    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purp->value);
+      goto done;
+    }
+  }
+
+  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
+  send_control_done(conn);
+
+ done:
+  return 0;
+}
+
+static const char *attachstream_keywords[] = {
+  "HOP", NULL
+};
+static const control_cmd_syntax_t attachstream_syntax = {
+  .min_args=2, .max_args=2,
+  .accept_keywords=true,
+  .allowed_keywords=attachstream_keywords
+};
+
+/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
+ * stream, and report success or failure. */
+static int
+handle_control_attachstream(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  entry_connection_t *ap_conn = NULL;
+  origin_circuit_t *circ = NULL;
+  crypt_path_t *cpath=NULL;
+  int hop=0, hop_line_ok=1;
+  const char *stream_id = smartlist_get(args->args, 0);
+  const char *circ_id = smartlist_get(args->args, 1);
+  int zero_circ = !strcmp(circ_id, "0");
+  const config_line_t *hoparg = config_line_find_case(args->kwargs, "HOP");
+
+  if (!(ap_conn = get_stream(stream_id))) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"", stream_id);
+    return 0;
+  } else if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  } else if (circ) {
+    if (hoparg) {
+      hop = (int) tor_parse_ulong(hoparg->value, 10, 0, INT_MAX,
+                                  &hop_line_ok, NULL);
+      if (!hop_line_ok) { /* broken hop line */
+        control_printf_endreply(conn, 552, "Bad value hop=%s",
+                                hoparg->value);
+        return 0;
+      }
+    }
+  }
+
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
+    control_write_endreply(conn, 555,
+                           "Connection is not managed by controller.");
+    return 0;
+  }
+
+  /* Do we need to detach it first? */
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
+    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
+    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
+    /* Un-mark it as ending, since we're going to reuse it. */
+    edge_conn->edge_has_sent_end = 0;
+    edge_conn->end_reason = 0;
+    if (tmpcirc)
+      circuit_detach_stream(tmpcirc, edge_conn);
+    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
+    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
+  }
+
+  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to non-open origin circuit");
+    return 0;
+  }
+  /* Is this a single hop circuit? */
+  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to this one-hop circuit.");
+    return 0;
+  }
+
+  if (circ && hop>0) {
+    /* find this hop in the circuit, and set cpath */
+    cpath = circuit_get_cpath_hop(circ, hop);
+    if (!cpath) {
+      control_printf_endreply(conn, 551, "Circuit doesn't have %d hops.", hop);
+      return 0;
+    }
+  }
+  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
+    control_write_endreply(conn, 551, "Unable to attach stream");
+    return 0;
+  }
+  send_control_done(conn);
+  return 0;
+}
+
+static const char *postdescriptor_keywords[] = {
+  "cache", "purpose", NULL,
+};
+
+static const control_cmd_syntax_t postdescriptor_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .allowed_keywords = postdescriptor_keywords,
+  .want_cmddata = true,
+};
+
+/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
+ * descriptor, and report success or failure. */
+static int
+handle_control_postdescriptor(control_connection_t *conn,
+                              const control_cmd_args_t *args)
+{
+  const char *msg=NULL;
+  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
+  int cache = 0; /* eventually, we may switch this to 1 */
+  const config_line_t *line;
+
+  line = config_line_find_case(args->kwargs, "purpose");
+  if (line) {
+    purpose = router_purpose_from_string(line->value);
+    if (purpose == ROUTER_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+  line = config_line_find_case(args->kwargs, "cache");
+  if (line) {
+    if (!strcasecmp(line->value, "no"))
+      cache = 0;
+    else if (!strcasecmp(line->value, "yes"))
+      cache = 1;
+    else {
+      control_printf_endreply(conn, 552, "Unknown cache request \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+
+  switch (router_load_single_router(args->cmddata, purpose, cache, &msg)) {
+  case -1:
+    if (!msg) msg = "Could not parse descriptor";
+    control_write_endreply(conn, 554, msg);
+    break;
+  case 0:
+    if (!msg) msg = "Descriptor not added";
+    control_write_endreply(conn, 251, msg);
+    break;
+  case 1:
+    send_control_done(conn);
+    break;
+  }
+
+ done:
+  return 0;
+}
+
+static const control_cmd_syntax_t redirectstream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, // XXX should be 3.
+};
+
+/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
+ * address of the named AP stream, and report success or failure. */
+static int
+handle_control_redirectstream(control_connection_t *conn,
+                              const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn = NULL;
+  char *new_addr = NULL;
+  uint16_t new_port = 0;
+  const smartlist_t *args = cmd_args->args;
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
+           || !ap_conn->socks_request) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  } else {
+    int ok = 1;
+    if (smartlist_len(args) > 2) { /* they included a port too */
+      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
+                                            10, 1, 65535, &ok, NULL);
+    }
+    if (!ok) {
+      control_printf_endreply(conn, 512, "Cannot parse port \"%s\"",
+                              (char*)smartlist_get(args, 2));
+    } else {
+      new_addr = tor_strdup(smartlist_get(args, 1));
+    }
+  }
+
+  if (!new_addr)
+    return 0;
+
+  strlcpy(ap_conn->socks_request->address, new_addr,
+          sizeof(ap_conn->socks_request->address));
+  if (new_port)
+    ap_conn->socks_request->port = new_port;
+  tor_free(new_addr);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closestream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, /* XXXX This is the original behavior, but
+                         * maybe we should change the spec. */
+};
+
+/** Called when we get a CLOSESTREAM command; try to close the named stream
+ * and report success or failure. */
+static int
+handle_control_closestream(control_connection_t *conn,
+                           const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn=NULL;
+  uint8_t reason=0;
+  int ok;
+  const smartlist_t *args = cmd_args->args;
+
+  tor_assert(smartlist_len(args) >= 2);
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0))))
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  else {
+    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
+                                       &ok, NULL);
+    if (!ok) {
+      control_printf_endreply(conn, 552, "Unrecognized reason \"%s\"",
+                              (char*)smartlist_get(args, 1));
+      ap_conn = NULL;
+    }
+  }
+  if (!ap_conn)
+    return 0;
+
+  connection_mark_unattached_ap(ap_conn, reason);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closecircuit_syntax = {
+  .min_args=1, .max_args=1,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+  // XXXX we might want to exclude unrecognized flags, but for now we
+  // XXXX just ignore them for backward compatibility.
+};
+
+/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
+ * and report success or failure. */
+static int
+handle_control_closecircuit(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  const char *circ_id = smartlist_get(args->args, 0);
+  origin_circuit_t *circ = NULL;
+
+  if (!(circ=get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  }
+
+  bool safe =  config_lines_contain_flag(args->kwargs, "IfUnused");
+
+  if (!safe || !circ->p_streams) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
+  }
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t resolve_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a RESOLVE command: start trying to resolve
+ * the listed addresses. */
+static int
+handle_control_resolve(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  smartlist_t *failed;
+  int is_reverse = 0;
+
+  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
+    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
+             "isn't listening for ADDRMAP events.  It probably won't see "
+             "the answer.");
+  }
+
+  {
+    const config_line_t *modearg = config_line_find_case(args->kwargs, "mode");
+    if (modearg && !strcasecmp(modearg->value, "reverse"))
+      is_reverse = 1;
+  }
+  failed = smartlist_new();
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strlen(line->value)) {
+      const char *addr = line->key;
+      if (dnsserv_launch_request(addr, is_reverse, conn)<0)
+        smartlist_add(failed, (char*)addr);
+    } else {
+      // XXXX arguably we should reject unrecognized keyword arguments,
+      // XXXX but the old implementation didn't do that.
+    }
+  }
+
+  send_control_done(conn);
+  SMARTLIST_FOREACH(failed, const char *, arg, {
+      control_event_address_mapped(arg, arg, time(NULL),
+                                   "internal", 0);
+  });
+
+  smartlist_free(failed);
+  return 0;
+}
+
+static const control_cmd_syntax_t protocolinfo_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a PROTOCOLINFO command: send back a reply. */
+static int
+handle_control_protocolinfo(control_connection_t *conn,
+                            const control_cmd_args_t *cmd_args)
+{
+  const char *bad_arg = NULL;
+  const smartlist_t *args = cmd_args->args;
+
+  conn->have_sent_protocolinfo = 1;
+
+  SMARTLIST_FOREACH(args, const char *, arg, {
+      int ok;
+      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
+      if (!ok) {
+        bad_arg = arg;
+        break;
+      }
+    });
+  if (bad_arg) {
+    control_printf_endreply(conn, 513, "No such version %s",
+                            escaped(bad_arg));
+    /* Don't tolerate bad arguments when not authenticated. */
+    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
+      connection_mark_for_close(TO_CONN(conn));
+    goto done;
+  } else {
+    const or_options_t *options = get_options();
+    int cookies = options->CookieAuthentication;
+    char *cfile = get_controller_cookie_file_name();
+    char *abs_cfile;
+    char *esc_cfile;
+    char *methods;
+    abs_cfile = make_path_absolute(cfile);
+    esc_cfile = esc_for_log(abs_cfile);
+    {
+      int passwd = (options->HashedControlPassword != NULL ||
+                    options->HashedControlSessionPassword != NULL);
+      smartlist_t *mlist = smartlist_new();
+      if (cookies) {
+        smartlist_add(mlist, (char*)"COOKIE");
+        smartlist_add(mlist, (char*)"SAFECOOKIE");
+      }
+      if (passwd)
+        smartlist_add(mlist, (char*)"HASHEDPASSWORD");
+      if (!cookies && !passwd)
+        smartlist_add(mlist, (char*)"NULL");
+      methods = smartlist_join_strings(mlist, ",", 0, NULL);
+      smartlist_free(mlist);
+    }
+
+    control_write_midreply(conn, 250, "PROTOCOLINFO 1");
+    control_printf_midreply(conn, 250, "AUTH METHODS=%s%s%s", methods,
+                            cookies?" COOKIEFILE=":"",
+                            cookies?esc_cfile:"");
+    control_printf_midreply(conn, 250, "VERSION Tor=%s", escaped(VERSION));
+    send_control_done(conn);
+
+    tor_free(methods);
+    tor_free(cfile);
+    tor_free(abs_cfile);
+    tor_free(esc_cfile);
+  }
+ done:
+  return 0;
+}
+
+static const control_cmd_syntax_t usefeature_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a USEFEATURE command: parse the feature list, and
+ * set up the control_connection's options properly. */
+static int
+handle_control_usefeature(control_connection_t *conn,
+                          const control_cmd_args_t *cmd_args)
+{
+  const smartlist_t *args = cmd_args->args;
+  int bad = 0;
+  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
+      if (!strcasecmp(arg, "VERBOSE_NAMES"))
+        ;
+      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
+        ;
+      else {
+        control_printf_endreply(conn, 552, "Unrecognized feature \"%s\"",
+                                arg);
+        bad = 1;
+        break;
+      }
+  } SMARTLIST_FOREACH_END(arg);
+
+  if (!bad) {
+    send_control_done(conn);
+  }
+
+  return 0;
+}
+
+static const control_cmd_syntax_t dropguards_syntax = {
+  .max_args = 0,
+};
+
+/** Implementation for the DROPGUARDS command. */
+static int
+handle_control_dropguards(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  (void) args; /* We don't take arguments. */
+
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
+  remove_all_entry_guards();
+  send_control_done(conn);
+
+  return 0;
+}
+
+static const char *hsfetch_keywords[] = {
+  "SERVER", NULL,
+};
+static const control_cmd_syntax_t hsfetch_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = hsfetch_keywords,
+};
+
+/** Implementation for the HSFETCH command. */
+static int
+handle_control_hsfetch(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+
+{
+  char digest[DIGEST_LEN], *desc_id = NULL;
+  smartlist_t *hsdirs = NULL;
+  static const char *v2_str = "v2-";
+  const size_t v2_str_len = strlen(v2_str);
+  rend_data_t *rend_query = NULL;
+  ed25519_public_key_t v3_pk;
+  uint32_t version;
+  const char *hsaddress = NULL;
+
+  /* Extract the first argument (either HSAddress or DescID). */
+  const char *arg1 = smartlist_get(args->args, 0);
+  /* Test if it's an HS address without the .onion part. */
+  if (rend_valid_v2_service_id(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_TWO;
+  } else if (strcmpstart(arg1, v2_str) == 0 &&
+             rend_valid_descriptor_id(arg1 + v2_str_len) &&
+             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
+                           REND_DESC_ID_V2_LEN_BASE32) ==
+                REND_DESC_ID_V2_LEN_BASE32) {
+    /* We have a well formed version 2 descriptor ID. Keep the decoded value
+     * of the id. */
+    desc_id = digest;
+    version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_THREE;
+    hs_parse_address(hsaddress, &v3_pk, NULL, NULL);
+  } else {
+    control_printf_endreply(conn, 513, "Invalid argument \"%s\"", arg1);
+    goto done;
+  }
+
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strcasecmp(line->key, "SERVER")) {
+      const char *server = line->value;
+
+      const node_t *node = node_get_by_hex_id(server, 0);
+      if (!node) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found", server);
+        goto done;
+      }
+      if (!hsdirs) {
+        /* Stores routerstatus_t cmddata for each specified server. */
+        hsdirs = smartlist_new();
+      }
+      /* Valid server, add it to our local list. */
+      smartlist_add(hsdirs, node->rs);
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  if (version == HS_VERSION_TWO) {
+    rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
+                                         REND_NO_AUTH);
+    if (rend_query == NULL) {
+      control_write_endreply(conn, 551, "Error creating the HS query");
+      goto done;
+    }
+  }
+
+  /* Using a descriptor ID, we force the user to provide at least one
+   * hsdir server using the SERVER= option. */
+  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
+    control_write_endreply(conn, 512, "SERVER option is required");
+    goto done;
+  }
+
+  /* We are about to trigger HSDir fetch so send the OK now because after
+   * that 650 event(s) are possible so better to have the 250 OK before them
+   * to avoid out of order replies. */
+  send_control_done(conn);
+
+  /* Trigger the fetch using the built rend query and possibly a list of HS
+   * directory to use. This function ignores the client cache thus this will
+   * always send a fetch command. */
+  if (version == HS_VERSION_TWO) {
+    rend_client_fetch_v2_desc(rend_query, hsdirs);
+  } else if (version == HS_VERSION_THREE) {
+    hs_control_hsfetch_command(&v3_pk, hsdirs);
+  }
+
+ done:
+  /* Contains data pointer that we don't own thus no cleanup. */
+  smartlist_free(hsdirs);
+  rend_data_free(rend_query);
+  return 0;
+}
+
+static const char *hspost_keywords[] = {
+  "SERVER", "HSADDRESS", NULL
+};
+static const control_cmd_syntax_t hspost_syntax = {
+  .min_args = 0, .max_args = 0,
+  .accept_keywords = true,
+  .want_cmddata = true,
+  .allowed_keywords = hspost_keywords
+};
+
+/** Implementation for the HSPOST command. */
+static int
+handle_control_hspost(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  smartlist_t *hs_dirs = NULL;
+  const char *encoded_desc = args->cmddata;
+  size_t encoded_desc_len = args->cmddata_len;
+  const char *onion_address = NULL;
+  const config_line_t *line;
+
+  for (line = args->kwargs; line; line = line->next) {
+    if (!strcasecmpstart(line->key, "SERVER")) {
+      const char *server = line->value;
+      const node_t *node = node_get_by_hex_id(server, 0);
+
+      if (!node || !node->rs) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found",
+                                server);
+        goto done;
+      }
+      /* Valid server, add it to our local list. */
+      if (!hs_dirs)
+        hs_dirs = smartlist_new();
+      smartlist_add(hs_dirs, node->rs);
+    } else if (!strcasecmpstart(line->key, "HSADDRESS")) {
+      const char *address = line->value;
+      if (!hs_address_is_valid(address)) {
+        control_write_endreply(conn, 512, "Malformed onion address");
+        goto done;
+      }
+      onion_address = address;
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  /* Handle the v3 case. */
+  if (onion_address) {
+    if (hs_control_hspost_command(encoded_desc, onion_address, hs_dirs) < 0) {
+      control_write_endreply(conn, 554, "Invalid descriptor");
+    } else {
+      send_control_done(conn);
+    }
+    goto done;
+  }
+
+  /* From this point on, it is only v2. */
+
+  /*  parse it. */
+  rend_encoded_v2_service_descriptor_t *desc =
+      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
+  desc->desc_str = tor_memdup_nulterm(encoded_desc, encoded_desc_len);
+
+  rend_service_descriptor_t *parsed = NULL;
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
+                                        &intro_size, &encoded_size,
+                                        &next_desc, desc->desc_str, 1)) {
+    /* Post the descriptor. */
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    if (!rend_get_service_id(parsed->pk, serviceid)) {
+      smartlist_t *descs = smartlist_new();
+      smartlist_add(descs, desc);
+
+      /* We are about to trigger HS descriptor upload so send the OK now
+       * because after that 650 event(s) are possible so better to have the
+       * 250 OK before them to avoid out of order replies. */
+      send_control_done(conn);
+
+      /* Trigger the descriptor upload */
+      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
+      smartlist_free(descs);
+    }
+
+    rend_service_descriptor_free(parsed);
+  } else {
+    control_write_endreply(conn, 554, "Invalid descriptor");
+  }
+
+  tor_free(intro_content);
+  rend_encoded_v2_service_descriptor_free(desc);
+ done:
+  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
+  return 0;
+}
+
+/* Helper function for ADD_ONION that adds an ephemeral service depending on
+ * the given hs_version.
+ *
+ * The secret key in pk depends on the hs_version. The ownership of the key
+ * used in pk is given to the HS subsystem so the caller must stop accessing
+ * it after.
+ *
+ * The port_cfgs is a list of service port. Ownership transferred to service.
+ * The max_streams refers to the MaxStreams= key.
+ * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
+ * The auth_type is the authentication type of the clients in auth_clients.
+ * The ownership of that list is transferred to the service.
+ *
+ * On success (RSAE_OKAY), the address_out points to a newly allocated string
+ * containing the onion address without the .onion part. On error, address_out
+ * is untouched. */
+static hs_service_add_ephemeral_status_t
+add_onion_helper_add_service(int hs_version,
+                             add_onion_secret_key_t *pk,
+                             smartlist_t *port_cfgs, int max_streams,
+                             int max_streams_close_circuit, int auth_type,
+                             smartlist_t *auth_clients, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+
+  tor_assert(pk);
+  tor_assert(port_cfgs);
+  tor_assert(address_out);
+
+  switch (hs_version) {
+  case HS_VERSION_TWO:
+    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
+                                     max_streams_close_circuit, auth_type,
+                                     auth_clients, address_out);
+    break;
+  case HS_VERSION_THREE:
+    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
+                                   max_streams_close_circuit, address_out);
+    break;
+  default:
+    tor_assert_unreached();
+  }
+
+  return ret;
+}
+
+/** The list of onion services that have been added via ADD_ONION that do not
+ * belong to any particular control connection.
+ */
+static smartlist_t *detached_onion_services = NULL;
+
+/**
+ * Return a list of detached onion services, or NULL if none exist.
+ **/
+smartlist_t *
+get_detached_onion_services(void)
+{
+  return detached_onion_services;
+}
+
+static const char *add_onion_keywords[] = {
+   "Port", "Flags", "MaxStreams", "ClientAuth", NULL
+};
+static const control_cmd_syntax_t add_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = add_onion_keywords
+};
+
+/** Called when we get a ADD_ONION command; parse the body, and set up
+ * the new ephemeral Onion Service. */
+static int
+handle_control_add_onion(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  /* Parse all of the arguments that do not involve handling cryptographic
+   * material first, since there's no reason to touch that at all if any of
+   * the other arguments are malformed.
+   */
+  smartlist_t *port_cfgs = smartlist_new();
+  smartlist_t *auth_clients = NULL;
+  smartlist_t *auth_created_clients = NULL;
+  int discard_pk = 0;
+  int detach = 0;
+  int max_streams = 0;
+  int max_streams_close_circuit = 0;
+  rend_auth_type_t auth_type = REND_NO_AUTH;
+  int non_anonymous = 0;
+  const config_line_t *arg;
+
+  for (arg = args->kwargs; arg; arg = arg->next) {
+    if (!strcasecmp(arg->key, "Port")) {
+      /* "Port=VIRTPORT[,TARGET]". */
+      rend_service_port_config_t *cfg =
+          rend_service_parse_port_config(arg->value, ",", NULL);
+      if (!cfg) {
+        control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+        goto out;
+      }
+      smartlist_add(port_cfgs, cfg);
+    } else if (!strcasecmp(arg->key, "MaxStreams")) {
+      /* "MaxStreams=[0..65535]". */
+      int ok = 0;
+      max_streams = (int)tor_parse_long(arg->value, 10, 0, 65535, &ok, NULL);
+      if (!ok) {
+        control_write_endreply(conn, 512, "Invalid MaxStreams");
+        goto out;
+      }
+    } else if (!strcasecmp(arg->key, "Flags")) {
+      /* "Flags=Flag[,Flag]", where Flag can be:
+       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
+       *                   the response.
+       *   * 'Detach' - Do not tie this onion service to any particular control
+       *                connection.
+       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
+       *                                exceeded.
+       *   * 'BasicAuth' - Client authorization using the 'basic' method.
+       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
+       *                      flag is present, tor must be in non-anonymous
+       *                      hidden service mode. If this flag is absent,
+       *                      tor must be in anonymous hidden service mode.
+       */
+      static const char *discard_flag = "DiscardPK";
+      static const char *detach_flag = "Detach";
+      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
+      static const char *basicauth_flag = "BasicAuth";
+      static const char *non_anonymous_flag = "NonAnonymous";
+
+      smartlist_t *flags = smartlist_new();
+      int bad = 0;
+
+      smartlist_split_string(flags, arg->value, ",", SPLIT_IGNORE_BLANK, 0);
+      if (smartlist_len(flags) < 1) {
+        control_write_endreply(conn, 512, "Invalid 'Flags' argument");
+        bad = 1;
+      }
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
+      {
+        if (!strcasecmp(flag, discard_flag)) {
+          discard_pk = 1;
+        } else if (!strcasecmp(flag, detach_flag)) {
+          detach = 1;
+        } else if (!strcasecmp(flag, max_s_close_flag)) {
+          max_streams_close_circuit = 1;
+        } else if (!strcasecmp(flag, basicauth_flag)) {
+          auth_type = REND_BASIC_AUTH;
+        } else if (!strcasecmp(flag, non_anonymous_flag)) {
+          non_anonymous = 1;
+        } else {
+          control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s",
+                                  escaped(flag));
+          bad = 1;
+          break;
+        }
+      } SMARTLIST_FOREACH_END(flag);
+      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
+      smartlist_free(flags);
+      if (bad)
+        goto out;
+
+    } else if (!strcasecmp(arg->key, "ClientAuth")) {
+      int created = 0;
+      rend_authorized_client_t *client =
+        add_onion_helper_clientauth(arg->value, &created, conn);
+      if (!client) {
+        goto out;
+      }
+
+      if (auth_clients != NULL) {
+        int bad = 0;
+        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
+          if (strcmp(ac->client_name, client->client_name) == 0) {
+            bad = 1;
+            break;
+          }
+        } SMARTLIST_FOREACH_END(ac);
+        if (bad) {
+          control_write_endreply(conn, 512, "Duplicate name in ClientAuth");
+          rend_authorized_client_free(client);
+          goto out;
+        }
+      } else {
+        auth_clients = smartlist_new();
+        auth_created_clients = smartlist_new();
+      }
+      smartlist_add(auth_clients, client);
+      if (created) {
+        smartlist_add(auth_created_clients, client);
+      }
+    } else {
+      tor_assert_nonfatal_unreached();
+      goto out;
+    }
+  }
+  if (smartlist_len(port_cfgs) == 0) {
+    control_write_endreply(conn, 512, "Missing 'Port' argument");
+    goto out;
+  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
+    control_write_endreply(conn, 512, "No auth type specified");
+    goto out;
+  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
+    control_write_endreply(conn, 512, "No auth clients specified");
+    goto out;
+  } else if ((auth_type == REND_BASIC_AUTH &&
+              smartlist_len(auth_clients) > 512) ||
+             (auth_type == REND_STEALTH_AUTH &&
+              smartlist_len(auth_clients) > 16)) {
+    control_write_endreply(conn, 512, "Too many auth clients");
+    goto out;
+  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
+                                                              get_options())) {
+    /* If we failed, and the non-anonymous flag is set, Tor must be in
+     * anonymous hidden service mode.
+     * The error message changes based on the current Tor config:
+     * 512 Tor is in anonymous hidden service mode
+     * 512 Tor is in non-anonymous hidden service mode
+     * (I've deliberately written them out in full here to aid searchability.)
+     */
+    control_printf_endreply(conn, 512,
+                            "Tor is in %sanonymous hidden service " "mode",
+                            non_anonymous ? "" : "non-");
+    goto out;
+  }
+
+  /* Parse the "keytype:keyblob" argument. */
+  int hs_version = 0;
+  add_onion_secret_key_t pk = { NULL };
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+
+  const char *onionkey = smartlist_get(args->args, 0);
+  if (add_onion_helper_keyarg(onionkey, discard_pk,
+                              &key_new_alg, &key_new_blob, &pk, &hs_version,
+                              conn) < 0) {
+    goto out;
+  }
+
+  /* Hidden service version 3 don't have client authentication support so if
+   * ClientAuth was given, send back an error. */
+  if (hs_version == HS_VERSION_THREE && auth_clients) {
+    control_write_endreply(conn, 513, "ClientAuth not supported");
+    goto out;
+  }
+
+  /* Create the HS, using private key pk, client authentication auth_type,
+   * the list of auth_clients, and port config port_cfg.
+   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
+   * regardless of success/failure.
+   */
+  char *service_id = NULL;
+  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
+                                         max_streams,
+                                         max_streams_close_circuit, auth_type,
+                                         auth_clients, &service_id);
+  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
+  auth_clients = NULL; /* so is auth_clients */
+  switch (ret) {
+  case RSAE_OKAY:
+  {
+    if (detach) {
+      if (!detached_onion_services)
+        detached_onion_services = smartlist_new();
+      smartlist_add(detached_onion_services, service_id);
+    } else {
+      if (!conn->ephemeral_onion_services)
+        conn->ephemeral_onion_services = smartlist_new();
+      smartlist_add(conn->ephemeral_onion_services, service_id);
+    }
+
+    tor_assert(service_id);
+    control_printf_midreply(conn, 250, "ServiceID=%s", service_id);
+    if (key_new_alg) {
+      tor_assert(key_new_blob);
+      control_printf_midreply(conn, 250, "PrivateKey=%s:%s",
+                              key_new_alg, key_new_blob);
+    }
+    if (auth_created_clients) {
+      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
+        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
+                                                auth_type);
+        tor_assert(encoded);
+        control_printf_midreply(conn, 250, "ClientAuth=%s:%s",
+                                ac->client_name, encoded);
+        memwipe(encoded, 0, strlen(encoded));
+        tor_free(encoded);
+      });
+    }
+
+    send_control_done(conn);
+    break;
+  }
+  case RSAE_BADPRIVKEY:
+    control_write_endreply(conn, 551, "Failed to generate onion address");
+    break;
+  case RSAE_ADDREXISTS:
+    control_write_endreply(conn, 550, "Onion address collision");
+    break;
+  case RSAE_BADVIRTPORT:
+    control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+    break;
+  case RSAE_BADAUTH:
+    control_write_endreply(conn, 512, "Invalid client authorization");
+    break;
+  case RSAE_INTERNAL: FALLTHROUGH;
+  default:
+    control_write_endreply(conn, 551, "Failed to add Onion Service");
+  }
+  if (key_new_blob) {
+    memwipe(key_new_blob, 0, strlen(key_new_blob));
+    tor_free(key_new_blob);
+  }
+
+ out:
+  if (port_cfgs) {
+    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
+                      rend_service_port_config_free(p));
+    smartlist_free(port_cfgs);
+  }
+
+  if (auth_clients) {
+    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
+                      rend_authorized_client_free(ac));
+    smartlist_free(auth_clients);
+  }
+  if (auth_created_clients) {
+    // Do not free entries; they are the same as auth_clients
+    smartlist_free(auth_created_clients);
+  }
+  return 0;
+}
+
+/** Helper function to handle parsing the KeyType:KeyBlob argument to the
+ * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
+ * and the private key not discarded, the algorithm and serialized private key,
+ * or NULL and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned key_new_blob.
+ *
+ * Note: The error messages returned are deliberately vague to avoid echoing
+ * key material.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC int
+add_onion_helper_keyarg(const char *arg, int discard_pk,
+                        const char **key_new_alg_out, char **key_new_blob_out,
+                        add_onion_secret_key_t *decoded_key, int *hs_version,
+                        control_connection_t *conn)
+{
+  smartlist_t *key_args = smartlist_new();
+  crypto_pk_t *pk = NULL;
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+  int ret = -1;
+
+  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(key_args) != 2) {
+    control_write_endreply(conn, 512, "Invalid key type/blob");
+    goto err;
+  }
+
+  /* The format is "KeyType:KeyBlob". */
+  static const char *key_type_new = "NEW";
+  static const char *key_type_best = "BEST";
+  static const char *key_type_rsa1024 = "RSA1024";
+  static const char *key_type_ed25519_v3 = "ED25519-V3";
+
+  const char *key_type = smartlist_get(key_args, 0);
+  const char *key_blob = smartlist_get(key_args, 1);
+
+  if (!strcasecmp(key_type_rsa1024, key_type)) {
+    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
+    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
+    if (!pk) {
+      control_write_endreply(conn, 512, "Failed to decode RSA key");
+      goto err;
+    }
+    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
+      crypto_pk_free(pk);
+      control_write_endreply(conn, 512, "Invalid RSA key size");
+      goto err;
+    }
+    decoded_key->v2 = pk;
+    *hs_version = HS_VERSION_TWO;
+  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
+    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
+    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
+                      strlen(key_blob)) != sizeof(sk->seckey)) {
+      tor_free(sk);
+      control_write_endreply(conn, 512, "Failed to decode ED25519-V3 key");
+      goto err;
+    }
+    decoded_key->v3 = sk;
+    *hs_version = HS_VERSION_THREE;
+  } else if (!strcasecmp(key_type_new, key_type)) {
+    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
+    if (!strcasecmp(key_type_rsa1024, key_blob)) {
+      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
+      pk = crypto_pk_new();
+      if (crypto_pk_generate_key(pk)) {
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_rsa1024);
+        goto err;
+      }
+      if (!discard_pk) {
+        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
+          crypto_pk_free(pk);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_rsa1024);
+          goto err;
+        }
+        key_new_alg = key_type_rsa1024;
+      }
+      decoded_key->v2 = pk;
+      *hs_version = HS_VERSION_TWO;
+    } else if (!strcasecmp(key_type_ed25519_v3, key_blob) ||
+               !strcasecmp(key_type_best, key_blob)) {
+      /* "ED25519-V3", ed25519 key, also currently "BEST" by default. */
+      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+      if (ed25519_secret_key_generate(sk, 1) < 0) {
+        tor_free(sk);
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_ed25519_v3);
+        goto err;
+      }
+      if (!discard_pk) {
+        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
+        key_new_blob = tor_malloc_zero(len);
+        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
+                          sizeof(sk->seckey), 0) != (len - 1)) {
+          tor_free(sk);
+          tor_free(key_new_blob);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_ed25519_v3);
+          goto err;
+        }
+        key_new_alg = key_type_ed25519_v3;
+      }
+      decoded_key->v3 = sk;
+      *hs_version = HS_VERSION_THREE;
+    } else {
+      control_write_endreply(conn, 513, "Invalid key type");
+      goto err;
+    }
+  } else {
+    control_write_endreply(conn, 513, "Invalid key type");
+    goto err;
+  }
+
+  /* Succeeded in loading or generating a private key. */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(key_args, char *, cp, {
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+  });
+  smartlist_free(key_args);
+
+  *key_new_alg_out = key_new_alg;
+  *key_new_blob_out = key_new_blob;
+
+  return ret;
+}
+
+/** Helper function to handle parsing a ClientAuth argument to the
+ * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
+ * and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned auth_client.
+ *
+ * If 'created' is specified, it will be set to 1 when a new cookie has
+ * been generated.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC rend_authorized_client_t *
+add_onion_helper_clientauth(const char *arg, int *created,
+                            control_connection_t *conn)
+{
+  int ok = 0;
+
+  tor_assert(arg);
+  tor_assert(created);
+
+  smartlist_t *auth_args = smartlist_new();
+  rend_authorized_client_t *client =
+    tor_malloc_zero(sizeof(rend_authorized_client_t));
+  smartlist_split_string(auth_args, arg, ":", 0, 0);
+  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
+    control_write_endreply(conn, 512, "Invalid ClientAuth syntax");
+    goto err;
+  }
+  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
+  if (smartlist_len(auth_args) == 2) {
+    char *decode_err_msg = NULL;
+    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
+                                client->descriptor_cookie,
+                                NULL, &decode_err_msg) < 0) {
+      tor_assert(decode_err_msg);
+      control_write_endreply(conn, 512, decode_err_msg);
+      tor_free(decode_err_msg);
+      goto err;
+    }
+    *created = 0;
+  } else {
+    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
+    *created = 1;
+  }
+
+  if (!rend_valid_client_name(client->client_name)) {
+    control_write_endreply(conn, 512, "Invalid name in ClientAuth");
+    goto err;
+  }
+
+  ok = 1;
+ err:
+  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
+  smartlist_free(auth_args);
+  if (!ok) {
+    rend_authorized_client_free(client);
+    client = NULL;
+  }
+  return client;
+}
+
+static const control_cmd_syntax_t del_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+};
+
+/** Called when we get a DEL_ONION command; parse the body, and remove
+ * the existing ephemeral Onion Service. */
+static int
+handle_control_del_onion(control_connection_t *conn,
+                         const control_cmd_args_t *cmd_args)
+{
+  int hs_version = 0;
+  smartlist_t *args = cmd_args->args;
+  tor_assert(smartlist_len(args) == 1);
+
+  const char *service_id = smartlist_get(args, 0);
+  if (rend_valid_v2_service_id(service_id)) {
+    hs_version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(service_id)) {
+    hs_version = HS_VERSION_THREE;
+  } else {
+    control_write_endreply(conn, 512, "Malformed Onion Service id");
+    goto out;
+  }
+
+  /* Determine if the onion service belongs to this particular control
+   * connection, or if it is in the global list of detached services.  If it
+   * is in neither, either the service ID is invalid in some way, or it
+   * explicitly belongs to a different control connection, and an error
+   * should be returned.
+   */
+  smartlist_t *services[2] = {
+    conn->ephemeral_onion_services,
+    detached_onion_services
+  };
+  smartlist_t *onion_services = NULL;
+  int idx = -1;
+  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
+    idx = smartlist_string_pos(services[i], service_id);
+    if (idx != -1) {
+      onion_services = services[i];
+      break;
+    }
+  }
+  if (onion_services == NULL) {
+    control_write_endreply(conn, 552, "Unknown Onion Service id");
+  } else {
+    int ret = -1;
+    switch (hs_version) {
+    case HS_VERSION_TWO:
+      ret = rend_service_del_ephemeral(service_id);
+      break;
+    case HS_VERSION_THREE:
+      ret = hs_service_del_ephemeral(service_id);
+      break;
+    default:
+      /* The ret value will be -1 thus hitting the warning below. This should
+       * never happen because of the check at the start of the function. */
+      break;
+    }
+    if (ret < 0) {
+      /* This should *NEVER* fail, since the service is on either the
+       * per-control connection list, or the global one.
+       */
+      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
+               escaped(service_id));
+      tor_fragile_assert();
+    }
+
+    /* Remove/scrub the service_id from the appropriate list. */
+    char *cp = smartlist_get(onion_services, idx);
+    smartlist_del(onion_services, idx);
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+
+    send_control_done(conn);
+  }
+
+ out:
+  return 0;
+}
+
+static const control_cmd_syntax_t obsolete_syntax = {
+  .max_args = UINT_MAX
+};
+
+/**
+ * Called when we get an obsolete command: tell the controller that it is
+ * obsolete.
+ */
+static int
+handle_control_obsolete(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  (void)args;
+  char *command = tor_strdup(conn->current_cmd);
+  tor_strupper(command);
+  control_printf_endreply(conn, 511, "%s is obsolete.", command);
+  tor_free(command);
+  return 0;
+}
+
+/**
+ * Function pointer to a handler function for a controller command.
+ **/
+typedef int (*handler_fn_t) (control_connection_t *conn,
+                             const control_cmd_args_t *args);
+
+/**
+ * Definition for a controller command.
+ */
+typedef struct control_cmd_def_t {
+  /**
+   * The name of the command. If the command is multiline, the name must
+   * begin with "+".  This is not case-sensitive. */
+  const char *name;
+  /**
+   * A function to execute the command.
+   */
+  handler_fn_t handler;
+  /**
+   * Zero or more CMD_FL_* flags, or'd together.
+   */
+  unsigned flags;
+  /**
+   * For parsed command: a syntax description.
+   */
+  const control_cmd_syntax_t *syntax;
+} control_cmd_def_t;
+
+/**
+ * Indicates that the command's arguments are sensitive, and should be
+ * memwiped after use.
+ */
+#define CMD_FL_WIPE (1u<<0)
+
+/** Macro: declare a command with a one-line argument, a given set of flags,
+ * and a syntax definition.
+ **/
+#define ONE_LINE(name, flags)                                   \
+  {                                                             \
+    #name,                                                      \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax,                                           \
+   }
+
+/**
+ * Macro: declare a command with a multi-line argument and a given set of
+ * flags.
+ **/
+#define MULTLINE(name, flags)                                   \
+  { "+"#name,                                                   \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax                                            \
+  }
+
+/**
+ * Macro: declare an obsolete command. (Obsolete commands give a different
+ * error than non-existent ones.)
+ **/
+#define OBSOLETE(name)                          \
+  { #name,                                      \
+      handle_control_obsolete,                  \
+      0,                                        \
+      &obsolete_syntax,                         \
+  }
+
+/**
+ * An array defining all the recognized controller commands.
+ **/
+static const control_cmd_def_t CONTROL_COMMANDS[] =
+{
+  ONE_LINE(setconf, 0),
+  ONE_LINE(resetconf, 0),
+  ONE_LINE(getconf, 0),
+  MULTLINE(loadconf, 0),
+  ONE_LINE(setevents, 0),
+  ONE_LINE(authenticate, CMD_FL_WIPE),
+  ONE_LINE(saveconf, 0),
+  ONE_LINE(signal, 0),
+  ONE_LINE(takeownership, 0),
+  ONE_LINE(dropownership, 0),
+  ONE_LINE(mapaddress, 0),
+  ONE_LINE(getinfo, 0),
+  ONE_LINE(extendcircuit, 0),
+  ONE_LINE(setcircuitpurpose, 0),
+  OBSOLETE(setrouterpurpose),
+  ONE_LINE(attachstream, 0),
+  MULTLINE(postdescriptor, 0),
+  ONE_LINE(redirectstream, 0),
+  ONE_LINE(closestream, 0),
+  ONE_LINE(closecircuit, 0),
+  ONE_LINE(usefeature, 0),
+  ONE_LINE(resolve, 0),
+  ONE_LINE(protocolinfo, 0),
+  ONE_LINE(authchallenge, CMD_FL_WIPE),
+  ONE_LINE(dropguards, 0),
+  ONE_LINE(hsfetch, 0),
+  MULTLINE(hspost, 0),
+  ONE_LINE(add_onion, CMD_FL_WIPE),
+  ONE_LINE(del_onion, CMD_FL_WIPE),
+};
+
+/**
+ * The number of entries in CONTROL_COMMANDS.
+ **/
+static const size_t N_CONTROL_COMMANDS = ARRAY_LENGTH(CONTROL_COMMANDS);
+
+/**
+ * Run a single control command, as defined by a control_cmd_def_t,
+ * with a given set of arguments.
+ */
+static int
+handle_single_control_command(const control_cmd_def_t *def,
+                              control_connection_t *conn,
+                              uint32_t cmd_data_len,
+                              char *args)
+{
+  int rv = 0;
+
+  control_cmd_args_t *parsed_args;
+  char *err=NULL;
+  tor_assert(def->syntax);
+  parsed_args = control_cmd_parse_args(conn->current_cmd,
+                                       def->syntax,
+                                       cmd_data_len, args,
+                                       &err);
+  if (!parsed_args) {
+    control_printf_endreply(conn, 512, "Bad arguments to %s: %s",
+                            conn->current_cmd, err?err:"");
+    tor_free(err);
+  } else {
+    if (BUG(err))
+      tor_free(err);
+    if (def->handler(conn, parsed_args))
+      rv = 0;
+
+    if (def->flags & CMD_FL_WIPE)
+      control_cmd_args_wipe(parsed_args);
+
+    control_cmd_args_free(parsed_args);
+  }
+
+  if (def->flags & CMD_FL_WIPE)
+    memwipe(args, 0, cmd_data_len);
+
+  return rv;
+}
+
+/**
+ * Run a given controller command, as selected by the current_cmd field of
+ * <b>conn</b>.
+ */
+int
+handle_control_command(control_connection_t *conn,
+                       uint32_t cmd_data_len,
+                       char *args)
+{
+  tor_assert(conn);
+  tor_assert(args);
+  tor_assert(args[cmd_data_len] == '\0');
+
+  for (unsigned i = 0; i < N_CONTROL_COMMANDS; ++i) {
+    const control_cmd_def_t *def = &CONTROL_COMMANDS[i];
+    if (!strcasecmp(conn->current_cmd, def->name)) {
+      return handle_single_control_command(def, conn, cmd_data_len, args);
+    }
+  }
+
+  control_printf_endreply(conn, 510, "Unrecognized command \"%s\"",
+                          conn->current_cmd);
+
+  return 0;
+}
+
+void
+control_cmd_free_all(void)
+{
+  if (detached_onion_services) { /* Free the detached onion services */
+    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
+    smartlist_free(detached_onion_services);
+  }
+}
diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h
new file mode 100644
index 0000000000..4b6d54abe7
--- /dev/null
+++ b/src/feature/control/control_cmd.h
@@ -0,0 +1,113 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.h
+ * \brief Header file for control_cmd.c.
+ **/
+
+#ifndef TOR_CONTROL_CMD_H
+#define TOR_CONTROL_CMD_H
+
+#include "lib/malloc/malloc.h"
+
+int handle_control_command(control_connection_t *conn,
+                           uint32_t cmd_data_len,
+                           char *args);
+void control_cmd_free_all(void);
+
+typedef struct control_cmd_args_t control_cmd_args_t;
+void control_cmd_args_free_(control_cmd_args_t *args);
+void control_cmd_args_wipe(control_cmd_args_t *args);
+
+#define control_cmd_args_free(v) \
+  FREE_AND_NULL(control_cmd_args_t, control_cmd_args_free_, (v))
+
+/**
+ * Definition for the syntax of a controller command, as parsed by
+ * control_cmd_parse_args.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+typedef struct control_cmd_syntax_t {
+  /**
+   * Lowest number of positional arguments that this command accepts.
+   * 0 for "it's okay not to have positional arguments."
+   **/
+  unsigned int min_args;
+  /**
+   * Highest number of positional arguments that this command accepts.
+   * UINT_MAX for no limit.
+   **/
+  unsigned int max_args;
+  /**
+   * If true, we should parse options after the positional arguments
+   * as a set of unordered flags and key=value arguments.
+   *
+   * Requires that max_args is not UINT_MAX.
+   **/
+  bool accept_keywords;
+  /**
+   * If accept_keywords is true, then only the keywords listed in this
+   * (NULL-terminated) array are valid keywords for this command.
+   **/
+  const char **allowed_keywords;
+  /**
+   * If accept_keywords is true, this option is passed to kvline_parse() as
+   * its flags.
+   **/
+  unsigned kvline_flags;
+  /**
+   * True iff this command wants to be followed by a multiline object.
+   **/
+  bool want_cmddata;
+  /**
+   * True iff this command needs access to the raw body of the input.
+   *
+   * This should not be needed for pure commands; it is purely a legacy
+   * option.
+   **/
+  bool store_raw_body;
+} control_cmd_syntax_t;
+
+#ifdef CONTROL_CMD_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/* ADD_ONION secret key to create an ephemeral service. The command supports
+ * multiple versions so this union stores the key and passes it to the HS
+ * subsystem depending on the requested version. */
+typedef union add_onion_secret_key_t {
+  /* Hidden service v2 secret key. */
+  crypto_pk_t *v2;
+  /* Hidden service v3 secret key. */
+  ed25519_secret_key_t *v3;
+} add_onion_secret_key_t;
+
+STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
+                                   const char **key_new_alg_out,
+                                   char **key_new_blob_out,
+                                   add_onion_secret_key_t *decoded_key,
+                                   int *hs_version,
+                                   control_connection_t *conn);
+
+STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg,
+                                   int *created, control_connection_t *conn);
+
+STATIC control_cmd_args_t *control_cmd_parse_args(
+                                   const char *command,
+                                   const control_cmd_syntax_t *syntax,
+                                   size_t body_len,
+                                   const char *body,
+                                   char **error_out);
+
+#endif /* defined(CONTROL_CMD_PRIVATE) */
+
+#ifdef CONTROL_MODULE_PRIVATE
+smartlist_t * get_detached_onion_services(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_CMD_H) */
diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h
new file mode 100644
index 0000000000..8d7a4f55b3
--- /dev/null
+++ b/src/feature/control/control_cmd_args_st.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd_args_st.h
+ * \brief Definition for control_cmd_args_t
+ **/
+
+#ifndef TOR_CONTROL_CMD_ST_H
+#define TOR_CONTROL_CMD_ST_H
+
+struct smartlist_t;
+struct config_line_t;
+
+/**
+ * Parsed arguments for a control command.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+struct control_cmd_args_t {
+  /**
+   * The command itself, as provided by the controller.  Not owned by this
+   * structure.
+   **/
+  const char *command;
+  /**
+   * Positional arguments to the command.
+   **/
+  struct smartlist_t *args;
+  /**
+   * Keyword arguments to the command.
+   **/
+  struct config_line_t *kwargs;
+  /**
+   * Number of bytes in <b>cmddata</b>; 0 if <b>cmddata</b> is not set.
+   **/
+  size_t cmddata_len;
+  /**
+   * A multiline object passed with this command.
+   **/
+  char *cmddata;
+  /**
+   * If set, a nul-terminated string containing the raw unparsed arguments.
+   **/
+  const char *raw_body;
+};
+
+#endif /* !defined(TOR_CONTROL_CMD_ST_H) */
diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h
index 177a916257..c9164f03b3 100644
--- a/src/feature/control/control_connection_st.h
+++ b/src/feature/control/control_connection_st.h
@@ -40,7 +40,8 @@ struct control_connection_t {
   /** A control command that we're reading from the inbuf, but which has not
    * yet arrived completely. */
   char *incoming_cmd;
+  /** The control command that we are currently processing. */
+  char *current_cmd;
 };
 
-#endif
-
+#endif /* !defined(CONTROL_CONNECTION_ST_H) */
diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c
new file mode 100644
index 0000000000..8cf6d6de0b
--- /dev/null
+++ b/src/feature/control/control_events.c
@@ -0,0 +1,2306 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.c
+ * \brief Implement the event-reporting part of the controller API.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/command.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/reasons.h"
+#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/control/control_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "lib/evloop/compat_libevent.h"
+
+static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
+static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/** An event mask of all the events that any controller is interested in
+ * receiving. */
+static event_mask_t global_event_mask = 0;
+
+/** True iff we have disabled log messages from being sent to the controller */
+static int disable_log_messages = 0;
+
+/** Macro: true if any control connection is interested in events of type
+ * <b>e</b>. */
+#define EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & EVENT_MASK_(e)))
+
+/** Macro: true if any event from the bitfield 'e' is interesting. */
+#define ANY_EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & (e)))
+
+static void send_control_event_impl(uint16_t event,
+                                    const char *format, va_list ap)
+  CHECK_PRINTF(2,0);
+static int control_event_status(int type, int severity, const char *format,
+                                va_list args)
+  CHECK_PRINTF(3,0);
+
+static void send_control_event(uint16_t event,
+                               const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+/** Table mapping event values to their names.  Used to implement SETEVENTS
+ * and GETINFO events/names, and to keep they in sync. */
+const struct control_event_t control_event_table[] = {
+  { EVENT_CIRCUIT_STATUS, "CIRC" },
+  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
+  { EVENT_STREAM_STATUS, "STREAM" },
+  { EVENT_OR_CONN_STATUS, "ORCONN" },
+  { EVENT_BANDWIDTH_USED, "BW" },
+  { EVENT_DEBUG_MSG, "DEBUG" },
+  { EVENT_INFO_MSG, "INFO" },
+  { EVENT_NOTICE_MSG, "NOTICE" },
+  { EVENT_WARN_MSG, "WARN" },
+  { EVENT_ERR_MSG, "ERR" },
+  { EVENT_NEW_DESC, "NEWDESC" },
+  { EVENT_ADDRMAP, "ADDRMAP" },
+  { EVENT_DESCCHANGED, "DESCCHANGED" },
+  { EVENT_NS, "NS" },
+  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
+  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
+  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
+  { EVENT_GUARD, "GUARD" },
+  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
+  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
+  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
+  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
+  { EVENT_GOT_SIGNAL, "SIGNAL" },
+  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
+  { EVENT_CONN_BW, "CONN_BW" },
+  { EVENT_CELL_STATS, "CELL_STATS" },
+  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
+  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
+  { EVENT_HS_DESC, "HS_DESC" },
+  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
+  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
+  { 0, NULL },
+};
+
+/** Given a log severity, return the corresponding control event code. */
+static inline int
+log_severity_to_event(int severity)
+{
+  switch (severity) {
+    case LOG_DEBUG: return EVENT_DEBUG_MSG;
+    case LOG_INFO: return EVENT_INFO_MSG;
+    case LOG_NOTICE: return EVENT_NOTICE_MSG;
+    case LOG_WARN: return EVENT_WARN_MSG;
+    case LOG_ERR: return EVENT_ERR_MSG;
+    default: return -1;
+  }
+}
+
+/** Helper: clear bandwidth counters of all origin circuits. */
+static void
+clear_circ_bw_fields(void)
+{
+  origin_circuit_t *ocirc;
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+  }
+  SMARTLIST_FOREACH_END(circ);
+}
+
+/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
+ * connection's event_mask field. */
+void
+control_update_global_event_mask(void)
+{
+  smartlist_t *conns = get_connection_array();
+  event_mask_t old_mask, new_mask;
+  old_mask = global_event_mask;
+  int any_old_per_sec_events = control_any_per_second_event_enabled();
+
+  global_event_mask = 0;
+  SMARTLIST_FOREACH(conns, connection_t *, _conn,
+  {
+    if (_conn->type == CONN_TYPE_CONTROL &&
+        STATE_IS_OPEN(_conn->state)) {
+      control_connection_t *conn = TO_CONTROL_CONN(_conn);
+      global_event_mask |= conn->event_mask;
+    }
+  });
+
+  new_mask = global_event_mask;
+
+  /* Handle the aftermath.  Set up the log callback to tell us only what
+   * we want to hear...*/
+  control_adjust_event_log_severity();
+
+  /* Macro: true if ev was false before and is true now. */
+#define NEWLY_ENABLED(ev) \
+  (! (old_mask & (ev)) && (new_mask & (ev)))
+
+  /* ...then, if we've started logging stream or circ bw, clear the
+   * appropriate fields. */
+  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
+    SMARTLIST_FOREACH(conns, connection_t *, conn,
+    {
+      if (conn->type == CONN_TYPE_AP) {
+        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
+        edge_conn->n_written = edge_conn->n_read = 0;
+      }
+    });
+  }
+  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
+    clear_circ_bw_fields();
+  }
+  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
+    uint64_t r, w;
+    control_get_bytes_rw_last_sec(&r, &w);
+  }
+  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
+    rescan_periodic_events(get_options());
+  }
+
+#undef NEWLY_ENABLED
+}
+
+/** Given a control event code for a message event, return the corresponding
+ * log severity. */
+static inline int
+event_to_log_severity(int event)
+{
+  switch (event) {
+    case EVENT_DEBUG_MSG: return LOG_DEBUG;
+    case EVENT_INFO_MSG: return LOG_INFO;
+    case EVENT_NOTICE_MSG: return LOG_NOTICE;
+    case EVENT_WARN_MSG: return LOG_WARN;
+    case EVENT_ERR_MSG: return LOG_ERR;
+    default: return -1;
+  }
+}
+
+/** Adjust the log severities that result in control_event_logmsg being called
+ * to match the severity of log messages that any controllers are interested
+ * in. */
+void
+control_adjust_event_log_severity(void)
+{
+  int i;
+  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
+
+  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      min_log_event = i;
+      break;
+    }
+  }
+  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      max_log_event = i;
+      break;
+    }
+  }
+  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
+    if (min_log_event > EVENT_NOTICE_MSG)
+      min_log_event = EVENT_NOTICE_MSG;
+    if (max_log_event < EVENT_ERR_MSG)
+      max_log_event = EVENT_ERR_MSG;
+  }
+  if (min_log_event <= max_log_event)
+    change_callback_log_severity(event_to_log_severity(min_log_event),
+                                 event_to_log_severity(max_log_event),
+                                 control_event_logmsg);
+  else
+    change_callback_log_severity(LOG_ERR, LOG_ERR,
+                                 control_event_logmsg);
+}
+
+/** Return true iff the event with code <b>c</b> is being sent to any current
+ * control connection.  This is useful if the amount of work needed to prepare
+ * to call the appropriate control_event_...() function is high.
+ */
+int
+control_event_is_interesting(int event)
+{
+  return EVENT_IS_INTERESTING(event);
+}
+
+/** Return true if any event that needs to fire once a second is enabled. */
+int
+control_any_per_second_event_enabled(void)
+{
+  return ANY_EVENT_IS_INTERESTING(
+      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CELL_STATS) |
+      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CONN_BW) |
+      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
+  );
+}
+
+/* The value of 'get_bytes_read()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_read = 0;
+/* The value of 'get_bytes_written()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_written = 0;
+
+/**
+ * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
+ * and written by Tor since the last call to this function.
+ *
+ * Call this only from the main thread.
+ */
+static void
+control_get_bytes_rw_last_sec(uint64_t *n_read,
+                              uint64_t *n_written)
+{
+  const uint64_t stats_n_bytes_read = get_bytes_read();
+  const uint64_t stats_n_bytes_written = get_bytes_written();
+
+  *n_read = stats_n_bytes_read - stats_prev_n_read;
+  *n_written = stats_n_bytes_written - stats_prev_n_written;
+  stats_prev_n_read = stats_n_bytes_read;
+  stats_prev_n_written = stats_n_bytes_written;
+}
+
+/**
+ * Run all the controller events (if any) that are scheduled to trigger once
+ * per second.
+ */
+void
+control_per_second_events(void)
+{
+  if (!control_any_per_second_event_enabled())
+    return;
+
+  uint64_t bytes_read, bytes_written;
+  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
+  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
+
+  control_event_stream_bandwidth_used();
+  control_event_conn_bandwidth_used();
+  control_event_circ_bandwidth_used();
+  control_event_circuit_cell_stats();
+}
+
+/** Represents an event that's queued to be sent to one or more
+ * controllers. */
+typedef struct queued_event_s {
+  uint16_t event;
+  char *msg;
+} queued_event_t;
+
+/** Pointer to int. If this is greater than 0, we don't allow new events to be
+ * queued. */
+static tor_threadlocal_t block_event_queue_flag;
+
+/** Holds a smartlist of queued_event_t objects that may need to be sent
+ * to one or more controllers */
+static smartlist_t *queued_control_events = NULL;
+
+/** True if the flush_queued_events_event is pending. */
+static int flush_queued_event_pending = 0;
+
+/** Lock to protect the above fields. */
+static tor_mutex_t *queued_control_events_lock = NULL;
+
+/** An event that should fire in order to flush the contents of
+ * queued_control_events. */
+static mainloop_event_t *flush_queued_events_event = NULL;
+
+void
+control_initialize_event_queue(void)
+{
+  if (queued_control_events == NULL) {
+    queued_control_events = smartlist_new();
+  }
+
+  if (flush_queued_events_event == NULL) {
+    struct event_base *b = tor_libevent_get_base();
+    if (b) {
+      flush_queued_events_event =
+        mainloop_event_new(flush_queued_events_cb, NULL);
+      tor_assert(flush_queued_events_event);
+    }
+  }
+
+  if (queued_control_events_lock == NULL) {
+    queued_control_events_lock = tor_mutex_new();
+    tor_threadlocal_init(&block_event_queue_flag);
+  }
+}
+
+static int *
+get_block_event_queue(void)
+{
+  int *val = tor_threadlocal_get(&block_event_queue_flag);
+  if (PREDICT_UNLIKELY(val == NULL)) {
+    val = tor_malloc_zero(sizeof(int));
+    tor_threadlocal_set(&block_event_queue_flag, val);
+  }
+  return val;
+}
+
+/** Helper: inserts an event on the list of events queued to be sent to
+ * one or more controllers, and schedules the events to be flushed if needed.
+ *
+ * This function takes ownership of <b>msg</b>, and may free it.
+ *
+ * We queue these events rather than send them immediately in order to break
+ * the dependency in our callgraph from code that generates events for the
+ * controller, and the network layer at large.  Otherwise, nearly every
+ * interesting part of Tor would potentially call every other interesting part
+ * of Tor.
+ */
+MOCK_IMPL(STATIC void,
+queue_control_event_string,(uint16_t event, char *msg))
+{
+  /* This is redundant with checks done elsewhere, but it's a last-ditch
+   * attempt to avoid queueing something we shouldn't have to queue. */
+  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
+    tor_free(msg);
+    return;
+  }
+
+  int *block_event_queue = get_block_event_queue();
+  if (*block_event_queue) {
+    tor_free(msg);
+    return;
+  }
+
+  queued_event_t *ev = tor_malloc(sizeof(*ev));
+  ev->event = event;
+  ev->msg = msg;
+
+  /* No queueing an event while queueing an event */
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  tor_assert(queued_control_events);
+  smartlist_add(queued_control_events, ev);
+
+  int activate_event = 0;
+  if (! flush_queued_event_pending && in_main_thread()) {
+    activate_event = 1;
+    flush_queued_event_pending = 1;
+  }
+
+  tor_mutex_release(queued_control_events_lock);
+
+  --*block_event_queue;
+
+  /* We just put an event on the queue; mark the queue to be
+   * flushed.  We only do this from the main thread for now; otherwise,
+   * we'd need to incur locking overhead in Libevent or use a socket.
+   */
+  if (activate_event) {
+    tor_assert(flush_queued_events_event);
+    mainloop_event_activate(flush_queued_events_event);
+  }
+}
+
+#define queued_event_free(ev) \
+  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
+
+/** Release all storage held by <b>ev</b>. */
+static void
+queued_event_free_(queued_event_t *ev)
+{
+  if (ev == NULL)
+    return;
+
+  tor_free(ev->msg);
+  tor_free(ev);
+}
+
+/** Send every queued event to every controller that's interested in it,
+ * and remove the events from the queue.  If <b>force</b> is true,
+ * then make all controllers send their data out immediately, since we
+ * may be about to shut down. */
+static void
+queued_events_flush_all(int force)
+{
+  /* Make sure that we get all the pending log events, if there are any. */
+  flush_pending_log_callbacks();
+
+  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
+    return;
+  }
+  smartlist_t *all_conns = get_connection_array();
+  smartlist_t *controllers = smartlist_new();
+  smartlist_t *queued_events;
+
+  int *block_event_queue = get_block_event_queue();
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  /* No queueing an event while flushing events. */
+  flush_queued_event_pending = 0;
+  queued_events = queued_control_events;
+  queued_control_events = smartlist_new();
+  tor_mutex_release(queued_control_events_lock);
+
+  /* Gather all the controllers that will care... */
+  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_CONTROL &&
+        !conn->marked_for_close &&
+        conn->state == CONTROL_CONN_STATE_OPEN) {
+      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
+
+      smartlist_add(controllers, control_conn);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
+    const event_mask_t bit = ((event_mask_t)1) << ev->event;
+    const size_t msg_len = strlen(ev->msg);
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      if (control_conn->event_mask & bit) {
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
+      }
+    } SMARTLIST_FOREACH_END(control_conn);
+
+    queued_event_free(ev);
+  } SMARTLIST_FOREACH_END(ev);
+
+  if (force) {
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      connection_flush(TO_CONN(control_conn));
+    } SMARTLIST_FOREACH_END(control_conn);
+  }
+
+  smartlist_free(queued_events);
+  smartlist_free(controllers);
+
+  --*block_event_queue;
+}
+
+/** Libevent callback: Flushes pending events to controllers that are
+ * interested in them. */
+static void
+flush_queued_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void) event;
+  (void) arg;
+  queued_events_flush_all(0);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is given by <b>msg</b>.
+ *
+ * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
+ * respect to the EXTENDED_EVENTS feature. */
+MOCK_IMPL(STATIC void,
+send_control_event_string,(uint16_t event,
+                           const char *msg))
+{
+  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
+  queue_control_event_string(event, tor_strdup(msg));
+}
+
+/** Helper for send_control_event and control_event_status:
+ * Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event_impl(uint16_t event,
+                        const char *format, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, format, ap);
+  if (len < 0) {
+    log_warn(LD_BUG, "Unable to format event for controller.");
+    return;
+  }
+
+  queue_control_event_string(event, buf);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event(uint16_t event,
+                   const char *format, ...)
+{
+  va_list ap;
+  va_start(ap, format);
+  send_control_event_impl(event, format, ap);
+  va_end(ap);
+}
+
+/** Something major has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+int
+control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                             int reason_code)
+{
+  const char *status;
+  char reasons[64] = "";
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
+    return 0;
+  tor_assert(circ);
+
+  switch (tp)
+    {
+    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case CIRC_EVENT_BUILT: status = "BUILT"; break;
+    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
+    case CIRC_EVENT_FAILED: status = "FAILED"; break;
+    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
+    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
+    char unk_reason_buf[16];
+    if (!reason_str) {
+      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
+      reason_str = unk_reason_buf;
+    }
+    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
+    } else {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=%s", reason_str);
+    }
+  }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS,
+                                "650 CIRC %lu %s%s%s%s\r\n",
+                                (unsigned long)circ->global_identifier,
+                                status, sp,
+                                circdesc,
+                                reasons);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/** Something minor has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+static int
+control_event_circuit_status_minor(origin_circuit_t *circ,
+                                   circuit_status_minor_event_t e,
+                                   int purpose, const struct timeval *tv)
+{
+  const char *event_desc;
+  char event_tail[160] = "";
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
+    return 0;
+  tor_assert(circ);
+
+  switch (e)
+    {
+    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
+      event_desc = "PURPOSE_CHANGED";
+
+      {
+        /* event_tail can currently be up to 68 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "");
+      }
+
+      break;
+    case CIRC_MINOR_EVENT_CANNIBALIZED:
+      event_desc = "CANNIBALIZED";
+
+      {
+        /* event_tail can currently be up to 130 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        const struct timeval *old_timestamp_began = tv;
+        char tbuf[ISO_TIME_USEC_LEN+1];
+        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
+
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "",
+                     tbuf);
+      }
+
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
+                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
+                       (unsigned long)circ->global_identifier,
+                       event_desc, sp,
+                       circdesc,
+                       event_tail);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
+ * interested controllers.
+ */
+int
+control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                      int old_purpose)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+                                            old_purpose,
+                                            NULL);
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
+ * created-time from <b>old_tv_created</b>: tell any interested controllers.
+ */
+int
+control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                   int old_purpose,
+                                   const struct timeval *old_tv_created)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_CANNIBALIZED,
+                                            old_purpose,
+                                            old_tv_created);
+}
+
+/** Something has happened to the stream associated with AP connection
+ * <b>conn</b>: tell any interested control connections. */
+int
+control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
+                            int reason_code)
+{
+  char reason_buf[64];
+  char addrport_buf[64];
+  const char *status;
+  circuit_t *circ;
+  origin_circuit_t *origin_circ = NULL;
+  char buf[256];
+  const char *purpose = "";
+  tor_assert(conn->socks_request);
+
+  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
+    return 0;
+
+  if (tp == STREAM_EVENT_CLOSED &&
+      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
+    return 0;
+
+  write_stream_target_to_buf(conn, buf, sizeof(buf));
+
+  reason_buf[0] = '\0';
+  switch (tp)
+    {
+    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
+    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
+    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
+    case STREAM_EVENT_FAILED: status = "FAILED"; break;
+    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
+    case STREAM_EVENT_NEW: status = "NEW"; break;
+    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
+    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
+    case STREAM_EVENT_REMAP: status = "REMAP"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (reason_code && (tp == STREAM_EVENT_FAILED ||
+                      tp == STREAM_EVENT_CLOSED ||
+                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
+    const char *reason_str = stream_end_reason_to_control_string(reason_code);
+    char *r = NULL;
+    if (!reason_str) {
+      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
+      reason_str = r;
+    }
+    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=END REMOTE_REASON=%s", reason_str);
+    else
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=%s", reason_str);
+    tor_free(r);
+  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
+    switch (reason_code) {
+    case REMAP_STREAM_SOURCE_CACHE:
+      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
+      break;
+    case REMAP_STREAM_SOURCE_EXIT:
+      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
+      break;
+    default:
+      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
+                   reason_code);
+      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
+      break;
+    }
+  }
+
+  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
+    /*
+     * When the control conn is an AF_UNIX socket and we have no address,
+     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
+     * dnsserv.c.
+     */
+    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
+      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
+                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
+    } else {
+      /*
+       * else leave it blank so control on AF_UNIX doesn't need to make
+       * something up.
+       */
+      addrport_buf[0] = '\0';
+    }
+  } else {
+    addrport_buf[0] = '\0';
+  }
+
+  if (tp == STREAM_EVENT_NEW_RESOLVE) {
+    purpose = " PURPOSE=DNS_REQUEST";
+  } else if (tp == STREAM_EVENT_NEW) {
+    if (conn->use_begindir) {
+      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
+      int linked_dir_purpose = -1;
+      if (linked && linked->type == CONN_TYPE_DIR)
+        linked_dir_purpose = linked->purpose;
+      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
+        purpose = " PURPOSE=DIR_UPLOAD";
+      else
+        purpose = " PURPOSE=DIR_FETCH";
+    } else
+      purpose = " PURPOSE=USER";
+  }
+
+  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+  if (circ && CIRCUIT_IS_ORIGIN(circ))
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+  send_control_event(EVENT_STREAM_STATUS,
+                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
+                     (ENTRY_TO_CONN(conn)->global_identifier),
+                     status,
+                        origin_circ?
+                           (unsigned long)origin_circ->global_identifier : 0ul,
+                        buf, reason_buf, addrport_buf, purpose);
+
+  /* XXX need to specify its intended exit, etc? */
+
+  return 0;
+}
+
+/** Called when the status of an OR connection <b>conn</b> changes: tell any
+ * interested control connections. <b>tp</b> is the new status for the
+ * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
+ * may be the reason why.
+ */
+int
+control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
+                             int reason)
+{
+  int ncircs = 0;
+  const char *status;
+  char name[128];
+  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
+
+  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
+    return 0;
+
+  switch (tp)
+    {
+    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
+    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
+    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
+    case OR_CONN_EVENT_NEW: status = "NEW"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (conn->chan) {
+    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
+  } else {
+    ncircs = 0;
+  }
+  ncircs += connection_or_get_num_circuits(conn);
+  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
+    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
+  }
+
+  orconn_target_get_name(name, sizeof(name), conn);
+  send_control_event(EVENT_OR_CONN_STATUS,
+                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
+                              name, status,
+                              reason ? " REASON=" : "",
+                              orconn_end_reason_to_control_string(reason),
+                              ncircs_buf,
+                              (conn->base_.global_identifier));
+
+  return 0;
+}
+
+/**
+ * Print out STREAM_BW event for a single conn
+ */
+int
+control_event_stream_bandwidth(edge_connection_t *edge_conn)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    if (!edge_conn->n_read && !edge_conn->n_written)
+      return 0;
+
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
+    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                       (edge_conn->base_.global_identifier),
+                       (unsigned long)edge_conn->n_read,
+                       (unsigned long)edge_conn->n_written,
+                       tbuf);
+
+    edge_conn->n_written = edge_conn->n_read = 0;
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth streams have used. */
+int
+control_event_stream_bandwidth_used(void)
+{
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    smartlist_t *conns = get_connection_array();
+    edge_connection_t *edge_conn;
+    struct timeval now;
+    char tbuf[ISO_TIME_USEC_LEN+1];
+
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
+    {
+        if (conn->type != CONN_TYPE_AP)
+          continue;
+        edge_conn = TO_EDGE_CONN(conn);
+        if (!edge_conn->n_read && !edge_conn->n_written)
+          continue;
+
+        tor_gettimeofday(&now);
+        format_iso_time_nospace_usec(tbuf, &now);
+        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                           (edge_conn->base_.global_identifier),
+                           (unsigned long)edge_conn->n_read,
+                           (unsigned long)edge_conn->n_written,
+                           tbuf);
+
+        edge_conn->n_written = edge_conn->n_read = 0;
+    }
+    SMARTLIST_FOREACH_END(conn);
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control connections
+ * how much bandwidth origin circuits have used. */
+int
+control_event_circ_bandwidth_used(void)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
+  }
+  SMARTLIST_FOREACH_END(circ);
+
+  return 0;
+}
+
+/**
+ * Emit a CIRC_BW event line for a specific circuit.
+ *
+ * This function sets the values it emits to 0, and does not emit
+ * an event if there is no new data to report since the last call.
+ *
+ * Therefore, it may be called at any frequency.
+ */
+int
+control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+
+  tor_assert(ocirc);
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  /* n_read_circ_bw and n_written_circ_bw are always updated
+   * when there is any new cell on a circuit, and set to 0 after
+   * the event, below.
+   *
+   * Therefore, checking them is sufficient to determine if there
+   * is new data to report. */
+  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
+    return 0;
+
+  tor_gettimeofday(&now);
+  format_iso_time_nospace_usec(tbuf, &now);
+  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
+                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
+                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
+                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
+                     ocirc->global_identifier,
+                     (unsigned long)ocirc->n_read_circ_bw,
+                     (unsigned long)ocirc->n_written_circ_bw,
+                     tbuf,
+                     (unsigned long)ocirc->n_delivered_read_circ_bw,
+                     (unsigned long)ocirc->n_overhead_read_circ_bw,
+                     (unsigned long)ocirc->n_delivered_written_circ_bw,
+                     (unsigned long)ocirc->n_overhead_written_circ_bw);
+  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+
+  return 0;
+}
+
+/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
+  * bandwidth counters. */
+int
+control_event_conn_bandwidth(connection_t *conn)
+{
+  const char *conn_type_str;
+  if (!get_options()->TestingEnableConnBwEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
+    return 0;
+  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
+    return 0;
+  switch (conn->type) {
+    case CONN_TYPE_OR:
+      conn_type_str = "OR";
+      break;
+    case CONN_TYPE_DIR:
+      conn_type_str = "DIR";
+      break;
+    case CONN_TYPE_EXIT:
+      conn_type_str = "EXIT";
+      break;
+    default:
+      return 0;
+  }
+  send_control_event(EVENT_CONN_BW,
+                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
+                     "READ=%lu WRITTEN=%lu\r\n",
+                     (conn->global_identifier),
+                     conn_type_str,
+                     (unsigned long)conn->n_read_conn_bw,
+                     (unsigned long)conn->n_written_conn_bw);
+  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth connections have used. */
+int
+control_event_conn_bandwidth_used(void)
+{
+  if (get_options()->TestingEnableConnBwEvent &&
+      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
+    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
+                      control_event_conn_bandwidth(conn));
+  }
+  return 0;
+}
+
+/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
+ * cells, removed cells, and waiting times by cell command and direction.
+ * Store results in <b>cell_stats</b>.  Free cell statistics of the
+ * circuit afterwards. */
+void
+sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
+{
+  memset(cell_stats, 0, sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
+                          const testing_cell_stats_entry_t *, ent) {
+    tor_assert(ent->command <= CELL_COMMAND_MAX_);
+    if (!ent->removed && !ent->exitward) {
+      cell_stats->added_cells_appward[ent->command] += 1;
+    } else if (!ent->removed && ent->exitward) {
+      cell_stats->added_cells_exitward[ent->command] += 1;
+    } else if (!ent->exitward) {
+      cell_stats->removed_cells_appward[ent->command] += 1;
+      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
+    } else {
+      cell_stats->removed_cells_exitward[ent->command] += 1;
+      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+  circuit_clear_testing_cell_stats(circ);
+}
+
+/** Helper: append a cell statistics string to <code>event_parts</code>,
+ * prefixed with <code>key</code>=.  Statistics consist of comma-separated
+ * key:value pairs with lower-case command strings as keys and cell
+ * numbers or total waiting times as values.  A key:value pair is included
+ * if the entry in <code>include_if_non_zero</code> is not zero, but with
+ * the (possibly zero) entry from <code>number_to_include</code>.  Both
+ * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
+ * entry in <code>include_if_non_zero</code> is positive, no string will
+ * be added to <code>event_parts</code>. */
+void
+append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
+                             const uint64_t *include_if_non_zero,
+                             const uint64_t *number_to_include)
+{
+  smartlist_t *key_value_strings = smartlist_new();
+  int i;
+  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
+    if (include_if_non_zero[i] > 0) {
+      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
+                             cell_command_to_string(i),
+                             (number_to_include[i]));
+    }
+  }
+  if (smartlist_len(key_value_strings) > 0) {
+    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
+    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
+    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
+    tor_free(joined);
+  }
+  smartlist_free(key_value_strings);
+}
+
+/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
+ * CELL_STATS event and write result string to <b>event_string</b>. */
+void
+format_cell_stats(char **event_string, circuit_t *circ,
+                  cell_stats_t *cell_stats)
+{
+  smartlist_t *event_parts = smartlist_new();
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "ID=%lu",
+                 (unsigned long)ocirc->global_identifier);
+  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
+                 (unsigned long)or_circ->p_circ_id);
+    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
+                 (or_circ->p_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "InboundAdded",
+                                 cell_stats->added_cells_appward,
+                                 cell_stats->added_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundRemoved",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->removed_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundTime",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->total_time_appward);
+  }
+  if (circ->n_chan) {
+    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
+                     (unsigned long)circ->n_circ_id);
+    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
+                 (circ->n_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "OutboundAdded",
+                                 cell_stats->added_cells_exitward,
+                                 cell_stats->added_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundRemoved",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->removed_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundTime",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->total_time_exitward);
+  }
+  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
+  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
+  smartlist_free(event_parts);
+}
+
+/** A second or more has elapsed: tell any interested control connection
+ * how many cells have been processed for a given circuit. */
+int
+control_event_circuit_cell_stats(void)
+{
+  cell_stats_t *cell_stats;
+  char *event_string;
+  if (!get_options()->TestingEnableCellStatsEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
+    return 0;
+  cell_stats = tor_malloc(sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!circ->testing_cell_stats)
+      continue;
+    sum_up_cell_stats_by_command(circ, cell_stats);
+    format_cell_stats(&event_string, circ, cell_stats);
+    send_control_event(EVENT_CELL_STATS,
+                       "650 CELL_STATS %s\r\n", event_string);
+    tor_free(event_string);
+  }
+  SMARTLIST_FOREACH_END(circ);
+  tor_free(cell_stats);
+  return 0;
+}
+
+/* about 5 minutes worth. */
+#define N_BW_EVENTS_TO_CACHE 300
+/* Index into cached_bw_events to next write. */
+static int next_measurement_idx = 0;
+/* number of entries set in n_measurements */
+static int n_measurements = 0;
+static struct cached_bw_event_s {
+  uint32_t n_read;
+  uint32_t n_written;
+} cached_bw_events[N_BW_EVENTS_TO_CACHE];
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth we used. */
+int
+control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
+{
+  cached_bw_events[next_measurement_idx].n_read = n_read;
+  cached_bw_events[next_measurement_idx].n_written = n_written;
+  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
+    next_measurement_idx = 0;
+  if (n_measurements < N_BW_EVENTS_TO_CACHE)
+    ++n_measurements;
+
+  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
+    send_control_event(EVENT_BANDWIDTH_USED,
+                       "650 BW %lu %lu\r\n",
+                       (unsigned long)n_read,
+                       (unsigned long)n_written);
+  }
+
+  return 0;
+}
+
+char *
+get_bw_samples(void)
+{
+  int i;
+  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
+    % N_BW_EVENTS_TO_CACHE;
+  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+
+  smartlist_t *elements = smartlist_new();
+
+  for (i = 0; i < n_measurements; ++i) {
+    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+    const struct cached_bw_event_s *bwe = &cached_bw_events[idx];
+
+    smartlist_add_asprintf(elements, "%u,%u",
+                           (unsigned)bwe->n_read,
+                           (unsigned)bwe->n_written);
+
+    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/** Called when we are sending a log message to the controllers: suspend
+ * sending further log messages to the controllers until we're done.  Used by
+ * CONN_LOG_PROTECT. */
+void
+disable_control_logging(void)
+{
+  ++disable_log_messages;
+}
+
+/** We're done sending a log message to the controllers: re-enable controller
+ * logging.  Used by CONN_LOG_PROTECT. */
+void
+enable_control_logging(void)
+{
+  if (--disable_log_messages < 0)
+    tor_assert(0);
+}
+
+/** We got a log message: tell any interested control connections. */
+void
+control_event_logmsg(int severity, log_domain_mask_t domain, const char *msg)
+{
+  int event;
+
+  /* Don't even think of trying to add stuff to a buffer from a cpuworker
+   * thread. (See #25987 for plan to fix.) */
+  if (! in_main_thread())
+    return;
+
+  if (disable_log_messages)
+    return;
+
+  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
+      severity <= LOG_NOTICE) {
+    char *esc = esc_for_log(msg);
+    ++disable_log_messages;
+    control_event_general_status(severity, "BUG REASON=%s", esc);
+    --disable_log_messages;
+    tor_free(esc);
+  }
+
+  event = log_severity_to_event(severity);
+  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
+    char *b = NULL;
+    const char *s;
+    if (strchr(msg, '\n')) {
+      char *cp;
+      b = tor_strdup(msg);
+      for (cp = b; *cp; ++cp)
+        if (*cp == '\r' || *cp == '\n')
+          *cp = ' ';
+    }
+    switch (severity) {
+      case LOG_DEBUG: s = "DEBUG"; break;
+      case LOG_INFO: s = "INFO"; break;
+      case LOG_NOTICE: s = "NOTICE"; break;
+      case LOG_WARN: s = "WARN"; break;
+      case LOG_ERR: s = "ERR"; break;
+      default: s = "UnknownLogSeverity"; break;
+    }
+    ++disable_log_messages;
+    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
+    if (severity == LOG_ERR) {
+      /* Force a flush, since we may be about to die horribly */
+      queued_events_flush_all(1);
+    }
+    --disable_log_messages;
+    tor_free(b);
+  }
+}
+
+/**
+ * Logging callback: called when there is a queued pending log callback.
+ */
+void
+control_event_logmsg_pending(void)
+{
+  if (! in_main_thread()) {
+    /* We can't handle this case yet, since we're using a
+     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
+     * use a different mechanism instead: see #25987.
+     **/
+    return;
+  }
+  tor_assert(flush_queued_events_event);
+  mainloop_event_activate(flush_queued_events_event);
+}
+
+/** Called whenever we receive new router descriptors: tell any
+ * interested control connections.  <b>routers</b> is a list of
+ * routerinfo_t's.
+ */
+int
+control_event_descriptors_changed(smartlist_t *routers)
+{
+  char *msg;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
+    return 0;
+
+  {
+    smartlist_t *names = smartlist_new();
+    char *ids;
+    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
+        router_get_verbose_nickname(b, ri);
+        smartlist_add(names, b);
+      });
+    ids = smartlist_join_strings(names, " ", 0, NULL);
+    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
+    send_control_event_string(EVENT_NEW_DESC,  msg);
+    tor_free(ids);
+    tor_free(msg);
+    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
+    smartlist_free(names);
+  }
+  return 0;
+}
+
+/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
+ * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
+ * <b>error</b> is non-NULL, it is an error code describing the failure
+ * mode of the mapping.
+ */
+int
+control_event_address_mapped(const char *from, const char *to, time_t expires,
+                             const char *error, const int cached)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
+    return 0;
+
+  if (expires < 3 || expires == TIME_MAX)
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s NEVER %s%s"
+                                "CACHED=\"%s\"\r\n",
+                                  from, to, error?error:"", error?" ":"",
+                                cached?"YES":"NO");
+  else {
+    char buf[ISO_TIME_LEN+1];
+    char buf2[ISO_TIME_LEN+1];
+    format_local_iso_time(buf,expires);
+    format_iso_time(buf2,expires);
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s \"%s\""
+                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
+                                from, to, buf,
+                                error?error:"", error?" ":"",
+                                buf2, cached?"YES":"NO");
+  }
+
+  return 0;
+}
+/** The network liveness has changed; this is called from circuitstats.c
+ * whenever we receive a cell, or when timeout expires and we assume the
+ * network is down. */
+int
+control_event_network_liveness_update(int liveness)
+{
+  if (liveness > 0) {
+    if (get_cached_network_liveness() <= 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(1);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS UP\r\n");
+    }
+    /* else was already live, no-op */
+  } else {
+    if (get_cached_network_liveness() > 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(0);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS DOWN\r\n");
+    }
+    /* else was already dead, no-op */
+  }
+
+  return 0;
+}
+
+/** Helper function for NS-style events. Constructs and sends an event
+ * of type <b>event</b> with string <b>event_string</b> out of the set of
+ * networkstatuses <b>statuses</b>. Currently it is used for NS events
+ * and NEWCONSENSUS events. */
+static int
+control_event_networkstatus_changed_helper(smartlist_t *statuses,
+                                           uint16_t event,
+                                           const char *event_string)
+{
+  smartlist_t *strs;
+  char *s, *esc = NULL;
+  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
+    return 0;
+
+  strs = smartlist_new();
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
+  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
+    {
+      s = networkstatus_getinfo_helper_single(rs);
+      if (!s) continue;
+      smartlist_add(strs, s);
+    });
+
+  s = smartlist_join_strings(strs, "", 0, NULL);
+  write_escaped_data(s, strlen(s), &esc);
+  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
+  smartlist_free(strs);
+  tor_free(s);
+  send_control_event_string(event,  esc);
+  send_control_event_string(event,
+                            "650 OK\r\n");
+
+  tor_free(esc);
+  return 0;
+}
+
+/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
+ * an NS event to any controller that cares. */
+int
+control_event_networkstatus_changed(smartlist_t *statuses)
+{
+  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
+}
+
+/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
+ * event consisting of an NS-style line for each relay in the consensus. */
+int
+control_event_newconsensus(const networkstatus_t *consensus)
+{
+  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
+    return 0;
+  return control_event_networkstatus_changed_helper(
+           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
+}
+
+/** Called when we compute a new circuitbuildtimeout */
+int
+control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                               const char *args)
+{
+  const char *type_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
+    return 0;
+
+  switch (type) {
+    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
+      type_string = "COMPUTED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESET:
+      type_string = "RESET";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
+      type_string = "SUSPENDED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_DISCARD:
+      type_string = "DISCARD";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESUME:
+      type_string = "RESUME";
+      break;
+    default:
+      type_string = "UNKNOWN";
+      break;
+  }
+
+  send_control_event(EVENT_BUILDTIMEOUT_SET,
+                     "650 BUILDTIMEOUT_SET %s %s\r\n",
+                     type_string, args);
+
+  return 0;
+}
+
+/** Called when a signal has been processed from signal_callback */
+int
+control_event_signal(uintptr_t signal_num)
+{
+  const char *signal_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
+    return 0;
+
+  for (unsigned i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if ((int)signal_num == signal_table[i].sig) {
+      signal_string = signal_table[i].signal_name;
+      break;
+    }
+  }
+
+  if (signal_string == NULL) {
+    log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
+             (unsigned long)signal_num);
+    return -1;
+  }
+
+  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
+                     signal_string);
+  return 0;
+}
+
+/** Called when a single local_routerstatus_t has changed: Sends an NS event
+ * to any controller that cares. */
+int
+control_event_networkstatus_changed_single(const routerstatus_t *rs)
+{
+  smartlist_t *statuses;
+  int r;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NS))
+    return 0;
+
+  statuses = smartlist_new();
+  smartlist_add(statuses, (void*)rs);
+  r = control_event_networkstatus_changed(statuses);
+  smartlist_free(statuses);
+  return r;
+}
+
+/** Our own router descriptor has changed; tell any controllers that care.
+ */
+int
+control_event_my_descriptor_changed(void)
+{
+  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
+  return 0;
+}
+
+/** Helper: sends a status event where <b>type</b> is one of
+ * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
+ * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
+ * string corresponding to <b>args</b>. */
+static int
+control_event_status(int type, int severity, const char *format, va_list args)
+{
+  char *user_buf = NULL;
+  char format_buf[160];
+  const char *status, *sev;
+
+  switch (type) {
+    case EVENT_STATUS_GENERAL:
+      status = "STATUS_GENERAL";
+      break;
+    case EVENT_STATUS_CLIENT:
+      status = "STATUS_CLIENT";
+      break;
+    case EVENT_STATUS_SERVER:
+      status = "STATUS_SERVER";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status type %d", type);
+      return -1;
+  }
+  switch (severity) {
+    case LOG_NOTICE:
+      sev = "NOTICE";
+      break;
+    case LOG_WARN:
+      sev = "WARN";
+      break;
+    case LOG_ERR:
+      sev = "ERR";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
+      return -1;
+  }
+  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
+                   status, sev)<0) {
+    log_warn(LD_BUG, "Format string too long.");
+    return -1;
+  }
+  tor_vasprintf(&user_buf, format, args);
+
+  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
+  tor_free(user_buf);
+  return 0;
+}
+
+#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
+  int r;                                                        \
+  do {                                                          \
+    va_list ap;                                                 \
+    if (!EVENT_IS_INTERESTING(event))                           \
+      return 0;                                                 \
+                                                                \
+    va_start(ap, format);                                       \
+    r = control_event_status((event), (sev), format, ap);       \
+    va_end(ap);                                                 \
+  } while (0)
+
+/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_general_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_general_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_client_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_client_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_server_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_server_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Called when the status of an entry guard with the given <b>nickname</b>
+ * and identity <b>digest</b> has changed to <b>status</b>: tells any
+ * controllers that care. */
+int
+control_event_guard(const char *nickname, const char *digest,
+                    const char *status)
+{
+  char hbuf[HEX_DIGEST_LEN+1];
+  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
+  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
+    return 0;
+
+  {
+    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
+    const node_t *node = node_get_by_id(digest);
+    if (node) {
+      node_get_verbose_nickname(node, buf);
+    } else {
+      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
+    }
+    send_control_event(EVENT_GUARD,
+                       "650 GUARD ENTRY %s %s\r\n", buf, status);
+  }
+  return 0;
+}
+
+/** Called when a configuration option changes. This is generally triggered
+ * by SETCONF requests and RELOAD/SIGHUP signals. The <b>elements</b> is
+ * a smartlist_t containing (key, value, ...) pairs in sequence.
+ * <b>value</b> can be NULL. */
+int
+control_event_conf_changed(const smartlist_t *elements)
+{
+  int i;
+  char *result;
+  smartlist_t *lines;
+  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) ||
+      smartlist_len(elements) == 0) {
+    return 0;
+  }
+  lines = smartlist_new();
+  for (i = 0; i < smartlist_len(elements); i += 2) {
+    char *k = smartlist_get(elements, i);
+    char *v = smartlist_get(elements, i+1);
+    if (v == NULL) {
+      smartlist_add_asprintf(lines, "650-%s", k);
+    } else {
+      smartlist_add_asprintf(lines, "650-%s=%s", k, v);
+    }
+  }
+  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
+  send_control_event(EVENT_CONF_CHANGED,
+    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
+  tor_free(result);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+  return 0;
+}
+
+/** We just generated a new summary of which countries we've seen clients
+ * from recently. Send a copy to the controller in case it wants to
+ * display it for the user. */
+void
+control_event_clients_seen(const char *controller_str)
+{
+  send_control_event(EVENT_CLIENTS_SEEN,
+    "650 CLIENTS_SEEN %s\r\n", controller_str);
+}
+
+/** A new pluggable transport called <b>transport_name</b> was
+ *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
+ *  "server" or "client" depending on the mode of the pluggable
+ *  transport.
+ *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
+ */
+void
+control_event_transport_launched(const char *mode, const char *transport_name,
+                                 tor_addr_t *addr, uint16_t port)
+{
+  send_control_event(EVENT_TRANSPORT_LAUNCHED,
+                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
+                     mode, transport_name, fmt_addr(addr), port);
+}
+
+/** A pluggable transport called <b>pt_name</b> has emitted a log message
+ * found in <b>message</b> at <b>severity</b> log level. */
+void
+control_event_pt_log(const char *log)
+{
+  send_control_event(EVENT_PT_LOG,
+                     "650 PT_LOG %s\r\n",
+                     log);
+}
+
+/** A pluggable transport has emitted a STATUS message found in
+ * <b>status</b>. */
+void
+control_event_pt_status(const char *status)
+{
+  send_control_event(EVENT_PT_STATUS,
+                     "650 PT_STATUS %s\r\n",
+                     status);
+}
+
+/** Convert rendezvous auth type to string for HS_DESC control events
+ */
+const char *
+rend_auth_type_to_string(rend_auth_type_t auth_type)
+{
+  const char *str;
+
+  switch (auth_type) {
+    case REND_NO_AUTH:
+      str = "NO_AUTH";
+      break;
+    case REND_BASIC_AUTH:
+      str = "BASIC_AUTH";
+      break;
+    case REND_STEALTH_AUTH:
+      str = "STEALTH_AUTH";
+      break;
+    default:
+      str = "UNKNOWN";
+  }
+
+  return str;
+}
+
+/** Return either the onion address if the given pointer is a non empty
+ * string else the unknown string. */
+static const char *
+rend_hsaddress_str_or_unknown(const char *onion_address)
+{
+  static const char *str_unknown = "UNKNOWN";
+  const char *str_ret = str_unknown;
+
+  /* No valid pointer, unknown it is. */
+  if (!onion_address) {
+    goto end;
+  }
+  /* Empty onion address thus we don't know, unknown it is. */
+  if (onion_address[0] == '\0') {
+    goto end;
+  }
+  /* All checks are good so return the given onion address. */
+  str_ret = onion_address;
+
+ end:
+  return str_ret;
+}
+
+/** send HS_DESC requested event.
+ *
+ * <b>rend_query</b> is used to fetch requested onion address and auth type.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
+ */
+void
+control_event_hs_descriptor_requested(const char *onion_address,
+                                      rend_auth_type_t auth_type,
+                                      const char *id_digest,
+                                      const char *desc_id,
+                                      const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC CREATED event when a local service generates a descriptor.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>desc_id</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number. If it is negative, it
+ * is ignored.
+ */
+void
+control_event_hs_descriptor_created(const char *onion_address,
+                                    const char *desc_id,
+                                    int replica)
+{
+  char *replica_field = NULL;
+
+  if (BUG(!onion_address || !desc_id)) {
+    return;
+  }
+
+  if (replica >= 0) {
+    tor_asprintf(&replica_field, " REPLICA=%d", replica);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
+                     onion_address, desc_id,
+                     replica_field ? replica_field : "");
+  tor_free(replica_field);
+}
+
+/** send HS_DESC upload event.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id</b> is the ID of requested hs descriptor.
+ */
+void
+control_event_hs_descriptor_upload(const char *onion_address,
+                                   const char *id_digest,
+                                   const char *desc_id,
+                                   const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!onion_address || !id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
+                     onion_address,
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hsv2_descriptor_received
+ * control_event_hsv2_descriptor_failed
+ * control_event_hsv3_descriptor_failed
+ *
+ * So do not call this function directly.
+ */
+static void
+event_hs_descriptor_receive_end(const char *action,
+                                const char *onion_address,
+                                const char *desc_id,
+                                rend_auth_type_t auth_type,
+                                const char *hsdir_id_digest,
+                                const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !onion_address)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s %s %s%s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     desc_id ? desc_id : "",
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hs_descriptor_uploaded
+ * control_event_hs_descriptor_upload_failed
+ *
+ * So do not call this function directly.
+ */
+void
+control_event_hs_descriptor_upload_end(const char *action,
+                                       const char *onion_address,
+                                       const char *id_digest,
+                                       const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !id_digest)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     node_describe_longname_by_id(id_digest),
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** For an HS descriptor query <b>rend_data</b>, using the
+ * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
+ * which descriptor ID in the query is the right one.
+ *
+ * Return a pointer of the binary descriptor ID found in the query's object
+ * or NULL if not found. */
+static const char *
+get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
+{
+  int replica;
+  const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
+
+  /* Possible if the fetch was done using a descriptor ID. This means that
+   * the HSFETCH command was used. */
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
+    goto end;
+  }
+
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
+  /* OK, we have an onion address so now let's find which descriptor ID
+   * is the one associated with the HSDir fingerprint. */
+  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
+       replica++) {
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
+
+    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
+      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
+        /* Found it! This descriptor ID is the right one. */
+        desc_id = digest;
+        goto end;
+      }
+    } SMARTLIST_FOREACH_END(fingerprint);
+  }
+
+ end:
+  return desc_id;
+}
+
+/** send HS_DESC RECEIVED event
+ *
+ * called when we successfully received a hidden service descriptor.
+ */
+void
+control_event_hsv2_descriptor_received(const char *onion_address,
+                                       const rend_data_t *rend_data,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/* Send HS_DESC RECEIVED event
+ *
+ * Called when we successfully received a hidden service descriptor. */
+void
+control_event_hsv3_descriptor_received(const char *onion_address,
+                                       const char *desc_id,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/** send HS_DESC UPLOADED event
+ *
+ * called when we successfully uploaded a hidden service descriptor.
+ */
+void
+control_event_hs_descriptor_uploaded(const char *id_digest,
+                                     const char *onion_address)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
+                                         id_digest, NULL);
+}
+
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
+ */
+void
+control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
+                                  desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC event to inform controller that the query to
+ * <b>onion_address</b> failed to retrieve hidden service descriptor
+ * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
+ * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
+ * field. */
+void
+control_event_hsv3_descriptor_failed(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !reason)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC_CONTENT event after completion of a successful fetch
+ * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
+ * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
+ * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
+ * not trigger the control event. */
+void
+control_event_hs_descriptor_content(const char *onion_address,
+                                    const char *desc_id,
+                                    const char *hsdir_id_digest,
+                                    const char *content)
+{
+  static const char *event_name = "HS_DESC_CONTENT";
+  char *esc_content = NULL;
+
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
+    return;
+  }
+
+  if (content == NULL) {
+    /* Point it to empty content so it can still be escaped. */
+    content = "";
+  }
+  write_escaped_data(content, strlen(content), &esc_content);
+
+  send_control_event(EVENT_HS_DESC_CONTENT,
+                     "650+%s %s %s %s\r\n%s650 OK\r\n",
+                     event_name,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     desc_id,
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     esc_content);
+  tor_free(esc_content);
+}
+
+/** Send HS_DESC event to inform controller upload of hidden service
+ * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
+ * is not NULL, add it to REASON= field.
+ */
+void
+control_event_hs_descriptor_upload_failed(const char *id_digest,
+                                          const char *onion_address,
+                                          const char *reason)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+  control_event_hs_descriptor_upload_end("FAILED", onion_address,
+                                         id_digest, reason);
+}
+
+void
+control_events_free_all(void)
+{
+  smartlist_t *queued_events = NULL;
+
+  stats_prev_n_read = stats_prev_n_written = 0;
+
+  if (queued_control_events_lock) {
+    tor_mutex_acquire(queued_control_events_lock);
+    flush_queued_event_pending = 0;
+    queued_events = queued_control_events;
+    queued_control_events = NULL;
+    tor_mutex_release(queued_control_events_lock);
+  }
+  if (queued_events) {
+    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
+                      queued_event_free(ev));
+    smartlist_free(queued_events);
+  }
+  if (flush_queued_events_event) {
+    mainloop_event_free(flush_queued_events_event);
+    flush_queued_events_event = NULL;
+  }
+  global_event_mask = 0;
+  disable_log_messages = 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* For testing: change the value of global_event_mask */
+void
+control_testing_set_global_event_mask(uint64_t mask)
+{
+  global_event_mask = mask;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control_events.h b/src/feature/control/control_events.h
new file mode 100644
index 0000000000..34986fdb89
--- /dev/null
+++ b/src/feature/control/control_events.h
@@ -0,0 +1,352 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.h
+ * \brief Header file for control_events.c.
+ **/
+
+#ifndef TOR_CONTROL_EVENTS_H
+#define TOR_CONTROL_EVENTS_H
+
+#include "core/or/ocirc_event.h"
+
+/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
+ * The various types are defined in control-spec.txt . */
+typedef enum circuit_status_minor_event_t {
+  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+  CIRC_MINOR_EVENT_CANNIBALIZED,
+} circuit_status_minor_event_t;
+
+#include "core/or/orconn_event.h"
+
+/** Used to indicate the type of a stream event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum stream_status_event_t {
+  STREAM_EVENT_SENT_CONNECT = 0,
+  STREAM_EVENT_SENT_RESOLVE = 1,
+  STREAM_EVENT_SUCCEEDED    = 2,
+  STREAM_EVENT_FAILED       = 3,
+  STREAM_EVENT_CLOSED       = 4,
+  STREAM_EVENT_NEW          = 5,
+  STREAM_EVENT_NEW_RESOLVE  = 6,
+  STREAM_EVENT_FAILED_RETRIABLE = 7,
+  STREAM_EVENT_REMAP        = 8
+} stream_status_event_t;
+
+/** Used to indicate the type of a buildtime event */
+typedef enum buildtimeout_set_event_t {
+  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
+  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
+  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
+  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
+  BUILDTIMEOUT_SET_EVENT_RESUME = 4
+} buildtimeout_set_event_t;
+
+/** Enum describing various stages of bootstrapping, for use with controller
+ * bootstrap status events. The values range from 0 to 100. */
+typedef enum {
+  BOOTSTRAP_STATUS_UNDEF=-1,
+  BOOTSTRAP_STATUS_STARTING=0,
+
+  /* Initial connection to any relay */
+
+  BOOTSTRAP_STATUS_CONN_PT=1,
+  BOOTSTRAP_STATUS_CONN_DONE_PT=2,
+  BOOTSTRAP_STATUS_CONN_PROXY=3,
+  BOOTSTRAP_STATUS_CONN_DONE_PROXY=4,
+  BOOTSTRAP_STATUS_CONN=5,
+  BOOTSTRAP_STATUS_CONN_DONE=10,
+  BOOTSTRAP_STATUS_HANDSHAKE=14,
+  BOOTSTRAP_STATUS_HANDSHAKE_DONE=15,
+
+  /* Loading directory info */
+
+  BOOTSTRAP_STATUS_ONEHOP_CREATE=20,
+  BOOTSTRAP_STATUS_REQUESTING_STATUS=25,
+  BOOTSTRAP_STATUS_LOADING_STATUS=30,
+  BOOTSTRAP_STATUS_LOADING_KEYS=40,
+  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
+  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
+  BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75,
+
+  /* Connecting to a relay for AP circuits */
+
+  BOOTSTRAP_STATUS_AP_CONN_PT=76,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77,
+  BOOTSTRAP_STATUS_AP_CONN_PROXY=78,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79,
+  BOOTSTRAP_STATUS_AP_CONN=80,
+  BOOTSTRAP_STATUS_AP_CONN_DONE=85,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE=89,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90,
+
+  /* Creating AP circuits */
+
+  BOOTSTRAP_STATUS_CIRCUIT_CREATE=95,
+  BOOTSTRAP_STATUS_DONE=100
+} bootstrap_status_t;
+
+/** Reason for remapping an AP connection's address: we have a cached
+ * answer. */
+#define REMAP_STREAM_SOURCE_CACHE 1
+/** Reason for remapping an AP connection's address: the exit node told us an
+ * answer. */
+#define REMAP_STREAM_SOURCE_EXIT 2
+
+void control_initialize_event_queue(void);
+
+void control_update_global_event_mask(void);
+void control_adjust_event_log_severity(void);
+
+#define EVENT_NS 0x000F
+int control_event_is_interesting(int event);
+
+void control_per_second_events(void);
+int control_any_per_second_event_enabled(void);
+
+int control_event_circuit_status(origin_circuit_t *circ,
+                                 circuit_status_event_t e, int reason);
+int control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                          int old_purpose);
+int control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                       int old_purpose,
+                                       const struct timeval *old_tv_created);
+int control_event_stream_status(entry_connection_t *conn,
+                                stream_status_event_t e,
+                                int reason);
+int control_event_or_conn_status(or_connection_t *conn,
+                                 or_conn_status_event_t e, int reason);
+int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
+int control_event_stream_bandwidth(edge_connection_t *edge_conn);
+int control_event_stream_bandwidth_used(void);
+int control_event_circ_bandwidth_used(void);
+int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
+int control_event_conn_bandwidth(connection_t *conn);
+int control_event_conn_bandwidth_used(void);
+int control_event_circuit_cell_stats(void);
+void control_event_logmsg(int severity, log_domain_mask_t domain,
+                          const char *msg);
+void control_event_logmsg_pending(void);
+int control_event_descriptors_changed(smartlist_t *routers);
+int control_event_address_mapped(const char *from, const char *to,
+                                 time_t expires, const char *error,
+                                 const int cached);
+int control_event_my_descriptor_changed(void);
+int control_event_network_liveness_update(int liveness);
+int control_event_networkstatus_changed(smartlist_t *statuses);
+
+int control_event_newconsensus(const networkstatus_t *consensus);
+int control_event_networkstatus_changed_single(const routerstatus_t *rs);
+int control_event_general_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_client_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_server_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+int control_event_general_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_client_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_server_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+
+int control_event_guard(const char *nickname, const char *digest,
+                        const char *status);
+int control_event_conf_changed(const smartlist_t *elements);
+int control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                                   const char *args);
+int control_event_signal(uintptr_t signal);
+
+void control_event_bootstrap(bootstrap_status_t status, int progress);
+MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
+                                                 int reason,
+                                                 or_connection_t *or_conn));
+void control_event_boot_dir(bootstrap_status_t status, int progress);
+void control_event_boot_first_orconn(void);
+void control_event_bootstrap_problem(const char *warn, const char *reason,
+                                     const connection_t *conn, int dowarn);
+char *control_event_boot_last_msg(void);
+void control_event_bootstrap_reset(void);
+
+void control_event_clients_seen(const char *controller_str);
+void control_event_transport_launched(const char *mode,
+                                      const char *transport_name,
+                                      tor_addr_t *addr, uint16_t port);
+void control_event_pt_log(const char *log);
+void control_event_pt_status(const char *status);
+
+void control_event_hs_descriptor_requested(const char *onion_address,
+                                           rend_auth_type_t auth_type,
+                                           const char *id_digest,
+                                           const char *desc_id,
+                                           const char *hsdir_index);
+void control_event_hs_descriptor_created(const char *onion_address,
+                                         const char *desc_id,
+                                         int replica);
+void control_event_hs_descriptor_upload(const char *onion_address,
+                                        const char *desc_id,
+                                        const char *hs_dir,
+                                        const char *hsdir_index);
+void control_event_hs_descriptor_upload_end(const char *action,
+                                            const char *onion_address,
+                                            const char *hs_dir,
+                                            const char *reason);
+void control_event_hs_descriptor_uploaded(const char *hs_dir,
+                                          const char *onion_address);
+/* Hidden service v2 HS_DESC specific. */
+void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                          const char *id_digest,
+                                          const char *reason);
+void control_event_hsv2_descriptor_received(const char *onion_address,
+                                            const rend_data_t *rend_data,
+                                            const char *id_digest);
+/* Hidden service v3 HS_DESC specific. */
+void control_event_hsv3_descriptor_failed(const char *onion_address,
+                                          const char *desc_id,
+                                          const char *hsdir_id_digest,
+                                          const char *reason);
+void control_event_hsv3_descriptor_received(const char *onion_address,
+                                            const char *desc_id,
+                                            const char *hsdir_id_digest);
+void control_event_hs_descriptor_upload_failed(const char *hs_dir,
+                                               const char *onion_address,
+                                               const char *reason);
+void control_event_hs_descriptor_content(const char *onion_address,
+                                         const char *desc_id,
+                                         const char *hsdir_fp,
+                                         const char *content);
+
+void control_events_free_all(void);
+
+#ifdef CONTROL_MODULE_PRIVATE
+char *get_bw_samples(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#ifdef CONTROL_EVENTS_PRIVATE
+/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
+ * connection is interested in events of type <b>e</b>.  We use this
+ * so that we can decide to skip generating event messages that nobody
+ * has interest in without having to walk over the global connection
+ * list to find out.
+ **/
+typedef uint64_t event_mask_t;
+
+/* Recognized asynchronous event types.  It's okay to expand this list
+ * because it is used both as a list of v0 event types, and as indices
+ * into the bitfield to determine which controllers want which events.
+ */
+/* This bitfield has no event zero    0x0000 */
+#define EVENT_MIN_                    0x0001
+#define EVENT_CIRCUIT_STATUS          0x0001
+#define EVENT_STREAM_STATUS           0x0002
+#define EVENT_OR_CONN_STATUS          0x0003
+#define EVENT_BANDWIDTH_USED          0x0004
+#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
+#define EVENT_NEW_DESC                0x0006
+#define EVENT_DEBUG_MSG               0x0007
+#define EVENT_INFO_MSG                0x0008
+#define EVENT_NOTICE_MSG              0x0009
+#define EVENT_WARN_MSG                0x000A
+#define EVENT_ERR_MSG                 0x000B
+#define EVENT_ADDRMAP                 0x000C
+/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
+   can reclaim 0x000D. */
+#define EVENT_DESCCHANGED             0x000E
+/* Exposed above */
+// #define EVENT_NS                   0x000F
+#define EVENT_STATUS_CLIENT           0x0010
+#define EVENT_STATUS_SERVER           0x0011
+#define EVENT_STATUS_GENERAL          0x0012
+#define EVENT_GUARD                   0x0013
+#define EVENT_STREAM_BANDWIDTH_USED   0x0014
+#define EVENT_CLIENTS_SEEN            0x0015
+#define EVENT_NEWCONSENSUS            0x0016
+#define EVENT_BUILDTIMEOUT_SET        0x0017
+#define EVENT_GOT_SIGNAL              0x0018
+#define EVENT_CONF_CHANGED            0x0019
+#define EVENT_CONN_BW                 0x001A
+#define EVENT_CELL_STATS              0x001B
+/* UNUSED :                           0x001C */
+#define EVENT_CIRC_BANDWIDTH_USED     0x001D
+#define EVENT_TRANSPORT_LAUNCHED      0x0020
+#define EVENT_HS_DESC                 0x0021
+#define EVENT_HS_DESC_CONTENT         0x0022
+#define EVENT_NETWORK_LIVENESS        0x0023
+#define EVENT_PT_LOG                  0x0024
+#define EVENT_PT_STATUS               0x0025
+#define EVENT_MAX_                    0x0025
+
+/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
+#define EVENT_CAPACITY_               0x0040
+
+/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
+ * different structure, as it can only handle a maximum left shift of 1<<63. */
+
+#if EVENT_MAX_ >= EVENT_CAPACITY_
+#error control_connection_t.event_mask has an event greater than its capacity
+#endif
+
+#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
+
+#define EVENT_MASK_NONE_             ((uint64_t)0x0)
+
+#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
+#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
+                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
+                                          - EVENT_MIN_))
+
+#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
+                                      & EVENT_MASK_BELOW_MAX_)
+
+/** Helper structure: temporarily stores cell statistics for a circuit. */
+typedef struct cell_stats_t {
+  /** Number of cells added in app-ward direction by command. */
+  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells added in exit-ward direction by command. */
+  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in app-ward direction by command. */
+  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in exit-ward direction by command. */
+  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in app-ward direction by command. */
+  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in exit-ward direction by command. */
+  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
+} cell_stats_t;
+
+void sum_up_cell_stats_by_command(circuit_t *circ,
+                                  cell_stats_t *cell_stats);
+void append_cell_stats_by_command(smartlist_t *event_parts,
+                                  const char *key,
+                                  const uint64_t *include_if_non_zero,
+                                  const uint64_t *number_to_include);
+void format_cell_stats(char **event_string, circuit_t *circ,
+                       cell_stats_t *cell_stats);
+
+/** Helper structure: maps event values to their names. */
+struct control_event_t {
+  uint16_t event_code;
+  const char *event_name;
+};
+
+extern const struct control_event_t control_event_table[];
+
+#ifdef TOR_UNIT_TESTS
+MOCK_DECL(STATIC void,
+          send_control_event_string,(uint16_t event, const char *msg));
+
+MOCK_DECL(STATIC void,
+          queue_control_event_string,(uint16_t event, char *msg));
+
+void control_testing_set_global_event_mask(uint64_t mask);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(CONTROL_EVENTS_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_EVENTS_H) */
diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c
new file mode 100644
index 0000000000..e0e77eb2d0
--- /dev/null
+++ b/src/feature/control/control_fmt.c
@@ -0,0 +1,181 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+
+/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
+ * <b>buf</b>, determine the address:port combination requested on
+ * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
+ * failure. */
+int
+write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
+{
+  char buf2[256];
+  if (conn->chosen_exit_name)
+    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
+      return -1;
+  if (!conn->socks_request)
+    return -1;
+  if (tor_snprintf(buf, len, "%s%s%s:%d",
+               conn->socks_request->address,
+               conn->chosen_exit_name ? buf2 : "",
+               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
+                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
+               conn->socks_request->port)<0)
+    return -1;
+  return 0;
+}
+
+/** Figure out the best name for the target router of an OR connection
+ * <b>conn</b>, and write it into the <b>len</b>-character buffer
+ * <b>name</b>. */
+void
+orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
+{
+  const node_t *node = node_get_by_id(conn->identity_digest);
+  if (node) {
+    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
+    node_get_verbose_nickname(node, name);
+  } else if (! tor_digest_is_zero(conn->identity_digest)) {
+    name[0] = '$';
+    base16_encode(name+1, len-1, conn->identity_digest,
+                  DIGEST_LEN);
+  } else {
+    tor_snprintf(name, len, "%s:%d",
+                 conn->base_.address, conn->base_.port);
+  }
+}
+
+/** Allocate and return a description of <b>circ</b>'s current status,
+ * including its path (if any). */
+char *
+circuit_describe_status_for_controller(origin_circuit_t *circ)
+{
+  char *rv;
+  smartlist_t *descparts = smartlist_new();
+
+  {
+    char *vpath = circuit_list_path_for_controller(circ);
+    if (*vpath) {
+      smartlist_add(descparts, vpath);
+    } else {
+      tor_free(vpath); /* empty path; don't put an extra space in the result */
+    }
+  }
+
+  {
+    cpath_build_state_t *build_state = circ->build_state;
+    smartlist_t *flaglist = smartlist_new();
+    char *flaglist_joined;
+
+    if (build_state->onehop_tunnel)
+      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
+    if (build_state->is_internal)
+      smartlist_add(flaglist, (void *)"IS_INTERNAL");
+    if (build_state->need_capacity)
+      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
+    if (build_state->need_uptime)
+      smartlist_add(flaglist, (void *)"NEED_UPTIME");
+
+    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
+    if (smartlist_len(flaglist)) {
+      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
+
+      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
+
+      tor_free(flaglist_joined);
+    }
+
+    smartlist_free(flaglist);
+  }
+
+  smartlist_add_asprintf(descparts, "PURPOSE=%s",
+                    circuit_purpose_to_controller_string(circ->base_.purpose));
+
+  {
+    const char *hs_state =
+      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
+
+    if (hs_state != NULL) {
+      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
+    }
+  }
+
+  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
+    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    const char *onion_address;
+    if (circ->rend_data) {
+      onion_address = rend_data_get_address(circ->rend_data);
+    } else {
+      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
+      onion_address = addr;
+    }
+    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
+  }
+
+  {
+    char tbuf[ISO_TIME_USEC_LEN+1];
+    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
+
+    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
+  }
+
+  // Show username and/or password if available.
+  if (circ->socks_username_len > 0) {
+    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
+                                     (size_t) circ->socks_username_len);
+    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
+                           socks_username_escaped);
+    tor_free(socks_username_escaped);
+  }
+  if (circ->socks_password_len > 0) {
+    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
+                                     (size_t) circ->socks_password_len);
+    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
+                           socks_password_escaped);
+    tor_free(socks_password_escaped);
+  }
+
+  rv = smartlist_join_strings(descparts, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
+  smartlist_free(descparts);
+
+  return rv;
+}
+
+/** Return a longname the node whose identity is <b>id_digest</b>. If
+ * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
+ * returned instead.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+MOCK_IMPL(const char *,
+node_describe_longname_by_id,(const char *id_digest))
+{
+  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
+  node_get_verbose_nickname_by_id(id_digest, longname);
+  return longname;
+}
diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h
new file mode 100644
index 0000000000..6446e37079
--- /dev/null
+++ b/src/feature/control/control_fmt.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.h
+ * \brief Header file for control_fmt.c.
+ **/
+
+#ifndef TOR_CONTROL_FMT_H
+#define TOR_CONTROL_FMT_H
+
+int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
+                               size_t len);
+void orconn_target_get_name(char *buf, size_t len,
+                            or_connection_t *conn);
+char *circuit_describe_status_for_controller(origin_circuit_t *circ);
+
+MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
+
+#endif /* !defined(TOR_CONTROL_FMT_H) */
diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c
new file mode 100644
index 0000000000..3e31bb9e8f
--- /dev/null
+++ b/src/feature/control/control_getinfo.c
@@ -0,0 +1,1654 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_getinfo.c
+ * \brief Implementation for miscellaneous controller getinfo commands.
+ */
+
+#define CONTROL_EVENTS_PRIVATE
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/policies.h"
+#include "core/or/versions.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/control/fmt_serverstatus.h"
+#include "feature/control/getinfo_geoip.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendcache.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/version/torversion.h"
+
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifndef _WIN32
+#include <pwd.h>
+#endif
+
+static char *list_getinfo_options(void);
+static char *download_status_to_string(const download_status_t *dl);
+
+/** Implementation helper for GETINFO: knows the answers for various
+ * trivial-to-implement questions. */
+static int
+getinfo_helper_misc(control_connection_t *conn, const char *question,
+                    char **answer, const char **errmsg)
+{
+  (void) conn;
+  if (!strcmp(question, "version")) {
+    *answer = tor_strdup(get_version());
+  } else if (!strcmp(question, "bw-event-cache")) {
+    *answer = get_bw_samples();
+  } else if (!strcmp(question, "config-file")) {
+    const char *a = get_torrc_fname(0);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-defaults-file")) {
+    const char *a = get_torrc_fname(1);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-text")) {
+    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
+  } else if (!strcmp(question, "info/names")) {
+    *answer = list_getinfo_options();
+  } else if (!strcmp(question, "dormant")) {
+    int dormant = rep_hist_circbuilding_dormant(time(NULL));
+    *answer = tor_strdup(dormant ? "1" : "0");
+  } else if (!strcmp(question, "events/names")) {
+    int i;
+    smartlist_t *event_names = smartlist_new();
+
+    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+      smartlist_add(event_names, (char *)control_event_table[i].event_name);
+    }
+
+    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
+
+    smartlist_free(event_names);
+  } else if (!strcmp(question, "signal/names")) {
+    smartlist_t *signal_names = smartlist_new();
+    int j;
+    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
+      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
+    }
+
+    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
+
+    smartlist_free(signal_names);
+  } else if (!strcmp(question, "features/names")) {
+    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
+  } else if (!strcmp(question, "address")) {
+    uint32_t addr;
+    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
+      *errmsg = "Address unknown";
+      return -1;
+    }
+    *answer = tor_dup_ip(addr);
+  } else if (!strcmp(question, "traffic/read")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
+  } else if (!strcmp(question, "traffic/written")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
+  } else if (!strcmp(question, "uptime")) {
+    long uptime_secs = get_uptime();
+    tor_asprintf(answer, "%ld", uptime_secs);
+  } else if (!strcmp(question, "process/pid")) {
+    int myPid = -1;
+
+#ifdef _WIN32
+      myPid = _getpid();
+#else
+      myPid = getpid();
+#endif
+
+    tor_asprintf(answer, "%d", myPid);
+  } else if (!strcmp(question, "process/uid")) {
+#ifdef _WIN32
+      *answer = tor_strdup("-1");
+#else
+      int myUid = geteuid();
+      tor_asprintf(answer, "%d", myUid);
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/user")) {
+#ifdef _WIN32
+      *answer = tor_strdup("");
+#else
+      int myUid = geteuid();
+      const struct passwd *myPwEntry = tor_getpwuid(myUid);
+
+      if (myPwEntry) {
+        *answer = tor_strdup(myPwEntry->pw_name);
+      } else {
+        *answer = tor_strdup("");
+      }
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/descriptor-limit")) {
+    int max_fds = get_max_sockets();
+    tor_asprintf(answer, "%d", max_fds);
+  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
+    tor_asprintf(answer, "%"PRIu64,
+                 (get_options()->MaxMemInQueues));
+  } else if (!strcmp(question, "fingerprint")) {
+    crypto_pk_t *server_key;
+    if (!server_mode(get_options())) {
+      *errmsg = "Not running in server mode";
+      return -1;
+    }
+    server_key = get_server_identity_key();
+    *answer = tor_malloc(HEX_DIGEST_LEN+1);
+    crypto_pk_get_fingerprint(server_key, *answer, 0);
+  }
+  return 0;
+}
+
+/** Awful hack: return a newly allocated string based on a routerinfo and
+ * (possibly) an extrainfo, sticking the read-history and write-history from
+ * <b>ei</b> into the resulting string.  The thing you get back won't
+ * necessarily have a valid signature.
+ *
+ * New code should never use this; it's for backward compatibility.
+ *
+ * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
+ * not be NUL-terminated. */
+static char *
+munge_extrainfo_into_routerinfo(const char *ri_body,
+                                const signed_descriptor_t *ri,
+                                const signed_descriptor_t *ei)
+{
+  char *out = NULL, *outp;
+  int i;
+  const char *router_sig;
+  const char *ei_body = signed_descriptor_get_body(ei);
+  size_t ri_len = ri->signed_descriptor_len;
+  size_t ei_len = ei->signed_descriptor_len;
+  if (!ei_body)
+    goto bail;
+
+  outp = out = tor_malloc(ri_len+ei_len+1);
+  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
+    goto bail;
+  ++router_sig;
+  memcpy(out, ri_body, router_sig-ri_body);
+  outp += router_sig-ri_body;
+
+  for (i=0; i < 2; ++i) {
+    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
+    const char *cp, *eol;
+    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
+      continue;
+    ++cp;
+    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
+      continue;
+    memcpy(outp, cp, eol-cp+1);
+    outp += eol-cp+1;
+  }
+  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
+  *outp++ = '\0';
+  tor_assert(outp-out < (int)(ri_len+ei_len+1));
+
+  return out;
+ bail:
+  tor_free(out);
+  return tor_strndup(ri_body, ri->signed_descriptor_len);
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * which ports are bound. */
+static int
+getinfo_helper_listeners(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  int type;
+  smartlist_t *res;
+
+  (void)control_conn;
+  (void)errmsg;
+
+  if (!strcmp(question, "net/listeners/or"))
+    type = CONN_TYPE_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/extor"))
+    type = CONN_TYPE_EXT_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/dir"))
+    type = CONN_TYPE_DIR_LISTENER;
+  else if (!strcmp(question, "net/listeners/socks"))
+    type = CONN_TYPE_AP_LISTENER;
+  else if (!strcmp(question, "net/listeners/trans"))
+    type = CONN_TYPE_AP_TRANS_LISTENER;
+  else if (!strcmp(question, "net/listeners/natd"))
+    type = CONN_TYPE_AP_NATD_LISTENER;
+  else if (!strcmp(question, "net/listeners/httptunnel"))
+    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  else if (!strcmp(question, "net/listeners/dns"))
+    type = CONN_TYPE_AP_DNS_LISTENER;
+  else if (!strcmp(question, "net/listeners/control"))
+    type = CONN_TYPE_CONTROL_LISTENER;
+  else
+    return 0; /* unknown key */
+
+  res = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    struct sockaddr_storage ss;
+    socklen_t ss_len = sizeof(ss);
+
+    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
+      continue;
+
+    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
+      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
+    } else {
+      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
+      smartlist_add(res, esc_for_log(tmp));
+      tor_free(tmp);
+    }
+
+  } SMARTLIST_FOREACH_END(conn);
+
+  *answer = smartlist_join_strings(res, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
+  smartlist_free(res);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * the current time in both local and UTC forms. */
+STATIC int
+getinfo_helper_current_time(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  if (!strcmp(question, "current-time/local"))
+    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else if (!strcmp(question, "current-time/utc"))
+    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else
+    return 0;
+
+  *answer = tor_strdup(timebuf);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * directory information. */
+STATIC int
+getinfo_helper_dir(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  (void) control_conn;
+  if (!strcmpstart(question, "desc/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/id/* "
+                "instead of desc/id/*.";
+      return 0;
+    }
+  } else if (!strcmpstart(question, "desc/name/")) {
+    const routerinfo_t *ri = NULL;
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node =
+      node_get_by_nickname(question+strlen("desc/name/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/name/* "
+                "instead of desc/name/*.";
+      return 0;
+    }
+  } else if (!strcmp(question, "desc/download-enabled")) {
+    int r = we_fetch_router_descriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmp(question, "desc/all-recent")) {
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
+      {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        if (body)
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+      });
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
+    /* XXXX Remove this once Torstat asks for extrainfos. */
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+        if (ei && body) {
+          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
+                                                        &ri->cache_info, ei));
+        } else if (body) {
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+        }
+      } SMARTLIST_FOREACH_END(ri);
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/client/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      rend_cache_entry_t *e = NULL;
+      if (!rend_cache_lookup_entry(question, -1, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      const char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_cache_lookup_encoded_as_client(&service_pk);
+      if (desc) {
+        *answer = tor_strdup(desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/service/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+    rend_cache_entry_t *e = NULL;
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_service_lookup_current_desc(&service_pk);
+      if (desc) {
+        /* Newly allocated string, we have ownership. */
+        *answer = desc;
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "md/all")) {
+    const smartlist_t *nodes = nodelist_get_list();
+    tor_assert(nodes);
+
+    if (smartlist_len(nodes) == 0) {
+      *answer = tor_strdup("");
+      return 0;
+    }
+
+    smartlist_t *microdescs = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
+      if (n->md && n->md->body) {
+        char *copy = tor_strndup(n->md->body, n->md->bodylen);
+        smartlist_add(microdescs, copy);
+      }
+    } SMARTLIST_FOREACH_END(n);
+
+    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
+    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
+    smartlist_free(microdescs);
+  } else if (!strcmpstart(question, "md/id/")) {
+    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmpstart(question, "md/name/")) {
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
+    /* XXXX duplicated code */
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmp(question, "md/download-enabled")) {
+    int r = we_fetch_microdescriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmpstart(question, "desc-annotations/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node =
+      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *annotations =
+        signed_descriptor_get_annotations(&ri->cache_info);
+      if (annotations)
+        *answer = tor_strndup(annotations,
+                              ri->cache_info.annotations_len);
+    }
+  } else if (!strcmpstart(question, "dir/server/")) {
+    size_t answer_len = 0;
+    char *url = NULL;
+    smartlist_t *descs = smartlist_new();
+    const char *msg;
+    int res;
+    char *cp;
+    tor_asprintf(&url, "/tor/%s", question+4);
+    res = dirserv_get_routerdescs(descs, url, &msg);
+    if (res) {
+      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
+      smartlist_free(descs);
+      tor_free(url);
+      *errmsg = msg;
+      return -1;
+    }
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      answer_len += sd->signed_descriptor_len);
+    cp = *answer = tor_malloc(answer_len+1);
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      {
+                        memcpy(cp, signed_descriptor_get_body(sd),
+                               sd->signed_descriptor_len);
+                        cp += sd->signed_descriptor_len;
+                      });
+    *cp = '\0';
+    tor_free(url);
+    smartlist_free(descs);
+  } else if (!strcmpstart(question, "dir/status/")) {
+    *answer = tor_strdup("");
+  } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
+    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
+      const cached_dir_t *consensus = dirserv_get_consensus("ns");
+      if (consensus)
+        *answer = tor_strdup(consensus->dir);
+    }
+    if (!*answer) { /* try loading it from disk */
+      tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns");
+      if (mapped) {
+        *answer = tor_memdup_nulterm(mapped->data, mapped->size);
+        tor_munmap_file(mapped);
+      }
+      if (!*answer) { /* generate an error */
+        *errmsg = "Could not open cached consensus. "
+          "Make sure FetchUselessDescriptors is set to 1.";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
+    routerlist_t *routerlist = router_get_routerlist();
+    if (!routerlist || !routerlist->routers ||
+        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
+      return -1;
+    }
+  } else if (!strcmpstart(question, "extra-info/digest/")) {
+    question += strlen("extra-info/digest/");
+    if (strlen(question) == HEX_DIGEST_LEN) {
+      char d[DIGEST_LEN];
+      signed_descriptor_t *sd = NULL;
+      if (base16_decode(d, sizeof(d), question, strlen(question))
+                        == sizeof(d)) {
+        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
+         * but I don't want to risk affecting other parts of the code,
+         * especially since the rules for using our own extrainfo (including
+         * when it might be freed) are different from those for using one
+         * we have downloaded. */
+        if (router_extrainfo_digest_is_me(d))
+          sd = &(router_get_my_extrainfo()->cache_info);
+        else
+          sd = extrainfo_get_by_descriptor_digest(d);
+      }
+      if (sd) {
+        const char *body = signed_descriptor_get_body(sd);
+        if (body)
+          *answer = tor_strndup(body, sd->signed_descriptor_len);
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** Given a smartlist of 20-byte digests, return a newly allocated string
+ * containing each of those digests in order, formatted in HEX, and terminated
+ * with a newline. */
+static char *
+digest_list_to_string(const smartlist_t *sl)
+{
+  int len;
+  char *result, *s;
+
+  /* Allow for newlines, and a \0 at the end */
+  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
+  result = tor_malloc_zero(len);
+
+  s = result;
+  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
+    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
+    s[HEX_DIGEST_LEN] = '\n';
+    s += HEX_DIGEST_LEN + 1;
+  } SMARTLIST_FOREACH_END(digest);
+  *s = '\0';
+
+  return result;
+}
+
+/** Turn a download_status_t into a human-readable description in a newly
+ * allocated string.  The format is specified in control-spec.txt, under
+ * the documentation for "GETINFO download/..." .  */
+static char *
+download_status_to_string(const download_status_t *dl)
+{
+  char *rv = NULL;
+  char tbuf[ISO_TIME_LEN+1];
+  const char *schedule_str, *want_authority_str;
+  const char *increment_on_str, *backoff_str;
+
+  if (dl) {
+    /* Get some substrings of the eventual output ready */
+    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
+
+    switch (dl->schedule) {
+      case DL_SCHED_GENERIC:
+        schedule_str = "DL_SCHED_GENERIC";
+        break;
+      case DL_SCHED_CONSENSUS:
+        schedule_str = "DL_SCHED_CONSENSUS";
+        break;
+      case DL_SCHED_BRIDGE:
+        schedule_str = "DL_SCHED_BRIDGE";
+        break;
+      default:
+        schedule_str = "unknown";
+        break;
+    }
+
+    switch (dl->want_authority) {
+      case DL_WANT_ANY_DIRSERVER:
+        want_authority_str = "DL_WANT_ANY_DIRSERVER";
+        break;
+      case DL_WANT_AUTHORITY:
+        want_authority_str = "DL_WANT_AUTHORITY";
+        break;
+      default:
+        want_authority_str = "unknown";
+        break;
+    }
+
+    switch (dl->increment_on) {
+      case DL_SCHED_INCREMENT_FAILURE:
+        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
+        break;
+      case DL_SCHED_INCREMENT_ATTEMPT:
+        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
+        break;
+      default:
+        increment_on_str = "unknown";
+        break;
+    }
+
+    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
+
+    /* Now assemble them */
+    tor_asprintf(&rv,
+                 "next-attempt-at %s\n"
+                 "n-download-failures %u\n"
+                 "n-download-attempts %u\n"
+                 "schedule %s\n"
+                 "want-authority %s\n"
+                 "increment-on %s\n"
+                 "backoff %s\n"
+                 "last-backoff-position %u\n"
+                 "last-delay-used %d\n",
+                 tbuf,
+                 dl->n_download_failures,
+                 dl->n_download_attempts,
+                 schedule_str,
+                 want_authority_str,
+                 increment_on_str,
+                 backoff_str,
+                 dl->last_backoff_position,
+                 dl->last_delay_used);
+  }
+
+  return rv;
+}
+
+/** Handle the consensus download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_networkstatus(const char *flavor,
+                                       download_status_t **dl_to_emit,
+                                       const char **errmsg)
+{
+  /*
+   * We get the one for the current bootstrapped status by default, or
+   * take an extra /bootstrap or /running suffix
+   */
+  if (strcmp(flavor, "ns") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
+  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
+  } else if (strcmp(flavor, "ns/running") == 0 ) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
+  } else if (strcmp(flavor, "microdesc") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/running") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
+  } else {
+    *errmsg = "Unknown flavor";
+  }
+}
+
+/** Handle the cert download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_cert(const char *fp_sk_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  const char *sk_req;
+  char id_digest[DIGEST_LEN];
+  char sk_digest[DIGEST_LEN];
+
+  /*
+   * We have to handle four cases; fp_sk_req is the request with
+   * a prefix of "downloads/cert/" snipped off.
+   *
+   * Case 1: fp_sk_req = "fps"
+   *  - We should emit a digest_list with a list of all the identity
+   *    fingerprints that can be queried for certificate download status;
+   *    get it by calling list_authority_ids_with_downloads().
+   *
+   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
+   *  - We want the default certificate for this identity fingerprint's
+   *    download status; this is the download we get from URLs starting
+   *    in /fp/ on the directory server.  We can get it with
+   *    id_only_download_status_for_authority_id().
+   *
+   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
+   *  - We want a list of all signing key digests for this identity
+   *    fingerprint which can be queried for certificate download status.
+   *    Get it with list_sk_digests_for_authority_id().
+   *
+   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
+   *         signing key digest sk
+   *   - We want the download status for the certificate for this specific
+   *     signing key and fingerprint.  These correspond to the ones we get
+   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
+   *     list_sk_digests_for_authority_id().
+   */
+
+  if (strcmp(fp_sk_req, "fps") == 0) {
+    *digest_list = list_authority_ids_with_downloads();
+    if (!(*digest_list)) {
+      *errmsg = "Failed to get list of authority identity digests (!)";
+    }
+  } else if (!strcmpstart(fp_sk_req, "fp/")) {
+    fp_sk_req += strlen("fp/");
+    /* Okay, look for another / to tell the fp from fp-sk cases */
+    sk_req = strchr(fp_sk_req, '/');
+    if (sk_req) {
+      /* okay, split it here and try to parse <fp> */
+      if (base16_decode(id_digest, DIGEST_LEN,
+                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
+        /* Skip past the '/' */
+        ++sk_req;
+        if (strcmp(sk_req, "sks") == 0) {
+          /* We're asking for the list of signing key fingerprints */
+          *digest_list = list_sk_digests_for_authority_id(id_digest);
+          if (!(*digest_list)) {
+            *errmsg = "Failed to get list of signing key digests for this "
+                      "authority identity digest";
+          }
+        } else {
+          /* We've got a signing key digest */
+          if (base16_decode(sk_digest, DIGEST_LEN,
+                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
+            *dl_to_emit =
+              download_status_for_authority_id_and_sk(id_digest, sk_digest);
+            if (!(*dl_to_emit)) {
+              *errmsg = "Failed to get download status for this identity/"
+                        "signing key digest pair";
+            }
+          } else {
+            *errmsg = "That didn't look like a signing key digest";
+          }
+        }
+      } else {
+        *errmsg = "That didn't look like an identity digest";
+      }
+    } else {
+      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
+      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
+        if (base16_decode(id_digest, DIGEST_LEN,
+                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
+          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
+          if (!(*dl_to_emit)) {
+            *errmsg = "Failed to get download status for this authority "
+                      "identity digest";
+          }
+        } else {
+          *errmsg = "That didn't look like a digest";
+        }
+      } else {
+        *errmsg = "That didn't look like a digest";
+      }
+    }
+  } else {
+    *errmsg = "Unknown certificate download status query";
+  }
+}
+
+/** Handle the routerdesc download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_desc(const char *desc_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  char desc_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: desc_req = "descs"
+   *   - Emit a list of all router descriptor digests, which we get by
+   *     calling router_get_descriptor_digests(); this can return NULL
+   *     if we have no current ns-flavor consensus.
+   *
+   * Case 2: desc_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using router_get_dl_status_by_descriptor_digest().
+   */
+
+  if (strcmp(desc_req, "descs") == 0) {
+    *digest_list = router_get_descriptor_digests();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
+    }
+    /*
+     * Microdescs don't use the download_status_t mechanism, so we don't
+     * answer queries about their downloads here; see microdesc.c.
+     */
+  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(desc_digest, DIGEST_LEN,
+                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such descriptor digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown router descriptor download status query";
+  }
+}
+
+/** Handle the bridge download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_bridge(const char *bridge_req,
+                                download_status_t **dl_to_emit,
+                                smartlist_t **digest_list,
+                                const char **errmsg)
+{
+  char bridge_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: bridge_req = "bridges"
+   *   - Emit a list of all bridge identity digests, which we get by
+   *     calling list_bridge_identities(); this can return NULL if we are
+   *     not using bridges.
+   *
+   * Case 2: bridge_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using get_bridge_dl_status_by_id().
+   */
+
+  if (strcmp(bridge_req, "bridges") == 0) {
+    *digest_list = list_bridge_identities();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to be using bridges";
+    }
+  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(bridge_digest, DIGEST_LEN,
+                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such bridge identity digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown bridge descriptor download status query";
+  }
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * download status information. */
+STATIC int
+getinfo_helper_downloads(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  download_status_t *dl_to_emit = NULL;
+  smartlist_t *digest_list = NULL;
+
+  /* Assert args are sane */
+  tor_assert(control_conn != NULL);
+  tor_assert(question != NULL);
+  tor_assert(answer != NULL);
+  tor_assert(errmsg != NULL);
+
+  /* We check for this later to see if we should supply a default */
+  *errmsg = NULL;
+
+  /* Are we after networkstatus downloads? */
+  if (!strcmpstart(question, "downloads/networkstatus/")) {
+    getinfo_helper_downloads_networkstatus(
+        question + strlen("downloads/networkstatus/"),
+        &dl_to_emit, errmsg);
+  /* Certificates? */
+  } else if (!strcmpstart(question, "downloads/cert/")) {
+    getinfo_helper_downloads_cert(
+        question + strlen("downloads/cert/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Router descriptors? */
+  } else if (!strcmpstart(question, "downloads/desc/")) {
+    getinfo_helper_downloads_desc(
+        question + strlen("downloads/desc/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Bridge descriptors? */
+  } else if (!strcmpstart(question, "downloads/bridge/")) {
+    getinfo_helper_downloads_bridge(
+        question + strlen("downloads/bridge/"),
+        &dl_to_emit, &digest_list, errmsg);
+  } else {
+    *errmsg = "Unknown download status query";
+  }
+
+  if (dl_to_emit) {
+    *answer = download_status_to_string(dl_to_emit);
+
+    return 0;
+  } else if (digest_list) {
+    *answer = digest_list_to_string(digest_list);
+    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
+    smartlist_free(digest_list);
+
+    return 0;
+  } else {
+    if (!(*errmsg)) {
+      *errmsg = "Unknown error";
+    }
+
+    return -1;
+  }
+}
+
+/** Implementation helper for GETINFO: knows how to generate summaries of the
+ * current states of things we send events about. */
+static int
+getinfo_helper_events(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  const or_options_t *options = get_options();
+  (void) control_conn;
+  if (!strcmp(question, "circuit-status")) {
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
+      origin_circuit_t *circ;
+      char *circdesc;
+      const char *state;
+      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
+        continue;
+      circ = TO_ORIGIN_CIRCUIT(circ_);
+
+      if (circ->base_.state == CIRCUIT_STATE_OPEN)
+        state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
+      else if (circ->cpath)
+        state = "EXTENDED";
+      else
+        state = "LAUNCHED";
+
+      circdesc = circuit_describe_status_for_controller(circ);
+
+      smartlist_add_asprintf(status, "%lu %s%s%s",
+                   (unsigned long)circ->global_identifier,
+                   state, *circdesc ? " " : "", circdesc);
+      tor_free(circdesc);
+    }
+    SMARTLIST_FOREACH_END(circ_);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "stream-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    char buf[256];
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      entry_connection_t *conn;
+      circuit_t *circ;
+      origin_circuit_t *origin_circ = NULL;
+      if (base_conn->type != CONN_TYPE_AP ||
+          base_conn->marked_for_close ||
+          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
+          base_conn->state == AP_CONN_STATE_NATD_WAIT)
+        continue;
+      conn = TO_ENTRY_CONN(base_conn);
+      switch (base_conn->state)
+        {
+        case AP_CONN_STATE_CONTROLLER_WAIT:
+        case AP_CONN_STATE_CIRCUIT_WAIT:
+          if (conn->socks_request &&
+              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
+            state = "NEWRESOLVE";
+          else
+            state = "NEW";
+          break;
+        case AP_CONN_STATE_RENDDESC_WAIT:
+        case AP_CONN_STATE_CONNECT_WAIT:
+          state = "SENTCONNECT"; break;
+        case AP_CONN_STATE_RESOLVE_WAIT:
+          state = "SENTRESOLVE"; break;
+        case AP_CONN_STATE_OPEN:
+          state = "SUCCEEDED"; break;
+        default:
+          log_warn(LD_BUG, "Asked for stream in unknown state %d",
+                   base_conn->state);
+          continue;
+        }
+      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+      if (circ && CIRCUIT_IS_ORIGIN(circ))
+        origin_circ = TO_ORIGIN_CIRCUIT(circ);
+      write_stream_target_to_buf(conn, buf, sizeof(buf));
+      smartlist_add_asprintf(status, "%lu %s %lu %s",
+                   (unsigned long) base_conn->global_identifier,state,
+                   origin_circ?
+                         (unsigned long)origin_circ->global_identifier : 0ul,
+                   buf);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "orconn-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      char name[128];
+      or_connection_t *conn;
+      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
+        continue;
+      conn = TO_OR_CONN(base_conn);
+      if (conn->base_.state == OR_CONN_STATE_OPEN)
+        state = "CONNECTED";
+      else if (conn->nickname)
+        state = "LAUNCHED";
+      else
+        state = "NEW";
+      orconn_target_get_name(name, sizeof(name), conn);
+      smartlist_add_asprintf(status, "%s %s", name, state);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmpstart(question, "address-mappings/")) {
+    time_t min_e, max_e;
+    smartlist_t *mappings;
+    question += strlen("address-mappings/");
+    if (!strcmp(question, "all")) {
+      min_e = 0; max_e = TIME_MAX;
+    } else if (!strcmp(question, "cache")) {
+      min_e = 2; max_e = TIME_MAX;
+    } else if (!strcmp(question, "config")) {
+      min_e = 0; max_e = 0;
+    } else if (!strcmp(question, "control")) {
+      min_e = 1; max_e = 1;
+    } else {
+      return 0;
+    }
+    mappings = smartlist_new();
+    addressmap_get_mappings(mappings, min_e, max_e, 1);
+    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
+    smartlist_free(mappings);
+  } else if (!strcmpstart(question, "status/")) {
+    /* Note that status/ is not a catch-all for events; there's only supposed
+     * to be a status GETINFO if there's a corresponding STATUS event. */
+    if (!strcmp(question, "status/circuit-established")) {
+      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
+    } else if (!strcmp(question, "status/enough-dir-info")) {
+      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
+    } else if (!strcmp(question, "status/good-server-descriptor") ||
+               !strcmp(question, "status/accepted-server-descriptor")) {
+      /* They're equivalent for now, until we can figure out how to make
+       * good-server-descriptor be what we want. See comment in
+       * control-spec.txt. */
+      *answer = tor_strdup(directories_have_accepted_server_descriptor()
+                           ? "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
+      *answer = tor_strdup(check_whether_orport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
+      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded")) {
+      tor_asprintf(answer, "OR=%d DIR=%d",
+                   check_whether_orport_reachable(options) ? 1 : 0,
+                   check_whether_dirport_reachable(options) ? 1 : 0);
+    } else if (!strcmp(question, "status/bootstrap-phase")) {
+      *answer = control_event_boot_last_msg();
+    } else if (!strcmpstart(question, "status/version/")) {
+      int is_server = server_mode(options);
+      networkstatus_t *c = networkstatus_get_latest_consensus();
+      version_status_t status;
+      const char *recommended;
+      if (c) {
+        recommended = is_server ? c->server_versions : c->client_versions;
+        status = tor_version_is_obsolete(VERSION, recommended);
+      } else {
+        recommended = "?";
+        status = VS_UNKNOWN;
+      }
+
+      if (!strcmp(question, "status/version/recommended")) {
+        *answer = tor_strdup(recommended);
+        return 0;
+      }
+      if (!strcmp(question, "status/version/current")) {
+        switch (status)
+          {
+          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
+          case VS_OLD: *answer = tor_strdup("obsolete"); break;
+          case VS_NEW: *answer = tor_strdup("new"); break;
+          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
+          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
+          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
+          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
+          default: tor_fragile_assert();
+          }
+      }
+    } else if (!strcmp(question, "status/clients-seen")) {
+      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
+      if (!bridge_stats) {
+        *errmsg = "No bridge-client stats available";
+        return -1;
+      }
+      *answer = bridge_stats;
+    } else if (!strcmp(question, "status/fresh-relay-descs")) {
+      if (!server_mode(options)) {
+        *errmsg = "Only relays have descriptors";
+        return -1;
+      }
+      routerinfo_t *r;
+      extrainfo_t *e;
+      if (router_build_fresh_descriptor(&r, &e) < 0) {
+        *errmsg = "Error generating descriptor";
+        return -1;
+      }
+      size_t size = r->cache_info.signed_descriptor_len + 1;
+      if (e) {
+        size += e->cache_info.signed_descriptor_len + 1;
+      }
+      tor_assert(r->cache_info.signed_descriptor_len);
+      char *descs = tor_malloc(size);
+      char *cp = descs;
+      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
+             r->cache_info.signed_descriptor_len);
+      cp += r->cache_info.signed_descriptor_len - 1;
+      if (e) {
+        if (cp[0] == '\0') {
+          cp[0] = '\n';
+        } else if (cp[0] != '\n') {
+          cp[1] = '\n';
+          cp++;
+        }
+        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
+               e->cache_info.signed_descriptor_len);
+        cp += e->cache_info.signed_descriptor_len - 1;
+      }
+      if (cp[0] == '\n') {
+        cp[0] = '\0';
+      } else if (cp[0] != '\0') {
+        cp[1] = '\0';
+      }
+      *answer = descs;
+      routerinfo_free(r);
+      extrainfo_free(e);
+    } else {
+      return 0;
+    }
+  }
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows how to enumerate hidden services
+ * created via the control port. */
+STATIC int
+getinfo_helper_onions(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
+
+  if (control_conn && !strcmp(question, "onions/current")) {
+    onion_list = control_conn->ephemeral_onion_services;
+  } else if (!strcmp(question, "onions/detached")) {
+    onion_list = get_detached_onion_services();
+  } else {
+    return 0;
+  }
+  if (!onion_list || smartlist_len(onion_list) == 0) {
+    if (answer) {
+      *answer = tor_strdup("");
+    }
+  } else {
+    if (answer) {
+      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about network
+ * liveness. */
+static int
+getinfo_helper_liveness(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+  if (strcmp(question, "network-liveness") == 0) {
+    if (get_cached_network_liveness()) {
+      *answer = tor_strdup("up");
+    } else {
+      *answer = tor_strdup("down");
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
+/** Callback function for GETINFO: on a given control connection, try to
+ * answer the question <b>q</b> and store the newly-allocated answer in
+ * *<b>a</b>. If an internal error occurs, return -1 and optionally set
+ * *<b>error_out</b> to point to an error message to be delivered to the
+ * controller. On success, _or if the key is not recognized_, return 0. Do not
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
+ */
+typedef int (*getinfo_helper_t)(control_connection_t *,
+                                const char *q, char **a,
+                                const char **error_out);
+
+/** A single item for the GETINFO question-to-answer-function table. */
+typedef struct getinfo_item_t {
+  const char *varname; /**< The value (or prefix) of the question. */
+  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
+                        * this entry is documentation-only. */
+  const char *desc; /**< Description of the variable. */
+  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
+} getinfo_item_t;
+
+#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
+#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
+#define DOC(name, desc) { name, NULL, desc, 0 }
+
+/** Table mapping questions accepted by GETINFO to the functions that know how
+ * to answer them. */
+static const getinfo_item_t getinfo_items[] = {
+  ITEM("version", misc, "The current version of Tor."),
+  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
+  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
+  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
+  ITEM("config-text", misc,
+       "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
+  ITEM("accounting/bytes", accounting,
+       "Number of bytes read/written so far in the accounting interval."),
+  ITEM("accounting/bytes-left", accounting,
+      "Number of bytes left to write/read so far in the accounting interval."),
+  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
+  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
+  ITEM("accounting/interval-start", accounting,
+       "Time when the accounting period starts."),
+  ITEM("accounting/interval-end", accounting,
+       "Time when the accounting period ends."),
+  ITEM("accounting/interval-wake", accounting,
+       "Time to wake up in this accounting period."),
+  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
+  ITEM("entry-guards", entry_guards,
+       "Which nodes are we using as entry guards?"),
+  ITEM("fingerprint", misc, NULL),
+  PREFIX("config/", config, "Current configuration values."),
+  DOC("config/names",
+      "List of configuration options, types, and documentation."),
+  DOC("config/defaults",
+      "List of default values for configuration options. "
+      "See also config/names"),
+  PREFIX("current-time/", current_time, "Current time."),
+  DOC("current-time/local", "Current time on the local system."),
+  DOC("current-time/utc", "Current UTC time."),
+  PREFIX("downloads/networkstatus/", downloads,
+         "Download statuses for networkstatus objects"),
+  DOC("downloads/networkstatus/ns",
+      "Download status for current-mode networkstatus download"),
+  DOC("downloads/networkstatus/ns/bootstrap",
+      "Download status for bootstrap-time networkstatus download"),
+  DOC("downloads/networkstatus/ns/running",
+      "Download status for run-time networkstatus download"),
+  DOC("downloads/networkstatus/microdesc",
+      "Download status for current-mode microdesc download"),
+  DOC("downloads/networkstatus/microdesc/bootstrap",
+      "Download status for bootstrap-time microdesc download"),
+  DOC("downloads/networkstatus/microdesc/running",
+      "Download status for run-time microdesc download"),
+  PREFIX("downloads/cert/", downloads,
+         "Download statuses for certificates, by id fingerprint and "
+         "signing key"),
+  DOC("downloads/cert/fps",
+      "List of authority fingerprints for which any download statuses "
+      "exist"),
+  DOC("downloads/cert/fp/<fp>",
+      "Download status for <fp> with the default signing key; corresponds "
+      "to /fp/ URLs on directory server."),
+  DOC("downloads/cert/fp/<fp>/sks",
+      "List of signing keys for which specific download statuses are "
+      "available for this id fingerprint"),
+  DOC("downloads/cert/fp/<fp>/<sk>",
+      "Download status for <fp> with signing key <sk>; corresponds "
+      "to /fp-sk/ URLs on directory server."),
+  PREFIX("downloads/desc/", downloads,
+         "Download statuses for router descriptors, by descriptor digest"),
+  DOC("downloads/desc/descs",
+      "Return a list of known router descriptor digests"),
+  DOC("downloads/desc/<desc>",
+      "Return a download status for a given descriptor digest"),
+  PREFIX("downloads/bridge/", downloads,
+         "Download statuses for bridge descriptors, by bridge identity "
+         "digest"),
+  DOC("downloads/bridge/bridges",
+      "Return a list of configured bridge identity digests with download "
+      "statuses"),
+  DOC("downloads/bridge/<desc>",
+      "Return a download status for a given bridge identity digest"),
+  ITEM("info/names", misc,
+       "List of GETINFO options, types, and documentation."),
+  ITEM("events/names", misc,
+       "Events that the controller can ask for with SETEVENTS."),
+  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
+  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
+  PREFIX("desc/id/", dir, "Router descriptors by ID."),
+  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
+  ITEM("desc/all-recent", dir,
+       "All non-expired, non-superseded router descriptors."),
+  ITEM("desc/download-enabled", dir,
+       "Do we try to download router descriptors?"),
+  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
+  ITEM("md/all", dir, "All known microdescriptors."),
+  PREFIX("md/id/", dir, "Microdescriptors by ID"),
+  PREFIX("md/name/", dir, "Microdescriptors by name"),
+  ITEM("md/download-enabled", dir,
+       "Do we try to download microdescriptors?"),
+  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
+  PREFIX("hs/client/desc/id", dir,
+         "Hidden Service descriptor in client's cache by onion."),
+  PREFIX("hs/service/desc/id/", dir,
+         "Hidden Service descriptor in services's cache by onion."),
+  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
+  ITEM("ns/all", networkstatus,
+       "Brief summary of router status (v2 directory format)"),
+  PREFIX("ns/id/", networkstatus,
+         "Brief summary of router status by ID (v2 directory format)."),
+  PREFIX("ns/name/", networkstatus,
+         "Brief summary of router status by nickname (v2 directory format)."),
+  PREFIX("ns/purpose/", networkstatus,
+         "Brief summary of router status by purpose (v2 directory format)."),
+  PREFIX("consensus/", networkstatus,
+         "Information about and from the ns consensus."),
+  ITEM("network-status", dir,
+       "Brief summary of router status (v1 directory format)"),
+  ITEM("network-liveness", liveness,
+       "Current opinion on whether the network is live"),
+  ITEM("circuit-status", events, "List of current circuits originating here."),
+  ITEM("stream-status", events,"List of current streams."),
+  ITEM("orconn-status", events, "A list of current OR connections."),
+  ITEM("dormant", misc,
+       "Is Tor dormant (not building circuits because it's idle)?"),
+  PREFIX("address-mappings/", events, NULL),
+  DOC("address-mappings/all", "Current address mappings."),
+  DOC("address-mappings/cache", "Current cached DNS replies."),
+  DOC("address-mappings/config",
+      "Current address mappings from configuration."),
+  DOC("address-mappings/control", "Current address mappings from controller."),
+  PREFIX("status/", events, NULL),
+  DOC("status/circuit-established",
+      "Whether we think client functionality is working."),
+  DOC("status/enough-dir-info",
+      "Whether we have enough up-to-date directory information to build "
+      "circuits."),
+  DOC("status/bootstrap-phase",
+      "The last bootstrap phase status event that Tor sent."),
+  DOC("status/clients-seen",
+      "Breakdown of client countries seen by a bridge."),
+  DOC("status/fresh-relay-descs",
+      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
+  DOC("status/version/recommended", "List of currently recommended versions."),
+  DOC("status/version/current", "Status of the current version."),
+  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
+  ITEM("traffic/read", misc,"Bytes read since the process was started."),
+  ITEM("traffic/written", misc,
+       "Bytes written since the process was started."),
+  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
+  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
+  ITEM("process/uid", misc, "User id running the tor process."),
+  ITEM("process/user", misc,
+       "Username under which the tor process is running."),
+  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
+  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
+  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
+  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
+  PREFIX("dir/status/", dir,
+         "v2 networkstatus docs as retrieved from a DirPort."),
+  ITEM("dir/status-vote/current/consensus", dir,
+       "v3 Networkstatus consensus as retrieved from a DirPort."),
+  ITEM("exit-policy/default", policies,
+       "The default value appended to the configured exit policy."),
+  ITEM("exit-policy/reject-private/default", policies,
+       "The default rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate."),
+  ITEM("exit-policy/reject-private/relay", policies,
+       "The relay-specific rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
+  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
+  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
+  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
+  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
+  ITEM("onions/current", onions,
+       "Onion services owned by the current control connection."),
+  ITEM("onions/detached", onions,
+       "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
+  { NULL, NULL, NULL, 0 }
+};
+
+/** Allocate and return a list of recognized GETINFO options. */
+static char *
+list_getinfo_options(void)
+{
+  int i;
+  smartlist_t *lines = smartlist_new();
+  char *ans;
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    if (!getinfo_items[i].desc)
+      continue;
+
+    smartlist_add_asprintf(lines, "%s%s -- %s\n",
+                 getinfo_items[i].varname,
+                 getinfo_items[i].is_prefix ? "*" : "",
+                 getinfo_items[i].desc);
+  }
+  smartlist_sort_strings(lines);
+
+  ans = smartlist_join_strings(lines, "", 0, NULL);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+
+  return ans;
+}
+
+/** Lookup the 'getinfo' entry <b>question</b>, and return
+ * the answer in <b>*answer</b> (or NULL if key not recognized).
+ * Return 0 if success or unrecognized, or -1 if recognized but
+ * internal error. */
+static int
+handle_getinfo_helper(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **err_out)
+{
+  int i;
+  *answer = NULL; /* unrecognized key by default */
+
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    int match;
+    if (getinfo_items[i].is_prefix)
+      match = !strcmpstart(question, getinfo_items[i].varname);
+    else
+      match = !strcmp(question, getinfo_items[i].varname);
+    if (match) {
+      tor_assert(getinfo_items[i].fn);
+      return getinfo_items[i].fn(control_conn, question, answer, err_out);
+    }
+  }
+
+  return 0; /* unrecognized */
+}
+
+const control_cmd_syntax_t getinfo_syntax = {
+  .max_args = UINT_MAX,
+};
+
+/** Called when we receive a GETINFO command.  Try to fetch all requested
+ * information, and reply with information or error message. */
+int
+handle_control_getinfo(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *ans = NULL;
+  int i;
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    const char *errmsg = NULL;
+
+    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
+      if (!errmsg)
+        errmsg = "Internal error";
+      control_write_endreply(conn, 551, errmsg);
+      goto done;
+    }
+    if (!ans) {
+      if (errmsg) /* use provided error message */
+        smartlist_add_strdup(unrecognized, errmsg);
+      else /* use default error message */
+        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
+    } else {
+      smartlist_add_strdup(answers, q);
+      smartlist_add(answers, ans);
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if (smartlist_len(unrecognized)) {
+    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
+    for (i=0; i < smartlist_len(unrecognized)-1; ++i)
+      control_write_midreply(conn, 552,
+                             (char *)smartlist_get(unrecognized, i));
+
+    control_write_endreply(conn, 552, (char *)smartlist_get(unrecognized, i));
+    goto done;
+  }
+
+  for (i = 0; i < smartlist_len(answers); i += 2) {
+    char *k = smartlist_get(answers, i);
+    char *v = smartlist_get(answers, i+1);
+    if (!strchr(v, '\n') && !strchr(v, '\r')) {
+      control_printf_midreply(conn, 250, "%s=%s", k, v);
+    } else {
+      control_printf_datareply(conn, 250, "%s=", k);
+      control_write_data(conn, v);
+    }
+  }
+  send_control_done(conn);
+
+ done:
+  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
+  smartlist_free(answers);
+  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
+  smartlist_free(unrecognized);
+
+  return 0;
+}
diff --git a/src/feature/control/control_getinfo.h b/src/feature/control/control_getinfo.h
new file mode 100644
index 0000000000..52978686d8
--- /dev/null
+++ b/src/feature/control/control_getinfo.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control.h
+ * \brief Header file for control.c.
+ **/
+
+#ifndef TOR_CONTROL_GETINFO_H
+#define TOR_CONTROL_GETINFO_H
+
+struct control_cmd_syntax_t;
+struct control_cmd_args_t;
+extern const struct control_cmd_syntax_t getinfo_syntax;
+
+int handle_control_getinfo(control_connection_t *conn,
+                           const struct control_cmd_args_t *args);
+
+#ifdef CONTROL_GETINFO_PRIVATE
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_networkstatus(
+    const char *flavor,
+    download_status_t **dl_to_emit,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_cert(
+    const char *fp_sk_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_desc(
+    const char *desc_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_bridge(
+    const char *bridge_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC int getinfo_helper_downloads(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_time(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+#endif /* defined(CONTROL_GETINFO_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_GETINFO_H) */
diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c
new file mode 100644
index 0000000000..5dec87491d
--- /dev/null
+++ b/src/feature/control/control_proto.c
@@ -0,0 +1,277 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+
+/** Append a NUL-terminated string <b>s</b> to the end of
+ * <b>conn</b>-\>outbuf.
+ */
+void
+connection_write_str_to_buf(const char *s, control_connection_t *conn)
+{
+  size_t len = strlen(s);
+  connection_buf_add(s, len, TO_CONN(conn));
+}
+
+/** Acts like sprintf, but writes its formatted string to the end of
+ * <b>conn</b>-\>outbuf. */
+void
+connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
+{
+  va_list ap;
+  char *buf = NULL;
+  int len;
+
+  va_start(ap,format);
+  len = tor_vasprintf(&buf, format, ap);
+  va_end(ap);
+
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
+
+  tor_free(buf);
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
+ * contents of <b>data</b> into *<b>out</b>, adding a period before any period
+ * that appears at the start of a line, and adding a period-CRLF line at
+ * the end. Replace all LF characters sequences with CRLF.  Return the number
+ * of bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+write_escaped_data(const char *data, size_t len, char **out)
+{
+  tor_assert(len < SIZE_MAX - 9);
+  size_t sz_out = len+8+1;
+  char *outp;
+  const char *start = data, *end;
+  size_t i;
+  int start_of_line;
+  for (i=0; i < len; ++i) {
+    if (data[i] == '\n') {
+      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
+      if (sz_out >= SIZE_T_CEILING) {
+        log_warn(LD_BUG, "Input to write_escaped_data was too long");
+        *out = tor_strdup(".\r\n");
+        return 3;
+      }
+    }
+  }
+  *out = outp = tor_malloc(sz_out);
+  end = data+len;
+  start_of_line = 1;
+  while (data < end) {
+    if (*data == '\n') {
+      if (data > start && data[-1] != '\r')
+        *outp++ = '\r';
+      start_of_line = 1;
+    } else if (*data == '.') {
+      if (start_of_line) {
+        start_of_line = 0;
+        *outp++ = '.';
+      }
+    } else {
+      start_of_line = 0;
+    }
+    *outp++ = *data++;
+  }
+  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
+    *outp++ = '\r';
+    *outp++ = '\n';
+  }
+  *outp++ = '.';
+  *outp++ = '\r';
+  *outp++ = '\n';
+  *outp = '\0'; /* NUL-terminate just in case. */
+  tor_assert(outp >= *out);
+  tor_assert((size_t)(outp - *out) <= sz_out);
+  return outp - *out;
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
+ * the contents of <b>data</b> into *<b>out</b>, removing any period
+ * that appears at the start of a line, and replacing all CRLF sequences
+ * with LF.   Return the number of
+ * bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+read_escaped_data(const char *data, size_t len, char **out)
+{
+  char *outp;
+  const char *next;
+  const char *end;
+
+  *out = outp = tor_malloc(len+1);
+
+  end = data+len;
+
+  while (data < end) {
+    /* we're at the start of a line. */
+    if (*data == '.')
+      ++data;
+    next = memchr(data, '\n', end-data);
+    if (next) {
+      size_t n_to_copy = next-data;
+      /* Don't copy a CR that precedes this LF. */
+      if (n_to_copy && *(next-1) == '\r')
+        --n_to_copy;
+      memcpy(outp, data, n_to_copy);
+      outp += n_to_copy;
+      data = next+1; /* This will point at the start of the next line,
+                      * or the end of the string, or a period. */
+    } else {
+      memcpy(outp, data, end-data);
+      outp += (end-data);
+      *outp = '\0';
+      return outp - *out;
+    }
+    *outp++ = '\n';
+  }
+
+  *outp = '\0';
+  return outp - *out;
+}
+
+/** Send a "DONE" message down the control connection <b>conn</b>. */
+void
+send_control_done(control_connection_t *conn)
+{
+  control_write_endreply(conn, 250, "OK");
+}
+
+/** Write a reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param s string reply content
+ */
+MOCK_IMPL(void,
+control_write_reply, (control_connection_t *conn, int code, int c,
+                      const char *s))
+{
+  connection_printf_to_buf(conn, "%03d%c%s\r\n", code, c, s);
+}
+
+/** Write a formatted reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param fmt format string
+ * @param ap va_list from caller
+ */
+void
+control_vprintf_reply(control_connection_t *conn, int code, int c,
+                      const char *fmt, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, fmt, ap);
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+  control_write_reply(conn, code, c, buf);
+  tor_free(buf);
+}
+
+/** Write an EndReplyLine */
+void
+control_write_endreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, ' ', s);
+}
+
+/** Write a formatted EndReplyLine */
+void
+control_printf_endreply(control_connection_t *conn, int code,
+                        const char *fmt, ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, ' ', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a MidReplyLine */
+void
+control_write_midreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '-', s);
+}
+
+/** Write a formatted MidReplyLine */
+void
+control_printf_midreply(control_connection_t *conn, int code, const char *fmt,
+                        ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '-', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a DataReplyLine */
+void
+control_write_datareply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '+', s);
+}
+
+/** Write a formatted DataReplyLine */
+void
+control_printf_datareply(control_connection_t *conn, int code, const char *fmt,
+                         ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '+', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a CmdData */
+void
+control_write_data(control_connection_t *conn, const char *data)
+{
+  char *esc = NULL;
+  size_t esc_len;
+
+  esc_len = write_escaped_data(data, strlen(data), &esc);
+  connection_buf_add(esc, esc_len, TO_CONN(conn));
+  tor_free(esc);
+}
diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h
new file mode 100644
index 0000000000..3182f3d415
--- /dev/null
+++ b/src/feature/control/control_proto.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.h
+ * \brief Header file for control_proto.c.
+ **/
+
+#ifndef TOR_CONTROL_PROTO_H
+#define TOR_CONTROL_PROTO_H
+
+void connection_write_str_to_buf(const char *s, control_connection_t *conn);
+void connection_printf_to_buf(control_connection_t *conn,
+                                     const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+size_t write_escaped_data(const char *data, size_t len, char **out);
+size_t read_escaped_data(const char *data, size_t len, char **out);
+void send_control_done(control_connection_t *conn);
+
+MOCK_DECL(void, control_write_reply, (control_connection_t *conn, int code,
+                                      int c, const char *s));
+void control_vprintf_reply(control_connection_t *conn, int code, int c,
+                           const char *fmt, va_list ap)
+  CHECK_PRINTF(4, 0);
+void control_write_endreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_endreply(control_connection_t *conn, int code,
+                             const char *fmt, ...)
+  CHECK_PRINTF(3, 4);
+void control_write_midreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_midreply(control_connection_t *conn, int code,
+                             const char *fmt,
+                             ...)
+  CHECK_PRINTF(3, 4);
+void control_write_datareply(control_connection_t *conn, int code,
+                             const char *s);
+void control_printf_datareply(control_connection_t *conn, int code,
+                              const char *fmt,
+                              ...)
+  CHECK_PRINTF(3, 4);
+void control_write_data(control_connection_t *conn, const char *data);
+
+#endif /* !defined(TOR_CONTROL_PROTO_H) */
diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c
index a1ddd2119a..33c5ba1336 100644
--- a/src/feature/control/fmt_serverstatus.c
+++ b/src/feature/control/fmt_serverstatus.c
@@ -9,8 +9,8 @@
 #include "app/config/config.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/voteflags.h"// XXXX remove
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -66,11 +66,9 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   smartlist_t *rs_entries;
   time_t now = time(NULL);
   time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  const or_options_t *options = get_options();
   /* We include v2 dir auths here too, because they need to answer
    * controllers. Eventually we'll deprecate this whole function;
    * see also networkstatus_getinfo_by_purpose(). */
-  int authdir = authdir_mode_publishes_statuses(options);
   tor_assert(router_status_out);
 
   rs_entries = smartlist_new();
@@ -78,10 +76,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
     const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
     tor_assert(node);
-    if (authdir) {
-      /* Update router status in routerinfo_t. */
-      dirserv_set_router_is_running(ri, now);
-    }
     if (for_controller) {
       char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
       char *cp = name_buf;
diff --git a/src/feature/control/fmt_serverstatus.h b/src/feature/control/fmt_serverstatus.h
index 4b95e5b59f..d9190cb7e1 100644
--- a/src/feature/control/fmt_serverstatus.h
+++ b/src/feature/control/fmt_serverstatus.h
@@ -15,4 +15,4 @@
 int list_server_status_v1(smartlist_t *routers, char **router_status_out,
                           int for_controller);
 
-#endif
+#endif /* !defined(TOR_FMT_SERVERSTATUS_H) */
diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h
index fe22137859..94759d0d18 100644
--- a/src/feature/control/getinfo_geoip.h
+++ b/src/feature/control/getinfo_geoip.h
@@ -11,4 +11,4 @@ int getinfo_helper_geoip(control_connection_t *control_conn,
                      const char *question, char **answer,
                      const char **errmsg);
 
-#endif
+#endif /* !defined(TOR_GETINFO_GEOIP_H) */
diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h
index 876a1f947b..bfd5f4dc04 100644
--- a/src/feature/dirauth/authmode.h
+++ b/src/feature/dirauth/authmode.h
@@ -29,7 +29,7 @@ authdir_mode_v3(const or_options_t *options)
 
 #define have_module_dirauth() (1)
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 #define authdir_mode(options) (((void)(options)),0)
 #define authdir_mode_handles_descs(options,purpose) \
@@ -41,6 +41,6 @@ authdir_mode_v3(const or_options_t *options)
 
 #define have_module_dirauth() (0)
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif /* TOR_MODE_H */
+#endif /* !defined(TOR_DIRAUTH_MODE_H) */
diff --git a/src/feature/dirauth/bridgeauth.c b/src/feature/dirauth/bridgeauth.c
new file mode 100644
index 0000000000..4aaefc7a6d
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.c
@@ -0,0 +1,55 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/voteflags.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Write out router status entries for all our bridge descriptors. Here, we
+ * also mark routers as running. */
+void
+bridgeauth_dump_bridge_status_to_file(time_t now)
+{
+  char *status;
+  char *fname = NULL;
+  char *thresholds = NULL;
+  char *published_thresholds_and_status = NULL;
+  char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
+
+  dirserv_set_bridges_running(now);
+  status = networkstatus_getinfo_by_purpose("bridge", now);
+
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
+  format_iso_time(published, now);
+  dirserv_compute_bridge_flag_thresholds();
+  thresholds = dirserv_get_flag_thresholds_line();
+  tor_asprintf(&published_thresholds_and_status,
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
+  if (write_str_to_file(fname,published_thresholds_and_status,0)<0) {
+    log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file.");
+  }
+  tor_free(thresholds);
+  tor_free(published_thresholds_and_status);
+  tor_free(fname);
+  tor_free(status);
+  tor_free(fingerprint_line);
+}
diff --git a/src/feature/dirauth/bridgeauth.h b/src/feature/dirauth/bridgeauth.h
new file mode 100644
index 0000000000..4905e9c3ee
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DIRAUTH_BRIDGEAUTH_H
+#define TOR_DIRAUTH_BRIDGEAUTH_H
+
+void bridgeauth_dump_bridge_status_to_file(time_t now);
+
+#endif /* !defined(TOR_DIRAUTH_BRIDGEAUTH_H) */
diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c
index 12f9399e9f..e60c8b86bd 100644
--- a/src/feature/dirauth/bwauth.c
+++ b/src/feature/dirauth/bwauth.c
@@ -20,6 +20,7 @@
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
+#include "lib/crypt_ops/crypto_format.h"
 #include "lib/encoding/keyval.h"
 
 /** Total number of routers with measured bandwidth; this is set by
@@ -198,14 +199,38 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
 }
 
 /**
- * Read the measured bandwidth list file, apply it to the list of
- * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
+ * Read the measured bandwidth list <b>from_file</b>:
+ * - store all the headers in <b>bw_file_headers</b>,
+ * - apply bandwidth lines to the list of vote_routerstatus_t in
+ *   <b>routerstatuses</b>,
+ * - cache bandwidth lines for dirserv_get_bandwidth_for_router(),
+ * - expire old entries in the measured bandwidth cache, and
+ * - store the DIGEST_SHA256 of the contents of the file in <b>digest_out</b>.
+ *
  * Returns -1 on error, 0 otherwise.
+ *
+ * If the file can't be read, or is empty:
+ * - <b>bw_file_headers</b> is empty,
+ * - <b>routerstatuses</b> is not modified,
+ * - the measured bandwidth cache is not modified, and
+ * - <b>digest_out</b> is the zero-byte digest.
+ *
+ * Otherwise, if there is an error later in the file:
+ * - <b>bw_file_headers</b> contains all the headers up to the error,
+ * - <b>routerstatuses</b> is updated with all the relay lines up to the error,
+ * - the measured bandwidth cache is updated with all the relay lines up to
+ *   the error,
+ * - if the timestamp is valid and recent, old entries in the  measured
+ *   bandwidth cache are expired, and
+ * - <b>digest_out</b> is the digest up to the first read error (if any).
+ *   The digest is taken over all the readable file contents, even if the
+ *   file is outdated or unparseable.
  */
 int
 dirserv_read_measured_bandwidths(const char *from_file,
                                  smartlist_t *routerstatuses,
-                                 smartlist_t *bw_file_headers)
+                                 smartlist_t *bw_file_headers,
+                                 uint8_t *digest_out)
 {
   FILE *fp = tor_fopen_cloexec(from_file, "r");
   int applied_lines = 0;
@@ -219,8 +244,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   int rv = -1;
   char *line = NULL;
   size_t n = 0;
-
-  /* Initialise line, so that we can't possibly run off the end. */
+  crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256);
 
   if (fp == NULL) {
     log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
@@ -228,16 +252,18 @@ dirserv_read_measured_bandwidths(const char *from_file,
     goto err;
   }
 
-  /* If fgets fails, line is either unmodified, or indeterminate. */
   if (tor_getline(&line,&n,fp) <= 0) {
     log_warn(LD_DIRSERV, "Empty bandwidth file");
     goto err;
   }
+  /* If the line could be gotten, add it to the digest */
+  crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
 
   if (!strlen(line) || line[strlen(line)-1] != '\n') {
     log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    /* Continue adding lines to the digest. */
+    goto continue_digest;
   }
 
   line[strlen(line)-1] = '\0';
@@ -245,14 +271,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
   if (!ok) {
     log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    goto continue_digest;
   }
 
-  now = time(NULL);
+  now = approx_time();
   if ((now - file_time) > MAX_MEASUREMENT_AGE) {
     log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
              (unsigned)(time(NULL) - file_time));
-    goto err;
+    goto continue_digest;
   }
 
   /* If timestamp was correct and bw_file_headers is not NULL,
@@ -267,6 +293,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   while (!feof(fp)) {
     measured_bw_line_t parsed_line;
     if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
       if (measured_bw_line_parse(&parsed_line, line,
                                  line_is_after_headers) != -1) {
         /* This condition will be true when the first complete valid bw line
@@ -305,6 +332,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
            "Applied %d measurements.", applied_lines);
   rv = 0;
 
+ continue_digest:
+  /* Continue parsing lines to return the digest of the Bandwidth File. */
+  while (!feof(fp)) {
+    if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
+    }
+  }
+
  err:
   if (line) {
     // we need to raw_free this buffer because we got it from tor_getdelim()
@@ -312,6 +347,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
   }
   if (fp)
     fclose(fp);
+  if (digest_out)
+    crypto_digest_get_digest(digest, (char *) digest_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
   return rv;
 }
 
@@ -327,6 +365,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
  * the header block yet. If we encounter an incomplete bw line, return -1 but
  * don't warn since there could be additional header lines coming. If we
  * encounter a proper bw line, return 0 (and we got past the headers).
+ *
+ * If the line contains "vote=0", stop parsing it, and return -1, so that the
+ * line is ignored during voting.
  */
 STATIC int
 measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h
index 4507728458..81c8affbd7 100644
--- a/src/feature/dirauth/bwauth.h
+++ b/src/feature/dirauth/bwauth.h
@@ -21,8 +21,8 @@
 
 int dirserv_read_measured_bandwidths(const char *from_file,
                                      smartlist_t *routerstatuses,
-                                     smartlist_t *bw_file_headers);
-
+                                     smartlist_t *bw_file_headers,
+                                     uint8_t *digest_out);
 int dirserv_query_measured_bw_cache_kb(const char *node_id,
                                        long *bw_out,
                                        time_t *as_of_out);
@@ -55,4 +55,4 @@ STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
 STATIC void dirserv_expire_measured_bw_cache(time_t now);
 #endif /* defined(BWAUTH_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_BWAUTH_H) */
diff --git a/src/feature/dirauth/dirauth_periodic.c b/src/feature/dirauth/dirauth_periodic.c
new file mode 100644
index 0000000000..02727d61b4
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.c
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "app/config/or_options_st.h"
+#include "core/mainloop/netstatus.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/mainloop/periodic.h"
+
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: if we're an authority, check on our authority
+ * certificate (the one that authenticates our authority signing key).
+ */
+static int
+check_authority_cert_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  (void)options;
+  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
+   * close to expiring and warn the admin if it is. */
+  v3_authority_check_key_expiry();
+#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
+  return CHECK_V3_CERTIFICATE_INTERVAL;
+}
+
+DECLARE_EVENT(check_authority_cert, DIRAUTH, 0);
+
+/**
+ * Scheduled callback: Run directory-authority voting functionality.
+ *
+ * The schedule is a bit complicated here, so dirvote_act() manages the
+ * schedule itself.
+ **/
+static int
+dirvote_callback(time_t now, const or_options_t *options)
+{
+  if (!authdir_mode_v3(options)) {
+    tor_assert_nonfatal_unreached();
+    return 3600;
+  }
+
+  time_t next = dirvote_act(options, now);
+  if (BUG(next == TIME_MAX)) {
+    /* This shouldn't be returned unless we called dirvote_act() without
+     * being an authority.  If it happens, maybe our configuration will
+     * fix itself in an hour or so? */
+    return 3600;
+  }
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(dirvote, DIRAUTH, FL(NEED_NET));
+
+/** Reschedule the directory-authority voting event.  Run this whenever the
+ * schedule has changed. */
+void
+reschedule_dirvote(const or_options_t *options)
+{
+  if (authdir_mode_v3(options)) {
+    periodic_event_reschedule(&dirvote_event);
+  }
+}
+
+/**
+ * Periodic callback: if we're an authority, record our measured stability
+ * information from rephist in an mtbf file.
+ */
+static int
+save_stability_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options)) {
+    if (rep_hist_record_mtbf_data(now, 1)<0) {
+      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+    }
+  }
+#define SAVE_STABILITY_INTERVAL (30*60)
+  return SAVE_STABILITY_INTERVAL;
+}
+
+DECLARE_EVENT(save_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're an authority, make sure we test
+ * the routers on the network for reachability.
+ */
+static int
+launch_reachability_tests_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options) &&
+      !net_is_disabled()) {
+    /* try to determine reachability of the other Tor relays */
+    dirserv_test_reachability(now);
+  }
+  return REACHABILITY_TEST_INTERVAL;
+}
+
+DECLARE_EVENT(launch_reachability_tests, AUTHORITIES, FL(NEED_NET));
+
+/**
+ * Periodic callback: if we're an authority, discount the stability
+ * information (and other rephist information) that's older.
+ */
+static int
+downrate_stability_callback(time_t now, const or_options_t *options)
+{
+  (void)options;
+  /* 1d. Periodically, we discount older stability information so that new
+   * stability info counts more, and save the stability information to disk as
+   * appropriate. */
+  time_t next = rep_hist_downrate_old_runs(now);
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(downrate_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're the bridge authority, write a networkstatus
+ * file to disk.
+ */
+static int
+write_bridge_ns_callback(time_t now, const or_options_t *options)
+{
+  if (options->BridgeAuthoritativeDir) {
+    bridgeauth_dump_bridge_status_to_file(now);
+#define BRIDGE_STATUSFILE_INTERVAL (30*60)
+     return BRIDGE_STATUSFILE_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(write_bridge_ns, BRIDGEAUTH, 0);
+
+void
+dirauth_register_periodic_events(void)
+{
+  periodic_events_register(&downrate_stability_event);
+  periodic_events_register(&launch_reachability_tests_event);
+  periodic_events_register(&save_stability_event);
+  periodic_events_register(&check_authority_cert_event);
+  periodic_events_register(&dirvote_event);
+  periodic_events_register(&write_bridge_ns_event);
+}
diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h
new file mode 100644
index 0000000000..866fbd35de
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIRVOTE_PERIODIC_H
+#define DIRVOTE_PERIODIC_H
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+void dirauth_register_periodic_events(void);
+void reschedule_dirvote(const or_options_t *options);
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+static inline void
+reschedule_dirvote(const or_options_t *options)
+{
+  (void)options;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(DIRVOTE_PERIODIC_H) */
diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c
new file mode 100644
index 0000000000..e38d391300
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.c
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_dirauth_initialize(void)
+{
+  dirauth_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_dirauth_shutdown(void)
+{
+  dirserv_free_fingerprint_list();
+  dirvote_free_all();
+  dirserv_clear_measured_bw_cache();
+  keypin_close_journal();
+}
+
+const struct subsys_fns_t sys_dirauth = {
+  .name = "dirauth",
+  .supported = true,
+  .level = 70,
+  .initialize = subsys_dirauth_initialize,
+  .shutdown = subsys_dirauth_shutdown,
+};
diff --git a/src/feature/dirauth/dirauth_sys.h b/src/feature/dirauth/dirauth_sys.h
new file mode 100644
index 0000000000..4e9b6a2ab4
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIRAUTH_SYS_H
+#define DIRAUTH_SYS_H
+
+extern const struct subsys_fns_t sys_dirauth;
+
+#endif /* !defined(DIRAUTH_SYS_H) */
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
index af8b3dc207..043bbfc227 100644
--- a/src/feature/dirauth/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -28,6 +28,7 @@
 #include "feature/nodelist/fmt_routerstatus.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -60,6 +61,9 @@
 #include "lib/encoding/confline.h"
 #include "lib/crypt_ops/crypto_format.h"
 
+/* Algorithm to use for the bandwidth file digest. */
+#define DIGEST_ALG_BW_FILE DIGEST_SHA256
+
 /**
  * \file dirvote.c
  * \brief Functions to compute directory consensus, and schedule voting.
@@ -216,7 +220,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                           networkstatus_t *v3_ns)
 {
   smartlist_t *chunks = smartlist_new();
-  char *packages = NULL;
   char fingerprint[FINGERPRINT_LEN+1];
   char digest[DIGEST_LEN];
   uint32_t addr;
@@ -242,19 +245,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                                                 v3_ns->server_versions);
   protocols_lines = format_protocols_lines_for_vote(v3_ns);
 
-  if (v3_ns->package_lines) {
-    smartlist_t *tmp = smartlist_new();
-    SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p,
-                      if (validate_recommended_package_line(p))
-                        smartlist_add_asprintf(tmp, "package %s\n", p));
-    smartlist_sort_strings(tmp);
-    packages = smartlist_join_strings(tmp, "", 0, NULL);
-    SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp));
-    smartlist_free(tmp);
-  } else {
-    packages = tor_strdup("");
-  }
-
     /* Get shared random commitments/reveals line(s). */
   shared_random_vote_str = sr_get_string_for_vote();
 
@@ -268,6 +258,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     char *flag_thresholds = dirserv_get_flag_thresholds_line();
     char *params;
     char *bw_headers_line = NULL;
+    char *bw_file_digest = NULL;
     authority_cert_t *cert = v3_ns->cert;
     char *methods =
       make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
@@ -307,6 +298,27 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
       tor_free(bw_file_headers);
     }
 
+    /* Create bandwidth-file-digest if applicable.
+     * v3_ns->b64_digest_bw_file will contain the digest when V3BandwidthsFile
+     * is configured and the bandwidth file could be read, even if it was not
+     * parseable.
+     */
+    if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) {
+      /* Encode the digest. */
+      char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0};
+      digest256_to_base64(b64_digest_bw_file,
+                          (const char *)v3_ns->bw_file_digest256);
+      /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */
+      char *digest_algo_b64_digest_bw_file = NULL;
+      tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s",
+                   crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE),
+                   b64_digest_bw_file);
+      /* No need for tor_strdup(""), format_line_if_present does it. */
+      bw_file_digest = format_line_if_present(
+          "bandwidth-file-digest", digest_algo_b64_digest_bw_file);
+      tor_free(digest_algo_b64_digest_bw_file);
+    }
+
     smartlist_add_asprintf(chunks,
                  "network-status-version 3\n"
                  "vote-status %s\n"
@@ -318,11 +330,11 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  "voting-delay %d %d\n"
                  "%s%s" /* versions */
                  "%s" /* protocols */
-                 "%s" /* packages */
                  "known-flags %s\n"
                  "flag-thresholds %s\n"
                  "params %s\n"
                  "%s" /* bandwidth file headers */
+                 "%s" /* bandwidth file digest */
                  "dir-source %s %s %s %s %d %d\n"
                  "contact %s\n"
                  "%s" /* shared randomness information */
@@ -334,11 +346,11 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  client_versions_line,
                  server_versions_line,
                  protocols_lines,
-                 packages,
                  flags,
                  flag_thresholds,
                  params,
                  bw_headers_line ? bw_headers_line : "",
+                 bw_file_digest ? bw_file_digest: "",
                  voter->nickname, fingerprint, voter->address,
                  fmt_addr32(addr), voter->dir_port, voter->or_port,
                  voter->contact,
@@ -351,6 +363,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     tor_free(methods);
     tor_free(shared_random_vote_str);
     tor_free(bw_headers_line);
+    tor_free(bw_file_digest);
 
     if (!tor_digest_is_zero(voter->legacy_id_digest)) {
       char fpbuf[HEX_DIGEST_LEN+1];
@@ -412,7 +425,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
 
   {
     networkstatus_t *v;
-    if (!(v = networkstatus_parse_vote_from_string(status, NULL,
+    if (!(v = networkstatus_parse_vote_from_string(status, strlen(status),
+                                                   NULL,
                                                    v3_ns->type))) {
       log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: "
               "<<%s>>",
@@ -430,7 +444,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
   tor_free(client_versions_line);
   tor_free(server_versions_line);
   tor_free(protocols_lines);
-  tor_free(packages);
 
   SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
   smartlist_free(chunks);
@@ -2409,7 +2422,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
   {
     networkstatus_t *c;
-    if (!(c = networkstatus_parse_vote_from_string(result, NULL,
+    if (!(c = networkstatus_parse_vote_from_string(result, strlen(result),
+                                                   NULL,
                                                    NS_TYPE_CONSENSUS))) {
       log_err(LD_BUG, "Generated a networkstatus consensus we couldn't "
               "parse.");
@@ -2567,7 +2581,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target,
       return -1;
     }
     for (alg = DIGEST_SHA1; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) {
-      if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
+      if (!fast_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
         if (fast_memeq(target->digests.d[alg], digests->d[alg],
                        DIGEST256_LEN)) {
           ++n_matches;
@@ -2763,7 +2777,7 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses)
       char d[HEX_DIGEST256_LEN+1];
       const char *alg_name =
         crypto_digest_algorithm_get_name(alg);
-      if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
+      if (fast_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
         continue;
       base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN);
       smartlist_add_asprintf(elements, "additional-digest %s %s %s\n",
@@ -3132,7 +3146,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
   *msg_out = NULL;
 
  again:
-  vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote,
+  vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body),
+                                              &end_of_vote,
                                               NS_TYPE_VOTE);
   if (!end_of_vote)
     end_of_vote = vote_body + strlen(vote_body);
@@ -3390,7 +3405,9 @@ dirvote_compute_consensuses(void)
                  flavor_name);
         continue;
       }
-      consensus = networkstatus_parse_vote_from_string(consensus_body, NULL,
+      consensus = networkstatus_parse_vote_from_string(consensus_body,
+                                                       strlen(consensus_body),
+                                                       NULL,
                                                        NS_TYPE_CONSENSUS);
       if (!consensus) {
         log_warn(LD_DIR, "Couldn't parse %s consensus we generated!",
@@ -3529,7 +3546,7 @@ dirvote_add_signatures_to_pending_consensus(
      * just in case we break detached signature processing at some point. */
     {
       networkstatus_t *v = networkstatus_parse_vote_from_string(
-                                             pc->body, NULL,
+                                             pc->body, strlen(pc->body), NULL,
                                              NS_TYPE_CONSENSUS);
       tor_assert(v);
       networkstatus_vote_free(v);
@@ -3654,7 +3671,9 @@ dirvote_publish_consensus(void)
       continue;
     }
 
-    if (networkstatus_set_current_consensus(pending->body, name, 0, NULL))
+    if (networkstatus_set_current_consensus(pending->body,
+                                            strlen(pending->body),
+                                            name, 0, NULL))
       log_warn(LD_DIR, "Error publishing %s consensus", name);
     else
       log_notice(LD_DIR, "Published %s consensus", name);
@@ -3791,8 +3810,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     smartlist_add_asprintf(chunks, "a %s\n",
                            fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
 
-  if (family)
-    smartlist_add_asprintf(chunks, "family %s\n", family);
+  if (family) {
+    if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) {
+      smartlist_add_asprintf(chunks, "family %s\n", family);
+    } else {
+      const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest;
+      char *canonical_family = nodefamily_canonicalize(family, id, 0);
+      smartlist_add_asprintf(chunks, "family %s\n", canonical_family);
+      tor_free(canonical_family);
+    }
+  }
 
   if (summary && strcmp(summary, "reject 1-65535"))
     smartlist_add_asprintf(chunks, "p %s\n", summary);
@@ -3869,8 +3896,7 @@ dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len,
                                ",");
   tor_assert(microdesc_consensus_methods);
 
-  if (digest256_to_base64(d64, md->digest)<0)
-    goto out;
+  digest256_to_base64(d64, md->digest);
 
   if (tor_snprintf(out_buf, out_buf_len, "m %s sha256=%s\n",
                    microdesc_consensus_methods, d64)<0)
@@ -3890,7 +3916,10 @@ static const struct consensus_method_range_t {
   int high;
 } microdesc_consensus_methods[] = {
   {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1},
-  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD},
+  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC,
+   MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1},
+  {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS,
+   MAX_SUPPORTED_CONSENSUS_METHOD},
   {-1, -1}
 };
 
@@ -4364,6 +4393,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs)
    * forget to add it to this clause. */
 }
 
+/** Space-separated list of all the flags that we will always vote on. */
+const char DIRVOTE_UNIVERSAL_FLAGS[] =
+  "Authority "
+  "Exit "
+  "Fast "
+  "Guard "
+  "HSDir "
+  "Stable "
+  "StaleDesc "
+  "V2Dir "
+  "Valid";
+/** Space-separated list of all flags that we may or may not vote on,
+ * depending on our configuration. */
+const char DIRVOTE_OPTIONAL_FLAGS[] =
+  "BadExit "
+  "Running";
+
 /** Return a new networkstatus_t* containing our current opinion. (For v3
  * authorities) */
 networkstatus_t *
@@ -4388,6 +4434,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   const int vote_on_reachability = running_long_enough_to_decide_unreachable();
   smartlist_t *microdescriptors = NULL;
   smartlist_t *bw_file_headers = NULL;
+  uint8_t bw_file_digest256[DIGEST256_LEN] = {0};
 
   tor_assert(private_key);
   tor_assert(cert);
@@ -4425,7 +4472,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
    * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
    */
   if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                     NULL);
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4479,8 +4527,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
       vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       rs = &vrs->status;
-      set_routerstatus_from_routerinfo(rs, node, ri, now,
-                                       listbadexits);
+      dirauth_set_routerstatus_from_routerinfo(rs, node, ri, now,
+                                               listbadexits);
 
       if (ri->cache_info.signing_key_cert) {
         memcpy(vrs->ed25519_id,
@@ -4530,7 +4578,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     /* Only set bw_file_headers when V3BandwidthsFile is configured */
     bw_file_headers = smartlist_new();
     dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
-                                     routerstatuses, bw_file_headers);
+                                     routerstatuses, bw_file_headers,
+                                     bw_file_digest256);
+
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4601,18 +4651,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   tor_assert_nonfatal(protover_all_supported(
                                v3_out->recommended_client_protocols, NULL));
 
-  v3_out->package_lines = smartlist_new();
-  {
-    config_line_t *cl;
-    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
-      if (validate_recommended_package_line(cl->value))
-        smartlist_add_strdup(v3_out->package_lines, cl->value);
-    }
-  }
-
   v3_out->known_flags = smartlist_new();
   smartlist_split_string(v3_out->known_flags,
-                         "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   if (vote_on_reachability)
     smartlist_add_strdup(v3_out->known_flags, "Running");
@@ -4627,6 +4668,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     smartlist_sort_strings(v3_out->net_params);
   }
   v3_out->bw_file_headers = bw_file_headers;
+  memcpy(v3_out->bw_file_digest256, bw_file_digest256, DIGEST256_LEN);
 
   voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
   voter->nickname = tor_strdup(options->Nickname);
diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h
index 02d88d19d1..b7df33a3a9 100644
--- a/src/feature/dirauth/dirvote.h
+++ b/src/feature/dirauth/dirvote.h
@@ -57,7 +57,7 @@
 #define MIN_SUPPORTED_CONSENSUS_METHOD 25
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 28
+#define MAX_SUPPORTED_CONSENSUS_METHOD 29
 
 /** Lowest consensus method where authorities vote on required/recommended
  * protocols. */
@@ -79,6 +79,12 @@
  * addresses. See #23828 and #20916. */
 #define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28
 
+/**
+ * Lowest consensus method where microdescriptor lines are put in canonical
+ * form for improved compressibility and ease of storage. See proposal 298.
+ **/
+#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29
+
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
  * get confused with the above macros.) */
@@ -92,6 +98,9 @@
 /** Maximum size of a line in a vote. */
 #define MAX_BW_FILE_HEADERS_LINE_LEN 1024
 
+extern const char DIRVOTE_UNIVERSAL_FLAGS[];
+extern const char DIRVOTE_OPTIONAL_FLAGS[];
+
 /*
  * Public API. Used outside of the dirauth subsystem.
  *
@@ -119,7 +128,7 @@ struct config_line_t;
 char *format_recommended_version_list(const struct config_line_t *line,
                                       int warn);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline time_t
 dirvote_act(const or_options_t *options, time_t now)
@@ -184,7 +193,7 @@ dirvote_add_signatures(const char *detached_signatures_body,
   return 0;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Item access */
 MOCK_DECL(const char*, dirvote_get_pending_consensus,
diff --git a/src/feature/dirauth/dsigs_parse.c b/src/feature/dirauth/dsigs_parse.c
index d88176fee9..c5c8e18866 100644
--- a/src/feature/dirauth/dsigs_parse.c
+++ b/src/feature/dirauth/dsigs_parse.c
@@ -127,7 +127,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos)
     }
     digests = detached_get_digests(sigs, flavor);
     tor_assert(digests);
-    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
+    if (!fast_mem_is_zero(digests->d[alg], digest_length)) {
       log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
                "signatures document", flavor, algname);
       continue;
diff --git a/src/feature/dirauth/dsigs_parse.h b/src/feature/dirauth/dsigs_parse.h
index fec51ba488..0cc53072f8 100644
--- a/src/feature/dirauth/dsigs_parse.h
+++ b/src/feature/dirauth/dsigs_parse.h
@@ -19,4 +19,4 @@ void ns_detached_signatures_free_(ns_detached_signatures_t *s);
 #define ns_detached_signatures_free(s) \
   FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
 
-#endif
+#endif /* !defined(TOR_DSIGS_PARSE_H) */
diff --git a/src/feature/dirauth/guardfraction.h b/src/feature/dirauth/guardfraction.h
index 72404907a4..9f01ded838 100644
--- a/src/feature/dirauth/guardfraction.h
+++ b/src/feature/dirauth/guardfraction.h
@@ -21,4 +21,4 @@ dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
 int dirserv_read_guardfraction_file(const char *fname,
                                  smartlist_t *vote_routerstatuses);
 
-#endif
+#endif /* !defined(TOR_GUARDFRACTION_H) */
diff --git a/src/feature/dirauth/keypin.c b/src/feature/dirauth/keypin.c
index 06cb9ba1ff..316b7d6c2f 100644
--- a/src/feature/dirauth/keypin.c
+++ b/src/feature/dirauth/keypin.c
@@ -438,7 +438,7 @@ keypin_load_journal_impl(const char *data, size_t size)
   tor_log(severity, LD_DIRSERV,
           "Loaded %d entries from keypin journal. "
           "Found %d corrupt lines (ignored), %d duplicates (harmless), "
-          "and %d conflicts (resolved in favor or more recent entry).",
+          "and %d conflicts (resolved in favor of more recent entry).",
           n_entries, n_corrupt_lines, n_duplicates, n_conflicts);
 
   return 0;
diff --git a/src/feature/dirauth/keypin.h b/src/feature/dirauth/keypin.h
index 722b6ca5fc..1de84f6d4a 100644
--- a/src/feature/dirauth/keypin.h
+++ b/src/feature/dirauth/keypin.h
@@ -11,10 +11,25 @@ int keypin_check_and_add(const uint8_t *rsa_id_digest,
                          const int replace_existing_entry);
 int keypin_check(const uint8_t *rsa_id_digest,
                  const uint8_t *ed25519_id_key);
+int keypin_close_journal(void);
 
+#ifdef HAVE_MODULE_DIRAUTH
 int keypin_open_journal(const char *fname);
-int keypin_close_journal(void);
 int keypin_load_journal(const char *fname);
+#else
+static inline int
+keypin_open_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+static inline int
+keypin_load_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 void keypin_clear(void);
 int keypin_check_lone_rsa(const uint8_t *rsa_id_digest);
 
@@ -44,4 +59,3 @@ MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent));
 #endif /* defined(KEYPIN_PRIVATE) */
 
 #endif /* !defined(TOR_KEYPIN_H) */
-
diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h
index 0f92be2f0d..61d20b7525 100644
--- a/src/feature/dirauth/ns_detached_signatures_st.h
+++ b/src/feature/dirauth/ns_detached_signatures_st.h
@@ -18,5 +18,5 @@ struct ns_detached_signatures_t {
                          * document_signature_t */
 };
 
-#endif
+#endif /* !defined(NS_DETACHED_SIGNATURES_ST_H) */
 
diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c
index 21b8e239ec..71e3195c01 100644
--- a/src/feature/dirauth/process_descs.c
+++ b/src/feature/dirauth/process_descs.c
@@ -216,9 +216,14 @@ dirserv_load_fingerprint_file(void)
 
 #define DISABLE_DISABLING_ED25519
 
-/** Check whether <b>router</b> has a nickname/identity key combination that
- * we recognize from the fingerprint list, or an IP we automatically act on
- * according to our configuration.  Return the appropriate router status.
+/** Check whether <b>router</b> has:
+ * - a nickname/identity key combination that we recognize from the fingerprint
+ *   list,
+ * - an IP we automatically act on according to our configuration,
+ * - an appropriate version, and
+ * - matching pinned keys.
+ *
+ * Return the appropriate router status.
  *
  * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
  * *<b>msg</b> to an explanation of why. */
@@ -236,7 +241,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
     return FP_REJECT;
   }
 
-  /* Check for the more usual versions to reject a router first. */
+  /* Check for the more common reasons to reject a router first. */
   const uint32_t r = dirserv_get_status_impl(d, router->nickname,
                                              router->addr, router->or_port,
                                              router->platform, msg, severity);
@@ -310,6 +315,47 @@ dirserv_would_reject_router(const routerstatus_t *rs)
   return (res & FP_REJECT) != 0;
 }
 
+/**
+ * Check whether the platform string in <b>platform</b> describes a platform
+ * that, as a directory authority, we want to reject.  If it does, return
+ * true, and set *<b>msg</b> (if present) to a rejection message.  Otherwise
+ * return false.
+ */
+STATIC bool
+dirserv_rejects_tor_version(const char *platform,
+                            const char **msg)
+{
+  if (!platform)
+    return false;
+
+  static const char please_upgrade_string[] =
+    "Tor version is insecure or unsupported. Please upgrade!";
+
+  /* Versions before Tor 0.2.9 are unsupported. Versions between 0.2.9.0 and
+   * 0.2.9.4 suffer from bug #20499, where relays don't keep their consensus
+   * up to date */
+  if (!tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
+    if (msg)
+      *msg = please_upgrade_string;
+    return true;
+  }
+
+  /* Series between Tor 0.3.0 and 0.3.4 inclusive are unsupported, and some
+   * have bug #27841, which makes them broken as intro points. Reject them.
+   *
+   * Also reject unstable versions of 0.3.5, since (as of this writing)
+   * they are almost none of the network. */
+  if (tor_version_as_new_as(platform,"0.3.0.0-alpha-dev") &&
+      !tor_version_as_new_as(platform,"0.3.5.7")) {
+    if (msg) {
+      *msg = please_upgrade_string;
+    }
+    return true;
+  }
+
+  return false;
+}
+
 /** Helper: As dirserv_router_get_status, but takes the router fingerprint
  * (hex, no spaces), nickname, address (used for logging only), IP address, OR
  * port and platform (logging only) as arguments.
@@ -342,22 +388,8 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
     }
   }
 
-  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
-   * missing some important security fixes too. Disable them. */
-  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
-    if (msg)
-      *msg = "Tor version is insecure or unsupported. Please upgrade!";
-    return FP_REJECT;
-  }
-
-  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
-   * keep their consensus up to date so they make bad guards.
-   * The simple fix is to just drop them from the network. */
-  if (platform &&
-      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
-      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
-    if (msg)
-      *msg = "Tor version contains bug 20499. Please upgrade!";
+  /* Check whether the version is obsolete, broken, insecure, etc... */
+  if (platform && dirserv_rejects_tor_version(platform, msg)) {
     return FP_REJECT;
   }
 
@@ -423,20 +455,32 @@ dirserv_free_fingerprint_list(void)
 
 /** Return -1 if <b>ri</b> has a private or otherwise bad address,
  * unless we're configured to not care. Return 0 if all ok. */
-static int
+STATIC int
 dirserv_router_has_valid_address(routerinfo_t *ri)
 {
   tor_addr_t addr;
+
   if (get_options()->DirAllowPrivateAddresses)
     return 0; /* whatever it is, we're fine with it */
+
   tor_addr_from_ipv4h(&addr, ri->addr);
+  if (tor_addr_is_null(&addr) || tor_addr_is_internal(&addr, 0)) {
+    log_info(LD_DIRSERV,
+             "Router %s published internal IPv4 address. Refusing.",
+             router_describe(ri));
+    return -1; /* it's a private IP, we should reject it */
+  }
 
-  if (tor_addr_is_internal(&addr, 0)) {
+  /* We only check internal v6 on non-null addresses because we do not require
+   * IPv6 and null IPv6 is normal. */
+  if (!tor_addr_is_null(&ri->ipv6_addr) &&
+      tor_addr_is_internal(&ri->ipv6_addr, 0)) {
     log_info(LD_DIRSERV,
-             "Router %s published internal IP address. Refusing.",
+             "Router %s published internal IPv6 address. Refusing.",
              router_describe(ri));
     return -1; /* it's a private IP, we should reject it */
   }
+
   return 0;
 }
 
@@ -519,7 +563,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
 /** As for dirserv_add_descriptor(), but accepts multiple documents, and
  * returns the most severe error that occurred for any one of them. */
 was_router_added_t
-dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
                                  const char *source,
                                  const char **msg)
 {
@@ -534,7 +579,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   int general = purpose == ROUTER_PURPOSE_GENERAL;
   tor_assert(msg);
 
-  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
+  r=ROUTER_ADDED_SUCCESSFULLY; /* Least severe return value. */
+
+  if (!string_is_utf8_no_bom(desc, desclen)) {
+    *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM.";
+    return ROUTER_AUTHDIR_REJECTS;
+  }
 
   format_iso_time(time_buf, now);
   if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
@@ -545,14 +595,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
                    !general ? router_purpose_to_string(purpose) : "",
                    !general ? "\n" : "")<0) {
     *msg = "Couldn't format annotations";
-    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
-     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
-    return -1;
+    return ROUTER_AUTHDIR_BUG_ANNOTATIONS;
   }
 
   s = desc;
   list = smartlist_new();
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0,
                                      annotation_buf, NULL)) {
     SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
         msg_out = NULL;
@@ -568,7 +616,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   smartlist_clear(list);
 
   s = desc;
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0,
                                      NULL, NULL)) {
     SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
         msg_out = NULL;
diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h
index ae2d6ad25d..e504daa7b7 100644
--- a/src/feature/dirauth/process_descs.h
+++ b/src/feature/dirauth/process_descs.h
@@ -12,27 +12,107 @@
 #ifndef TOR_RECV_UPLOADS_H
 #define TOR_RECV_UPLOADS_H
 
-int dirserv_load_fingerprint_file(void);
+// for was_router_added_t.
+#include "feature/nodelist/routerlist.h"
+
 void dirserv_free_fingerprint_list(void);
-int dirserv_add_own_fingerprint(crypto_pk_t *pk);
 
+#ifdef HAVE_MODULE_DIRAUTH
+int dirserv_load_fingerprint_file(void);
 enum was_router_added_t dirserv_add_multiple_descriptors(
-                                     const char *desc, uint8_t purpose,
+                                     const char *desc, size_t desclen,
+                                     uint8_t purpose,
                                      const char *source,
                                      const char **msg);
 enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
                                                const char **msg,
                                                const char *source);
 
+int dirserv_would_reject_router(const routerstatus_t *rs);
 int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
                                    int complain,
                                    int *valid_out);
+int dirserv_add_own_fingerprint(crypto_pk_t *pk);
 uint32_t dirserv_router_get_status(const routerinfo_t *router,
                                    const char **msg,
                                    int severity);
 void dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                       uint32_t authstatus);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+static inline int
+dirserv_load_fingerprint_file(void)
+{
+  return 0;
+}
+static inline enum was_router_added_t
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
+                                 const char *source,
+                                 const char **msg)
+{
+  (void)desc;
+  (void)desclen;
+  (void)purpose;
+  (void)source;
+  (void)msg;
+  return (enum was_router_added_t)0;
+}
+static inline enum was_router_added_t
+dirserv_add_descriptor(routerinfo_t *ri,
+                       const char **msg,
+                       const char *source)
+{
+  (void)ri;
+  (void)msg;
+  (void)source;
+  return (enum was_router_added_t)0;
+}
+static inline int
+dirserv_would_reject_router(const routerstatus_t *rs)
+{
+  (void)rs;
+  return 0;
+}
+static inline int
+authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                               int complain,
+                               int *valid_out)
+{
+  (void)ri;
+  (void)msg;
+  (void)complain;
+  (void)valid_out;
+  return 0;
+}
+static inline int
+dirserv_add_own_fingerprint(crypto_pk_t *pk)
+{
+  (void)pk;
+  return 0;
+}
+static inline uint32_t
+dirserv_router_get_status(const routerinfo_t *router,
+                          const char **msg,
+                          int severity)
+{
+  (void)router;
+  (void)msg;
+  (void)severity;
+  return 0;
+}
+static inline void
+dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                 uint32_t authstatus)
+{
+  (void)node;
+  (void)authstatus;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-int dirserv_would_reject_router(const routerstatus_t *rs);
+#ifdef TOR_UNIT_TESTS
+STATIC int dirserv_router_has_valid_address(routerinfo_t *ri);
+STATIC bool dirserv_rejects_tor_version(const char *platform,
+                                        const char **msg);
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_RECV_UPLOADS_H) */
diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h
index 5a938673ff..46d0e7ee2e 100644
--- a/src/feature/dirauth/reachability.h
+++ b/src/feature/dirauth/reachability.h
@@ -24,13 +24,36 @@
 #define REACHABILITY_TEST_CYCLE_PERIOD \
   (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
 
+void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
+void dirserv_test_reachability(time_t now);
+
+#ifdef HAVE_MODULE_DIRAUTH
+int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old);
 void dirserv_orconn_tls_done(const tor_addr_t *addr,
                              uint16_t or_port,
                              const char *digest_rcvd,
                              const struct ed25519_public_key_t *ed_id_rcvd);
-int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
-                                            const routerinfo_t *ri_old);
-void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
-void dirserv_test_reachability(time_t now);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+static inline int
+dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old)
+{
+  (void)ri;
+  (void)ri_old;
+  return 0;
+}
+static inline void
+dirserv_orconn_tls_done(const tor_addr_t *addr,
+                        uint16_t or_port,
+                        const char *digest_rcvd,
+                        const struct ed25519_public_key_t *ed_id_rcvd)
+{
+  (void)addr;
+  (void)or_port;
+  (void)digest_rcvd;
+  (void)ed_id_rcvd;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif
+#endif /* !defined(TOR_REACHABILITY_H) */
diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h
index 8200d78f72..af17e945e8 100644
--- a/src/feature/dirauth/recommend_pkg.h
+++ b/src/feature/dirauth/recommend_pkg.h
@@ -12,6 +12,18 @@
 #ifndef TOR_RECOMMEND_PKG_H
 #define TOR_RECOMMEND_PKG_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 int validate_recommended_package_line(const char *line);
 
-#endif
+#else
+
+static inline int
+validate_recommended_package_line(const char *line)
+{
+  (void) line;
+  return 0;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(TOR_RECOMMEND_PKG_H) */
diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c
index 34b2283250..a45f0a29c3 100644
--- a/src/feature/dirauth/shared_random.c
+++ b/src/feature/dirauth/shared_random.c
@@ -90,7 +90,7 @@
 #include "core/or/or.h"
 #include "feature/dirauth/shared_random.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
@@ -120,8 +120,8 @@ static const char sr_flag_ns_str[] = "shared-rand-participate";
 static int32_t num_srv_agreements_from_vote;
 
 /* Return a heap allocated copy of the SRV <b>orig</b>. */
-STATIC sr_srv_t *
-srv_dup(const sr_srv_t *orig)
+sr_srv_t *
+sr_srv_dup(const sr_srv_t *orig)
 {
   sr_srv_t *duplicate = NULL;
 
@@ -224,7 +224,7 @@ verify_commit_and_reveal(const sr_commit_t *commit)
 STATIC int
 commit_has_reveal_value(const sr_commit_t *commit)
 {
-  return !tor_mem_is_zero(commit->encoded_reveal,
+  return !fast_mem_is_zero(commit->encoded_reveal,
                           sizeof(commit->encoded_reveal));
 }
 
@@ -486,7 +486,7 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
   {
     /* Send a reveal value for this commit if we have one. */
     const char *reveal_str = commit->encoded_reveal;
-    if (tor_mem_is_zero(commit->encoded_reveal,
+    if (fast_mem_is_zero(commit->encoded_reveal,
                         sizeof(commit->encoded_reveal))) {
       reveal_str = "";
     }
@@ -1253,8 +1253,8 @@ sr_act_post_consensus(const networkstatus_t *consensus)
      * decided by the majority. */
     sr_state_unset_fresh_srv();
     /* Set the SR values from the given consensus. */
-    sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv));
-    sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv));
+    sr_state_set_previous_srv(sr_srv_dup(consensus->sr_info.previous_srv));
+    sr_state_set_current_srv(sr_srv_dup(consensus->sr_info.current_srv));
   }
 
   /* Prepare our state so that it's ready for the next voting period. */
diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h
index 25d95ebbc7..7ff9f15512 100644
--- a/src/feature/dirauth/shared_random.h
+++ b/src/feature/dirauth/shared_random.h
@@ -110,7 +110,7 @@ int sr_init(int save_to_disk);
 void sr_save_and_cleanup(void);
 void sr_act_post_consensus(const networkstatus_t *consensus);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline int
 sr_init(int save_to_disk)
@@ -131,7 +131,7 @@ sr_act_post_consensus(const networkstatus_t *consensus)
   (void) consensus;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Public methods used only by dirauth code. */
 
@@ -154,6 +154,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
 void sr_compute_srv(void);
 sr_commit_t *sr_generate_our_commit(time_t timestamp,
                                     const authority_cert_t *my_rsa_cert);
+sr_srv_t *sr_srv_dup(const sr_srv_t *orig);
 
 #ifdef SHARED_RANDOM_PRIVATE
 
@@ -172,7 +173,6 @@ STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes,
                                              int current);
 
 STATIC void save_commit_to_state(sr_commit_t *commit);
-STATIC sr_srv_t *srv_dup(const sr_srv_t *orig);
 STATIC int commitments_are_the_same(const sr_commit_t *commit_one,
                                     const sr_commit_t *commit_two);
 STATIC int commit_is_authoritative(const sr_commit_t *commit,
diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c
index b3e4a4ef92..4078d6a24a 100644
--- a/src/feature/dirauth/shared_random_state.c
+++ b/src/feature/dirauth/shared_random_state.c
@@ -12,7 +12,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/nodelist/networkstatus.h"
@@ -22,6 +22,7 @@
 #include "feature/dirauth/shared_random_state.h"
 #include "feature/dircommon/voting_schedule.h"
 #include "lib/encoding/confline.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
@@ -50,24 +51,21 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue";
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t);
 
-/* These next two are duplicates or near-duplicates from config.c */
-#define VAR(name, conftype, member, initvalue)                              \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member),      \
-      initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) }
-/* As VAR, but the option name and member name are the same. */
-#define V(member, conftype, initvalue) \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
+
 /* Our persistent state magic number. */
 #define SR_DISK_STATE_MAGIC 0x98AB1254
 
 static int
 disk_state_validate_cb(void *old_state, void *state, void *default_state,
                        int from_setconf, char **msg);
-static void disk_state_free_cb(void *);
 
 /* Array of variables that are saved to disk as a persistent state. */
-static config_var_t state_vars[] = {
-  V(Version,                    UINT, "0"),
+static const config_var_t state_vars[] = {
+  V(Version,                    POSINT, "0"),
   V(TorVersion,                 STRING, NULL),
   V(ValidAfter,                 ISOTIME, NULL),
   V(ValidUntil,                 ISOTIME, NULL),
@@ -82,25 +80,45 @@ static config_var_t state_vars[] = {
 
 /* "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST,
-  offsetof(sr_disk_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines)
+static const struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(sr_disk_state_t, ExtraLines),
 };
 
 /* Configuration format of sr_disk_state_t. */
 static const config_format_t state_format = {
   sizeof(sr_disk_state_t),
-  SR_DISK_STATE_MAGIC,
-  offsetof(sr_disk_state_t, magic_),
+  {
+   "sr_disk_state_t",
+   SR_DISK_STATE_MAGIC,
+   offsetof(sr_disk_state_t, magic_),
+  },
   NULL,
   NULL,
   state_vars,
   disk_state_validate_cb,
-  disk_state_free_cb,
+  NULL,
   &state_extra_var,
+  -1,
 };
 
+/* Global configuration manager for the shared-random state file */
+static config_mgr_t *shared_random_state_mgr = NULL;
+
+/** Return the configuration manager for the shared-random state file. */
+static const config_mgr_t *
+get_srs_mgr(void)
+{
+  if (PREDICT_UNLIKELY(shared_random_state_mgr == NULL)) {
+    shared_random_state_mgr = config_mgr_new(&state_format);
+    config_mgr_freeze(shared_random_state_mgr);
+  }
+  return shared_random_state_mgr;
+}
+
+static void state_query_del_(sr_state_object_t obj_type, void *data);
+
 /* Return a string representation of a protocol phase. */
 STATIC const char *
 get_phase_str(sr_phase_t phase)
@@ -260,23 +278,22 @@ disk_state_free_(sr_disk_state_t *state)
   if (state == NULL) {
     return;
   }
-  config_free(&state_format, state);
+  config_free(get_srs_mgr(), state);
 }
 
 /* Allocate a new disk state, initialize it and return it. */
 static sr_disk_state_t *
 disk_state_new(time_t now)
 {
-  sr_disk_state_t *new_state = tor_malloc_zero(sizeof(*new_state));
+  sr_disk_state_t *new_state = config_new(get_srs_mgr());
 
-  new_state->magic_ = SR_DISK_STATE_MAGIC;
   new_state->Version = SR_PROTO_VERSION;
   new_state->TorVersion = tor_strdup(get_version());
   new_state->ValidUntil = get_state_valid_until_time(now);
   new_state->ValidAfter = now;
 
   /* Init config format. */
-  config_init(&state_format, new_state);
+  config_init(get_srs_mgr(), new_state);
   return new_state;
 }
 
@@ -344,12 +361,6 @@ disk_state_validate_cb(void *old_state, void *state, void *default_state,
   return 0;
 }
 
-static void
-disk_state_free_cb(void *state)
-{
-  disk_state_free_(state);
-}
-
 /* Parse the Commit line(s) in the disk state and translate them to the
  * the memory state. Return 0 on success else -1 on error. */
 static int
@@ -535,7 +546,7 @@ disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
   tor_assert(commit);
   tor_assert(line);
 
-  if (!tor_mem_is_zero(commit->encoded_reveal,
+  if (!fast_mem_is_zero(commit->encoded_reveal,
                        sizeof(commit->encoded_reveal))) {
     /* Add extra whitespace so we can format the line correctly. */
     tor_asprintf(&reveal_str, " %s", commit->encoded_reveal);
@@ -580,11 +591,12 @@ disk_state_reset(void)
   config_free_lines(sr_disk_state->ExtraLines);
   tor_free(sr_disk_state->TorVersion);
 
-  /* Clean up the struct */
-  memset(sr_disk_state, 0, sizeof(*sr_disk_state));
+  /* Clear other fields. */
+  sr_disk_state->ValidAfter = 0;
+  sr_disk_state->ValidUntil = 0;
+  sr_disk_state->Version = 0;
 
   /* Reset it with useful data */
-  sr_disk_state->magic_ = SR_DISK_STATE_MAGIC;
   sr_disk_state->TorVersion = tor_strdup(get_version());
 }
 
@@ -679,7 +691,7 @@ disk_state_load_from_disk_impl(const char *fname)
     }
 
     disk_state = disk_state_new(time(NULL));
-    config_assign(&state_format, disk_state, lines, 0, &errmsg);
+    config_assign(get_srs_mgr(), disk_state, lines, 0, &errmsg);
     config_free_lines(lines);
     if (errmsg) {
       log_warn(LD_DIR, "SR: Reading state error: %s", errmsg);
@@ -732,7 +744,7 @@ disk_state_save_to_disk(void)
   /* Make sure that our disk state is up to date with our memory state
    * before saving it to disk. */
   disk_state_update();
-  state = config_dump(&state_format, NULL, sr_disk_state, 0, 0);
+  state = config_dump(get_srs_mgr(), NULL, sr_disk_state, 0, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&content,
                "# Tor shared random state file last generated on %s "
@@ -833,6 +845,9 @@ state_query_get_commit(const char *rsa_fpr)
 static void *
 state_query_get_(sr_state_object_t obj_type, const void *data)
 {
+  if (BUG(!sr_state))
+    return NULL;
+
   void *obj = NULL;
 
   switch (obj_type) {
@@ -861,23 +876,44 @@ state_query_get_(sr_state_object_t obj_type, const void *data)
 }
 
 /* Helper function: This handles the PUT state action using an
- * <b>obj_type</b> and <b>data</b> needed for the action. */
+ * <b>obj_type</b> and <b>data</b> needed for the action.
+ * PUT frees the previous data before replacing it, if needed. */
 static void
 state_query_put_(sr_state_object_t obj_type, void *data)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
     sr_commit_t *commit = data;
     tor_assert(commit);
+    /* commit_add_to_state() frees the old commit, if there is one */
     commit_add_to_state(commit, sr_state);
     break;
   }
   case SR_STATE_OBJ_CURSRV:
-    sr_state->current_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->current_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_CURSRV, NULL);
+      sr_state->current_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_PREVSRV:
-    sr_state->previous_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->previous_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_PREVSRV, NULL);
+      sr_state->previous_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_VALID_AFTER:
     sr_state->valid_after = *((time_t *) data);
@@ -897,6 +933,9 @@ state_query_put_(sr_state_object_t obj_type, void *data)
 static void
 state_query_del_all_(sr_state_object_t obj_type)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
@@ -907,7 +946,7 @@ state_query_del_all_(sr_state_object_t obj_type)
     } DIGESTMAP_FOREACH_END;
     break;
   }
-  /* The following object are _NOT_ suppose to be removed. */
+  /* The following objects are _NOT_ supposed to be removed. */
   case SR_STATE_OBJ_CURSRV:
   case SR_STATE_OBJ_PREVSRV:
   case SR_STATE_OBJ_PHASE:
@@ -925,6 +964,9 @@ state_query_del_(sr_state_object_t obj_type, void *data)
 {
   (void) data;
 
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_PREVSRV:
     tor_free(sr_state->previous_srv);
@@ -999,16 +1041,16 @@ state_del_previous_srv(void)
   state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL);
 }
 
-/* Rotate SRV value by freeing the previous value, assigning the current
- * value to the previous one and nullifying the current one. */
+/* Rotate SRV value by setting the previous SRV to the current SRV, and
+ * clearing the current SRV. */
 STATIC void
 state_rotate_srv(void)
 {
   /* First delete previous SRV from the state. Object will be freed. */
   state_del_previous_srv();
-  /* Set previous SRV with the current one. */
-  sr_state_set_previous_srv(sr_state_get_current_srv());
-  /* Nullify the current srv. */
+  /* Set previous SRV to a copy of the current one. */
+  sr_state_set_previous_srv(sr_srv_dup(sr_state_get_current_srv()));
+  /* Free and NULL the current srv. */
   sr_state_set_current_srv(NULL);
 }
 
@@ -1024,12 +1066,15 @@ sr_state_set_valid_after(time_t valid_after)
 sr_phase_t
 sr_state_get_phase(void)
 {
-  void *ptr;
+  void *ptr=NULL;
   state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PHASE, NULL, &ptr);
+  tor_assert(ptr);
   return *(sr_phase_t *) ptr;
 }
 
-/* Return the previous SRV value from our state. Value CAN be NULL. */
+/* Return the previous SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_previous_srv(void)
 {
@@ -1048,7 +1093,9 @@ sr_state_set_previous_srv(const sr_srv_t *srv)
               NULL);
 }
 
-/* Return the current SRV value from our state. Value CAN be NULL. */
+/* Return the current SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_current_srv(void)
 {
@@ -1240,6 +1287,7 @@ sr_state_free_all(void)
   /* Nullify our global state. */
   sr_state = NULL;
   sr_disk_state = NULL;
+  config_mgr_free(shared_random_state_mgr);
 }
 
 /* Save our current state in memory to disk. */
diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h
index 92acdf1157..7869f92b4f 100644
--- a/src/feature/dirauth/vote_microdesc_hash_st.h
+++ b/src/feature/dirauth/vote_microdesc_hash_st.h
@@ -18,5 +18,5 @@ struct vote_microdesc_hash_t {
   char *microdesc_hash_line;
 };
 
-#endif
+#endif /* !defined(VOTE_MICRODESC_HASH_ST_H) */
 
diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c
index 54c70b989a..f552af98c4 100644
--- a/src/feature/dirauth/voteflags.c
+++ b/src/feature/dirauth/voteflags.c
@@ -29,6 +29,7 @@
 
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
 #include "lib/container/order.h"
@@ -95,7 +96,7 @@ real_uptime(const routerinfo_t *router, time_t now)
  */
 static int
 dirserv_thinks_router_is_unreliable(time_t now,
-                                    routerinfo_t *router,
+                                    const routerinfo_t *router,
                                     int need_uptime, int need_capacity)
 {
   if (need_uptime) {
@@ -238,7 +239,7 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
   uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
   long *tks;
   double *mtbfs, *wfus;
-  smartlist_t *nodelist;
+  const smartlist_t *nodelist;
   time_t now = time(NULL);
   const or_options_t *options = get_options();
 
@@ -531,38 +532,45 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
   node->is_running = answer;
 }
 
-/** Extract status information from <b>ri</b> and from other authority
- * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
- * set.
- *
- * We assume that ri-\>is_running has already been set, e.g. by
- *   dirserv_set_router_is_running(ri, now);
+/* Check <b>node</b> and <b>ri</b> on whether or not we should publish a
+ * relay's IPv6 addresses. */
+static int
+should_publish_node_ipv6(const node_t *node, const routerinfo_t *ri,
+                         time_t now)
+{
+  const or_options_t *options = get_options();
+
+  return options->AuthDirHasIPv6Connectivity == 1 &&
+    !tor_addr_is_null(&ri->ipv6_addr) &&
+    ((node->last_reachable6 >= now - REACHABLE_TIMEOUT) ||
+     router_is_me(ri));
+}
+
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>, as per
+ * <b>set_routerstatus_from_routerinfo</b>.  Additionally, sets information
+ * in from the authority subsystem.
  */
 void
-set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                 node_t *node,
-                                 routerinfo_t *ri,
-                                 time_t now,
-                                 int listbadexits)
+dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                         node_t *node,
+                                         const routerinfo_t *ri,
+                                         time_t now,
+                                         int listbadexits)
 {
   const or_options_t *options = get_options();
   uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
 
-  memset(rs, 0, sizeof(routerstatus_t));
-
-  rs->is_authority =
-    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
-
-  /* Already set by compute_performance_thresholds. */
-  rs->is_exit = node->is_exit;
-  rs->is_stable = node->is_stable =
-    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
-  rs->is_fast = node->is_fast =
-    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
-  rs->is_flagged_running = node->is_running; /* computed above */
+  /* Set these flags so that set_routerstatus_from_routerinfo can copy them.
+   */
+  node->is_stable = !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
+  node->is_fast = !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
+  node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now);
 
-  rs->is_valid = node->is_valid;
+  set_routerstatus_from_routerinfo(rs, node, ri);
 
+  /* Override rs->is_possible_guard. */
   if (node->is_fast && node->is_stable &&
       ri->supports_tunnelled_dir_requests &&
       ((options->AuthDirGuardBWGuarantee &&
@@ -578,29 +586,16 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
     rs->is_possible_guard = 0;
   }
 
+  /* Override rs->is_bad_exit */
   rs->is_bad_exit = listbadexits && node->is_bad_exit;
-  rs->is_hs_dir = node->is_hs_dir =
-    dirserv_thinks_router_is_hs_dir(ri, node, now);
-
-  rs->is_named = rs->is_unnamed = 0;
-
-  rs->published_on = ri->cache_info.published_on;
-  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
-  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
-         DIGEST_LEN);
-  rs->addr = ri->addr;
-  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
-  rs->or_port = ri->or_port;
-  rs->dir_port = ri->dir_port;
-  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
-  if (options->AuthDirHasIPv6Connectivity == 1 &&
-      !tor_addr_is_null(&ri->ipv6_addr) &&
-      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
-    /* We're configured as having IPv6 connectivity. There's an IPv6
-       OR port and it's reachable so copy it to the routerstatus.  */
-    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
-    rs->ipv6_orport = ri->ipv6_orport;
-  } else {
+
+  /* Set rs->is_staledesc. */
+  rs->is_staledesc =
+    (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now;
+
+  if (! should_publish_node_ipv6(node, ri, now)) {
+    /* We're not configured as having IPv6 connectivity or the node isn't:
+     * zero its IPv6 information. */
     tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
     rs->ipv6_orport = 0;
   }
@@ -642,3 +637,20 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs)
     rs->is_hs_dir = 0;
   }
 }
+
+/** Use dirserv_set_router_is_running() to set bridges as running if they're
+ * reachable.
+ *
+ * This function is called from set_bridge_running_callback() when running as
+ * a bridge authority.
+ */
+void
+dirserv_set_bridges_running(time_t now)
+{
+  routerlist_t *rl = router_get_routerlist();
+
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+    if (ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      dirserv_set_router_is_running(ri, now);
+  } SMARTLIST_FOREACH_END(ri);
+}
diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h
index aa7b6ed082..c4f36e7817 100644
--- a/src/feature/dirauth/voteflags.h
+++ b/src/feature/dirauth/voteflags.h
@@ -12,20 +12,28 @@
 #ifndef TOR_VOTEFLAGS_H
 #define TOR_VOTEFLAGS_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
 char *dirserv_get_flag_thresholds_line(void);
 void dirserv_compute_bridge_flag_thresholds(void);
 int running_long_enough_to_decide_unreachable(void);
 
-void set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                      node_t *node,
-                                      routerinfo_t *ri, time_t now,
-                                      int listbadexits);
+void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                              node_t *node,
+                                              const routerinfo_t *ri,
+                                              time_t now,
+                                              int listbadexits);
 
 void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+void dirserv_set_bridges_running(time_t now);
 
 #ifdef VOTEFLAGS_PRIVATE
+/** Any descriptor older than this age causes the authorities to set the
+ * StaleDesc flag. */
+#define DESC_IS_STALE_INTERVAL (18*60*60)
 STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
-#endif
+#endif /* defined(VOTEFLAGS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_VOTEFLAGS_H) */
diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h
index 71dca8c3a2..a28802f905 100644
--- a/src/feature/dircache/cached_dir_st.h
+++ b/src/feature/dircache/cached_dir_st.h
@@ -21,5 +21,5 @@ struct cached_dir_t {
   int refcnt; /**< Reference count for this cached_dir_t. */
 };
 
-#endif
+#endif /* !defined(CACHED_DIR_ST_H) */
 
diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c
index cf4fe8701d..2ec9981c03 100644
--- a/src/feature/dircache/conscache.c
+++ b/src/feature/dircache/conscache.c
@@ -92,7 +92,7 @@ consensus_cache_open(const char *subdir, int max_entries)
    */
 #define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
   storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
-#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
+#else /* !defined(MUST_UNMAP_TO_UNLINK) */
   /* Otherwise, we can just tell the storagedir to use the same limits
    * as this cache. */
   storagedir_max_entries = max_entries;
diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c
index 025361fa60..397efa0341 100644
--- a/src/feature/dircache/consdiffmgr.c
+++ b/src/feature/dircache/consdiffmgr.c
@@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = {
 
 static int consdiffmgr_ensure_space_for_files(int n);
 static int consensus_queue_compression_work(const char *consensus,
+                                            size_t consensus_len,
                                             const networkstatus_t *as_parsed);
 static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
                                           consensus_cache_entry_t *diff_to);
@@ -509,8 +510,25 @@ get_max_age_to_cache(void)
                                         MAX_MAX_AGE_TO_CACHE);
 }
 
+#ifdef TOR_UNIT_TESTS
+/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For
+ * testing. */
+int
+consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                  const networkstatus_t *as_parsed)
+{
+  size_t len = strlen(consensus);
+  /* make a non-nul-terminated copy so that we can have a better chance
+   * of catching errors. */
+  char *ctmp = tor_memdup(consensus, len);
+  int r = consdiffmgr_add_consensus(ctmp, len, as_parsed);
+  tor_free(ctmp);
+  return r;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /**
- * Given a string containing a networkstatus consensus, and the results of
+ * Given a buffer containing a networkstatus consensus, and the results of
  * having parsed that consensus, add that consensus to the cache if it is not
  * already present and not too old.  Create new consensus diffs from or to
  * that consensus as appropriate.
@@ -519,6 +537,7 @@ get_max_age_to_cache(void)
  */
 int
 consdiffmgr_add_consensus(const char *consensus,
+                          size_t consensus_len,
                           const networkstatus_t *as_parsed)
 {
   if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
@@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus,
   }
 
   /* We don't have it. Add it to the cache. */
-  return consensus_queue_compression_work(consensus, as_parsed);
+  return consensus_queue_compression_work(consensus, consensus_len, as_parsed);
 }
 
 /**
@@ -1387,19 +1406,21 @@ typedef struct consensus_diff_worker_job_t {
 } consensus_diff_worker_job_t;
 
 /** Given a consensus_cache_entry_t, check whether it has a label claiming
- * that it was compressed.  If so, uncompress its contents into <b>out</b> and
- * set <b>outlen</b> to hold their size.  If not, just copy the body into
- * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
- * -1 on failure.
- *
- * In all cases, the output is nul-terminated. */
+ * that it was compressed.  If so, uncompress its contents into *<b>out</b> and
+ * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer
+ * that the caller will need to free.  If not, just set *<b>out</b> and
+ * <b>outlen</b> to its extent in memory.  Return 0 on success, -1 on failure.
+ **/
 STATIC int
-uncompress_or_copy(char **out, size_t *outlen,
-                   consensus_cache_entry_t *ent)
+uncompress_or_set_ptr(const char **out, size_t *outlen,
+                      char **owned_out,
+                      consensus_cache_entry_t *ent)
 {
   const uint8_t *body;
   size_t bodylen;
 
+  *owned_out = NULL;
+
   if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
     return -1;
 
@@ -1410,8 +1431,17 @@ uncompress_or_copy(char **out, size_t *outlen,
   if (lv_compression)
     method = compression_method_get_by_name(lv_compression);
 
-  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+  int rv;
+  if (method == NO_METHOD) {
+    *out = (const char *)body;
+    *outlen = bodylen;
+    rv = 0;
+  } else {
+    rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen,
                         method, 1, LOG_WARN);
+    *out = *owned_out;
+  }
+  return rv;
 }
 
 /**
@@ -1478,16 +1508,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
 
   char *consensus_diff;
   {
-    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    const char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    char *owned1 = NULL, *owned2 = NULL;
     size_t diff_from_nt_len, diff_to_nt_len;
 
-    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
-                           job->diff_from) < 0) {
+    if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1,
+                              job->diff_from) < 0) {
       return WQ_RPL_REPLY;
     }
-    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
-                           job->diff_to) < 0) {
-      tor_free(diff_from_nt);
+    if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2,
+                              job->diff_to) < 0) {
+      tor_free(owned1);
       return WQ_RPL_REPLY;
     }
     tor_assert(diff_from_nt);
@@ -1496,9 +1527,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
     // XXXX ugh; this is going to calculate the SHA3 of both its
     // XXXX inputs again, even though we already have that. Maybe it's time
     // XXXX to change the API here?
-    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
-    tor_free(diff_from_nt);
-    tor_free(diff_to_nt);
+    consensus_diff = consensus_diff_generate(diff_from_nt,
+                                             diff_from_nt_len,
+                                             diff_to_nt,
+                                             diff_to_nt_len);
+    tor_free(owned1);
+    tor_free(owned2);
   }
   if (!consensus_diff) {
     /* Couldn't generate consensus; we'll leave the reply blank. */
@@ -1746,8 +1780,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_)
                           (const uint8_t *)consensus, bodylen);
   {
     const char *start, *end;
-    if (router_get_networkstatus_v3_signed_boundaries(consensus,
-                                                        &start, &end) < 0) {
+    if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen,
+                                                      &start, &end) < 0) {
       start = consensus;
       end = consensus+bodylen;
     }
@@ -1811,14 +1845,15 @@ static int background_compression = 0;
  */
 static int
 consensus_queue_compression_work(const char *consensus,
+                                 size_t consensus_len,
                                  const networkstatus_t *as_parsed)
 {
   tor_assert(consensus);
   tor_assert(as_parsed);
 
   consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
-  job->consensus = tor_strdup(consensus);
-  job->consensus_len = strlen(consensus);
+  job->consensus = tor_memdup_nulterm(consensus, consensus_len);
+  job->consensus_len = strlen(job->consensus);
   job->flavor = as_parsed->flavor;
 
   char va_str[ISO_TIME_LEN+1];
diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h
index 39e8fa31cb..b1b3323b6c 100644
--- a/src/feature/dircache/consdiffmgr.h
+++ b/src/feature/dircache/consdiffmgr.h
@@ -22,6 +22,7 @@ typedef struct consdiff_cfg_t {
 struct consensus_cache_entry_t; // from conscache.h
 
 int consdiffmgr_add_consensus(const char *consensus,
+                              size_t consensus_len,
                               const networkstatus_t *as_parsed);
 
 consdiff_status_t consdiffmgr_find_consensus(
@@ -68,8 +69,14 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
 STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
                                     consensus_cache_entry_t *ent,
                                     const char *label);
-STATIC int uncompress_or_copy(char **out, size_t *outlen,
-                              consensus_cache_entry_t *ent);
+STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen,
+                                 char **owned_out,
+                                 consensus_cache_entry_t *ent);
 #endif /* defined(CONSDIFFMGR_PRIVATE) */
 
+#ifdef TOR_UNIT_TESTS
+int consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                      const networkstatus_t *as_parsed);
+#endif
+
 #endif /* !defined(TOR_CONSDIFFMGR_H) */
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
index e8cb284165..59cdcc5e02 100644
--- a/src/feature/dircache/dircache.c
+++ b/src/feature/dircache/dircache.c
@@ -49,7 +49,8 @@
 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
 #define ROBOTS_CACHE_LIFETIME (24*60*60)
 #define MICRODESC_CACHE_LIFETIME (48*60*60)
-
+/* Bandwidth files change every hour. */
+#define BANDWIDTH_CACHE_LIFETIME (30*60)
 /** Parse an HTTP request string <b>headers</b> of the form
  * \verbatim
  * "\%s [http[s]://]\%s HTTP/1..."
@@ -123,7 +124,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
                            long cache_lifetime)
 {
   char date[RFC1123_TIME_LEN+1];
-  time_t now = time(NULL);
+  time_t now = approx_time();
   buf_t *buf = buf_new_with_capacity(1024);
 
   tor_assert(conn);
@@ -166,22 +167,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
   buf_free(buf);
 }
 
-/** As write_http_response_header_impl, but sets encoding and content-typed
- * based on whether the response will be <b>compressed</b> or not. */
+/** As write_http_response_header_impl, but translates method into
+ * encoding */
 static void
 write_http_response_headers(dir_connection_t *conn, ssize_t length,
                             compress_method_t method,
                             const char *extra_headers, long cache_lifetime)
 {
-  const char *methodname = compression_method_get_name(method);
-  const char *doctype;
-  if (method == NO_METHOD)
-    doctype = "text/plain";
-  else
-    doctype = "application/octet-stream";
   write_http_response_header_impl(conn, length,
-                                  doctype,
-                                  methodname,
+                                  "text/plain",
+                                  compression_method_get_name(method),
                                   extra_headers,
                                   cache_lifetime);
 }
@@ -357,12 +352,15 @@ static int handle_get_robots(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
                                 const get_handler_args_t *args);
+static int handle_get_next_bandwidth(dir_connection_t *conn,
+                                     const get_handler_args_t *args);
 
 /** Table for handling GET requests. */
 static const url_table_ent_t url_table[] = {
   { "/tor/", 0, handle_get_frontpage },
   { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
   { "/tor/status-vote/current/", 1, handle_get_status_vote },
+  { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth },
   { "/tor/status-vote/next/", 1, handle_get_status_vote },
   { "/tor/micro/d/", 1, handle_get_microdesc },
   { "/tor/server/", 1, handle_get_descriptor },
@@ -495,28 +493,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
 }
 
 /** Warn that the cached consensus <b>consensus</b> of type
- * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
- * warning to avoid logging an entry on every request.
+ * <b>flavor</b> too new or too old, based on <b>is_too_new</b>,
+ * and will not be served to clients. Rate-limit the warning to avoid logging
+ * an entry on every request.
  */
 static void
-warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
-                          const char *flavor, time_t now)
+warn_consensus_is_not_reasonably_live(
+                          const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now, bool is_too_new)
 {
-#define TOO_OLD_WARNING_INTERVAL (60*60)
-  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60)
+  static ratelim_t warned[2] = { RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL),
+                                RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL) };
   char timestamp[ISO_TIME_LEN+1];
-  time_t valid_until;
-  char *dupes;
+  /* valid_after if is_too_new, valid_until if !is_too_new */
+  time_t valid_time = 0;
+  char *dupes = NULL;
 
-  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
-    return;
-
-  if ((dupes = rate_limit_log(&warned, now))) {
-    format_local_iso_time(timestamp, valid_until);
-    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
-             "serve it to clients. It was valid until %s local time and we "
-             "continued to serve it for up to 24 hours after it expired.%s",
-             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+  if (is_too_new) {
+    if (consensus_cache_entry_get_valid_after(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[1], now);
+  } else {
+    if (consensus_cache_entry_get_valid_until(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[0], now);
+  }
+
+  if (dupes) {
+    format_local_iso_time(timestamp, valid_time);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not "
+             "serve it to clients. It was valid %s %s local time and we "
+             "continued to serve it for up to 24 hours %s.%s",
+             flavor ? flavor : "",
+             flavor ? " " : "",
+             is_too_new ? "new" : "old",
+             is_too_new ? "after" : "until",
+             timestamp,
+             is_too_new ? "before it was valid" : "after it expired",
+             dupes);
     tor_free(dupes);
   }
 }
@@ -859,7 +876,6 @@ handle_get_current_consensus(dir_connection_t *conn,
 
   if (req.diff_only && !cached_consensus) {
     write_short_http_response(conn, 404, "No such diff available");
-    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -870,19 +886,30 @@ handle_get_current_consensus(dir_connection_t *conn,
                                            &compression_used);
   }
 
-  time_t fresh_until, valid_until;
-  int have_fresh_until = 0, have_valid_until = 0;
+  time_t valid_after, fresh_until, valid_until;
+  int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0;
   if (cached_consensus) {
+    have_valid_after =
+      !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after);
     have_fresh_until =
       !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
     have_valid_until =
       !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
   }
 
-  if (cached_consensus && have_valid_until &&
+  if (cached_consensus && have_valid_after &&
+      !networkstatus_valid_after_is_reasonably_live(valid_after, now)) {
+    write_short_http_response(conn, 404, "Consensus is too new");
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          1);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (
+      cached_consensus && have_valid_until &&
       !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
     write_short_http_response(conn, 404, "Consensus is too old");
-    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          0);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -924,7 +951,7 @@ handle_get_current_consensus(dir_connection_t *conn,
     goto done;
   }
 
-  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+  if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
     log_debug(LD_DIRSERV,
               "Client asked for network status lists, but we've been "
               "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1033,7 +1060,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
         }
       });
 
-    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), estimated_len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto vote_done;
     }
@@ -1045,13 +1072,11 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
       if (compress_method != NO_METHOD) {
         conn->compress_state = tor_compress_new(1, compress_method,
                            choose_compression_level(estimated_len));
-        SMARTLIST_FOREACH(items, const char *, c,
-                 connection_buf_add_compress(c, strlen(c), conn, 0));
-        connection_buf_add_compress("", 0, conn, 1);
-      } else {
-        SMARTLIST_FOREACH(items, const char *, c,
-                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
       }
+
+      SMARTLIST_FOREACH(items, const char *, c,
+                        connection_dir_buf_add(c, strlen(c), conn,
+                                               c_sl_idx == c_sl_len - 1));
     } else {
       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
           connection_buf_add(compress_method != NO_METHOD ?
@@ -1094,7 +1119,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
       write_short_http_response(conn, 404, "Not found");
       goto done;
     }
-    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
       log_info(LD_DIRSERV,
                "Client asked for server descriptors, but we've been "
                "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1192,7 +1217,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
         msg = "Not found";
       write_short_http_response(conn, 404, msg);
     } else {
-      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
         log_info(LD_DIRSERV,
                  "Client asked for server descriptors, but we've been "
                  "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1288,9 +1313,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
                       len += c->cache_info.signed_descriptor_len);
 
-    if (global_write_bucket_low(TO_CONN(conn),
-                                compress_method != NO_METHOD ? len/2 : len,
-                                2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto keys_done;
     }
@@ -1302,19 +1326,13 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     if (compress_method != NO_METHOD) {
       conn->compress_state = tor_compress_new(1, compress_method,
                                               choose_compression_level(len));
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add_compress(
-                c->cache_info.signed_descriptor_body,
-                c->cache_info.signed_descriptor_len,
-                conn, 0));
-      connection_buf_add_compress("", 0, conn, 1);
-    } else {
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add(c->cache_info.signed_descriptor_body,
-                                    c->cache_info.signed_descriptor_len,
-                                    TO_CONN(conn)));
     }
-  keys_done:
+
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+          connection_dir_buf_add(c->cache_info.signed_descriptor_body,
+                                 c->cache_info.signed_descriptor_len,
+                                 conn, c_sl_idx == c_sl_len - 1));
+ keys_done:
     smartlist_free(certs);
     goto done;
   }
@@ -1371,9 +1389,11 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn,
   const char *pubkey_str = NULL;
   const char *url = args->url;
 
-  /* Reject unencrypted dir connections */
-  if (!connection_dir_is_encrypted(conn)) {
-    write_short_http_response(conn, 404, "Not found");
+  /* Reject non anonymous dir connections (which also tests if encrypted). We
+   * do not allow single hop clients to query an HSDir. */
+  if (!connection_dir_is_anonymous(conn)) {
+    write_short_http_response(conn, 503,
+                              "Rejecting single hop HS v3 descriptor request");
     goto done;
   }
 
@@ -1438,6 +1458,39 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
   return 0;
 }
 
+/** Helper function for GET the bandwidth file used for the next vote */
+static int
+handle_get_next_bandwidth(dir_connection_t *conn,
+                          const get_handler_args_t *args)
+{
+  log_debug(LD_DIR, "Getting next bandwidth.");
+  const or_options_t *options = get_options();
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+
+  if (options->V3BandwidthsFile) {
+    char *bandwidth = read_file_to_str(options->V3BandwidthsFile,
+                                       RFTS_IGNORE_MISSING, NULL);
+    if (bandwidth != NULL) {
+      ssize_t len = strlen(bandwidth);
+      write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len,
+                                 compress_method, BANDWIDTH_CACHE_LIFETIME);
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(len/2));
+        log_debug(LD_DIR, "Compressing bandwidth file.");
+      } else {
+        log_debug(LD_DIR, "Not compressing bandwidth file.");
+      }
+      connection_dir_buf_add((const char*)bandwidth, len, conn, 1);
+      tor_free(bandwidth);
+      return 0;
+    }
+  }
+  write_short_http_response(conn, 404, "Not found");
+  return 0;
+}
+
 /** Helper function for GET robots.txt or /tor/robots.txt */
 static int
 handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
@@ -1580,10 +1633,15 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     goto done;
   }
 
-  /* Handle HS descriptor publish request. */
-  /* XXX: This should be disabled with a consensus param until we want to
-   * the prop224 be deployed and thus use. */
-  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+  /* Handle HS descriptor publish request. We force an anonymous connection
+   * (which also tests for encrypted). We do not allow single-hop client to
+   * post a descriptor onto an HSDir. */
+  if (!strcmpstart(url, "/tor/hs/")) {
+    if (!connection_dir_is_anonymous(conn)) {
+      write_short_http_response(conn, 503,
+                                "Rejecting single hop HS descriptor post");
+      goto done;
+    }
     const char *msg = "HS descriptor stored successfully.";
 
     /* We most probably have a publish request for an HS descriptor. */
@@ -1608,8 +1666,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     const char *msg = "[None]";
     uint8_t purpose = authdir_mode_bridge(options) ?
                       ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
-                                             conn->base_.address, &msg);
+    was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len,
+                                           purpose, conn->base_.address, &msg);
     tor_assert(msg);
 
     if (r == ROUTER_ADDED_SUCCESSFULLY) {
diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h
index 236ea649ef..de0d205f6a 100644
--- a/src/feature/dircache/dircache.h
+++ b/src/feature/dircache/dircache.h
@@ -38,6 +38,6 @@ STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
                                       const char **end_pos);
 
 STATIC unsigned parse_accept_encoding_header(const char *h);
-#endif
+#endif /* defined(DIRCACHE_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCACHE_H) */
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
index 213c490314..79400bf15f 100644
--- a/src/feature/dircache/dirserv.c
+++ b/src/feature/dircache/dirserv.c
@@ -234,6 +234,7 @@ free_cached_dir_(void *_d)
  * validation is performed. */
 void
 dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
+                                           size_t networkstatus_len,
                                            const char *flavor_name,
                                            const common_digests_t *digests,
                                            const uint8_t *sha3_as_signed,
@@ -244,7 +245,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
   if (!cached_consensuses)
     cached_consensuses = strmap_new();
 
-  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
+  new_networkstatus =
+    new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len),
+                   published);
   memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
   memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
          DIGEST256_LEN);
@@ -580,11 +583,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
       /* Absent objects count as "done". */
       return SRFS_DONE;
     }
-    if (conn->compress_state) {
-      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
-    } else {
-      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add((const char*)body, bodylen, conn, 0);
+
     return SRFS_DONE;
   } else {
     cached_dir_t *cached = spooled->cached_dir_ref;
@@ -619,14 +620,10 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
     if (BUG(remaining < 0))
       return SRFS_ERR;
     ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
-    if (conn->compress_state) {
-      connection_buf_add_compress(
-              ptr + spooled->cached_dir_offset,
-              bytes, conn, 0);
-    } else {
-      connection_buf_add(ptr + spooled->cached_dir_offset,
-                              bytes, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add(ptr + spooled->cached_dir_offset,
+                           bytes, conn, 0);
+
     spooled->cached_dir_offset += bytes;
     if (spooled->cached_dir_offset >= (off_t)total_len) {
       return SRFS_DONE;
diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h
index 890b10fd80..7f944459da 100644
--- a/src/feature/dircache/dirserv.h
+++ b/src/feature/dircache/dirserv.h
@@ -84,6 +84,7 @@ int directory_too_idle_to_fetch_descriptors(const or_options_t *options,
 
 cached_dir_t *dirserv_get_consensus(const char *flavor_name);
 void dirserv_set_cached_consensus_networkstatus(const char *consensus,
+                                              size_t consensus_len,
                                               const char *flavor_name,
                                               const common_digests_t *digests,
                                               const uint8_t *sha3_as_signed,
diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h
index 2f5706cdd9..8e35532435 100644
--- a/src/feature/dirclient/dir_server_st.h
+++ b/src/feature/dirclient/dir_server_st.h
@@ -51,4 +51,4 @@ struct dir_server_t {
                                **/
 };
 
-#endif
+#endif /* !defined(DIR_SERVER_ST_H) */
diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c
index 6725fc3369..fa82bdc1e7 100644
--- a/src/feature/dirclient/dirclient.c
+++ b/src/feature/dirclient/dirclient.c
@@ -14,7 +14,7 @@
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/shared_random.h"
@@ -2205,24 +2205,31 @@ handle_response_fetch_consensus(dir_connection_t *conn,
   if (looks_like_a_consensus_diff(body, body_len)) {
     /* First find our previous consensus. Maybe it's in ram, maybe not. */
     cached_dir_t *cd = dirserv_get_consensus(flavname);
-    const char *consensus_body;
-    char *owned_consensus = NULL;
+    const char *consensus_body = NULL;
+    size_t consensus_body_len;
+    tor_mmap_t *mapped_consensus = NULL;
     if (cd) {
       consensus_body = cd->dir;
+      consensus_body_len = cd->dir_len;
     } else {
-      owned_consensus = networkstatus_read_cached_consensus(flavname);
-      consensus_body = owned_consensus;
+      mapped_consensus = networkstatus_map_cached_consensus(flavname);
+      if (mapped_consensus) {
+        consensus_body = mapped_consensus->data;
+        consensus_body_len = mapped_consensus->size;
+      }
     }
     if (!consensus_body) {
       log_warn(LD_DIR, "Received a consensus diff, but we can't find "
                "any %s-flavored consensus in our current cache.",flavname);
+      tor_munmap_file(mapped_consensus);
       networkstatus_consensus_download_failed(0, flavname);
       // XXXX if this happens too much, see below
       return -1;
     }
 
-    new_consensus = consensus_diff_apply(consensus_body, body);
-    tor_free(owned_consensus);
+    new_consensus = consensus_diff_apply(consensus_body, consensus_body_len,
+                                         body, body_len);
+    tor_munmap_file(mapped_consensus);
     if (new_consensus == NULL) {
       log_warn(LD_DIR, "Could not apply consensus diff received from server "
                "'%s:%d'", conn->base_.address, conn->base_.port);
@@ -2244,7 +2251,9 @@ handle_response_fetch_consensus(dir_connection_t *conn,
     sourcename = "downloaded";
   }
 
-  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+  if ((r=networkstatus_set_current_consensus(consensus,
+                                             strlen(consensus),
+                                             flavname, 0,
                                              conn->identity_digest))<0) {
     log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
            "Unable to load %s consensus directory %s from "
@@ -2522,7 +2531,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn,
            conn->base_.port);
   tor_assert(conn->requested_resource &&
              !strcmpstart(conn->requested_resource, "d/"));
-  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  tor_assert_nonfatal(!fast_mem_is_zero(conn->identity_digest, DIGEST_LEN));
   which = smartlist_new();
   dir_split_resource_into_fingerprints(conn->requested_resource+2,
                                        which, NULL,
diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h
index 1a93265dc3..be4374c7cf 100644
--- a/src/feature/dirclient/dirclient.h
+++ b/src/feature/dirclient/dirclient.h
@@ -167,6 +167,6 @@ STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
 
 STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
                                      const char *resource);
-#endif
+#endif /* defined(DIRCLIENT_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCLIENT_H) */
diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h
index 99e0d0225b..681712b059 100644
--- a/src/feature/dirclient/dlstatus.h
+++ b/src/feature/dirclient/dlstatus.h
@@ -53,6 +53,6 @@ STATIC void next_random_exponential_delay_range(int *low_bound_out,
 /* no more than triple the previous delay */
 #define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
 
-#endif
+#endif /* defined(DLSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_DLSTATUS_H) */
diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h
index 11555a1dcc..39a5ad2860 100644
--- a/src/feature/dirclient/download_status_st.h
+++ b/src/feature/dirclient/download_status_st.h
@@ -61,5 +61,5 @@ struct download_status_t {
                         * only updated if backoff == 1 */
 };
 
-#endif
+#endif /* !defined(DOWNLOAD_STATUS_ST_H) */
 
diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c
index d0f7594ce3..8e93953f73 100644
--- a/src/feature/dircommon/consdiff.c
+++ b/src/feature/dircommon/consdiff.c
@@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest,(const char *cons,
+consensus_compute_digest,(const char *cons, size_t len,
                           consensus_digest_t *digest_out))
 {
   int r = crypto_digest256((char*)digest_out->sha3_256,
-                           cons, strlen(cons), DIGEST_SHA3_256);
+                           cons, len, DIGEST_SHA3_256);
   return r;
 }
 
@@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons,
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest_as_signed,(const char *cons,
+consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out))
 {
   return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
-                                                    cons);
+                                                    cons, len);
 }
 
 /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
@@ -1229,7 +1229,8 @@ consdiff_apply_diff(const smartlist_t *cons1,
   cons2_str = consensus_join_lines(cons2);
 
   consensus_digest_t cons2_digests;
-  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+  if (consensus_compute_digest(cons2_str, strlen(cons2_str),
+                               &cons2_digests) < 0) {
     /* LCOV_EXCL_START -- digest can't fail */
     log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
         "resulting from applying a consensus diff.");
@@ -1283,12 +1284,13 @@ consdiff_apply_diff(const smartlist_t *cons1,
  * generated cdlines will become invalid.
  */
 STATIC int
-consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+consensus_split_lines(smartlist_t *out,
+                      const char *s, size_t len,
+                      memarea_t *area)
 {
-  const char *end_of_str = s + strlen(s);
-  tor_assert(*end_of_str == '\0');
+  const char *end_of_str = s + len;
 
-  while (*s) {
+  while (s < end_of_str) {
     const char *eol = memchr(s, '\n', end_of_str - s);
     if (!eol) {
       /* File doesn't end with newline. */
@@ -1334,25 +1336,25 @@ consensus_join_lines(const smartlist_t *inp)
  * success, retun a newly allocated string containing that diff.  On failure,
  * return NULL. */
 char *
-consensus_diff_generate(const char *cons1,
-                        const char *cons2)
+consensus_diff_generate(const char *cons1, size_t cons1len,
+                        const char *cons2, size_t cons2len)
 {
   consensus_digest_t d1, d2;
   smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
   int r1, r2;
   char *result = NULL;
 
-  r1 = consensus_compute_digest_as_signed(cons1, &d1);
-  r2 = consensus_compute_digest(cons2, &d2);
+  r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1);
+  r2 = consensus_compute_digest(cons2, cons2len, &d2);
   if (BUG(r1 < 0 || r2 < 0))
     return NULL; // LCOV_EXCL_LINE
 
   memarea_t *area = memarea_new();
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, cons1, area) < 0)
+  if (consensus_split_lines(lines1, cons1, cons1len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, cons2, area) < 0)
+  if (consensus_split_lines(lines2, cons2, cons2len, area) < 0)
     goto done;
 
   result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
@@ -1375,7 +1377,9 @@ consensus_diff_generate(const char *cons1,
  * consensus.  On failure, return NULL. */
 char *
 consensus_diff_apply(const char *consensus,
-                     const char *diff)
+                     size_t consensus_len,
+                     const char *diff,
+                     size_t diff_len)
 {
   consensus_digest_t d1;
   smartlist_t *lines1 = NULL, *lines2 = NULL;
@@ -1383,15 +1387,15 @@ consensus_diff_apply(const char *consensus,
   char *result = NULL;
   memarea_t *area = memarea_new();
 
-  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1);
   if (BUG(r1 < 0))
     goto done;
 
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, consensus, area) < 0)
+  if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, diff, area) < 0)
+  if (consensus_split_lines(lines2, diff, diff_len, area) < 0)
     goto done;
 
   result = consdiff_apply_diff(lines1, lines2, &d1);
diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h
index 98217e6d46..b63fcb2cc6 100644
--- a/src/feature/dircommon/consdiff.h
+++ b/src/feature/dircommon/consdiff.h
@@ -7,10 +7,10 @@
 
 #include "core/or/or.h"
 
-char *consensus_diff_generate(const char *cons1,
-                              const char *cons2);
-char *consensus_diff_apply(const char *consensus,
-                           const char *diff);
+char *consensus_diff_generate(const char *cons1, size_t cons1len,
+                              const char *cons2, size_t cons2len);
+char *consensus_diff_apply(const char *consensus, size_t consensus_len,
+                           const char *diff, size_t diff_len);
 
 int looks_like_a_consensus_diff(const char *document, size_t len);
 
@@ -78,7 +78,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
 STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
                         const smartlist_slice_t *slice1,
                         const smartlist_slice_t *slice2);
-STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+STATIC int consensus_split_lines(smartlist_t *out,
+                                 const char *s, size_t len,
                                  struct memarea_t *area);
 STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
                                   const char *s);
@@ -86,10 +87,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
 STATIC int line_str_eq(const cdline_t *a, const char *b);
 
 MOCK_DECL(STATIC int,
-          consensus_compute_digest,(const char *cons,
+          consensus_compute_digest,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
-          consensus_compute_digest_as_signed,(const char *cons,
+          consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                               consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
           consensus_digest_eq,(const uint8_t *d1,
diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h
index 8c59cc7a46..a858560c29 100644
--- a/src/feature/dircommon/dir_connection_st.h
+++ b/src/feature/dircommon/dir_connection_st.h
@@ -64,4 +64,4 @@ struct dir_connection_t {
 #endif /* defined(MEASUREMENTS_21206) */
 };
 
-#endif
+#endif /* !defined(DIR_CONNECTION_ST_H) */
diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c
index 9e6f72e9ac..8e5b413326 100644
--- a/src/feature/dircommon/directory.c
+++ b/src/feature/dircommon/directory.c
@@ -7,6 +7,10 @@
 
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/channeltls.h"
 #include "feature/dircache/dircache.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -15,6 +19,10 @@
 #include "feature/stats/geoip_stats.h"
 #include "lib/compress/compress.h"
 
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 
@@ -167,6 +175,82 @@ connection_dir_is_encrypted(const dir_connection_t *conn)
   return TO_CONN(conn)->linked;
 }
 
+/** Return true iff the given directory connection <b>dir_conn</b> is
+ * anonymous, that is, it is on a circuit via a public relay and not directly
+ * from a client or bridge.
+ *
+ * For client circuits via relays: true for 2-hop+ paths.
+ * For client circuits via bridges: true for 3-hop+ paths.
+ *
+ * This first test if the connection is encrypted since it is a strong
+ * requirement for anonymity. */
+bool
+connection_dir_is_anonymous(const dir_connection_t *dir_conn)
+{
+  const connection_t *conn, *linked_conn;
+  const edge_connection_t *edge_conn;
+  const circuit_t *circ;
+
+  tor_assert(dir_conn);
+
+  if (!connection_dir_is_encrypted(dir_conn)) {
+    return false;
+  }
+
+  /*
+   * Buckle up, we'll do a deep dive into the connection in order to get the
+   * final connection channel of that connection in order to figure out if
+   * this is a client or relay link.
+   *
+   * We go: dir_conn -> linked_conn -> edge_conn -> on_circuit -> p_chan.
+   */
+
+  conn = TO_CONN(dir_conn);
+  linked_conn = conn->linked_conn;
+
+  /* The dir connection should be connected to an edge connection. It can not
+   * be closed or marked for close. */
+  if (linked_conn == NULL || linked_conn->magic != EDGE_CONNECTION_MAGIC ||
+      conn->linked_conn_is_closed || conn->linked_conn->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not linked to edge");
+    return false;
+  }
+
+  edge_conn = TO_EDGE_CONN((connection_t *) linked_conn);
+  circ = edge_conn->on_circuit;
+
+  /* Can't be a circuit we initiated and without a circuit, no channel. */
+  if (circ == NULL || CIRCUIT_IS_ORIGIN(circ)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not on OR circuit");
+    return false;
+  }
+
+  /* It is possible that the circuit was closed because one of the channel was
+   * closed or a DESTROY cell was received. Either way, this connection can
+   * not continue so return that it is not anonymous since we can not know for
+   * sure if it is. */
+  if (circ->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "circuit marked for close");
+    return false;
+  }
+
+  /* Get the previous channel to learn if it is a client or relay link. We
+   * BUG() because if the circuit is not mark for close, we ought to have a
+   * p_chan else we have a code flow issue. */
+  if (BUG(CONST_TO_OR_CIRCUIT(circ)->p_chan == NULL)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "no p_chan on circuit");
+    return false;
+  }
+
+  /* Will be true if the channel is an unauthenticated peer which is only true
+   * for clients and bridges. */
+  return !channel_is_client(CONST_TO_OR_CIRCUIT(circ)->p_chan);
+}
+
 /** Parse an HTTP request line at the start of a headers string.  On failure,
  * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
  * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h
index ba3f8c1b0e..4fc743ad3d 100644
--- a/src/feature/dircommon/directory.h
+++ b/src/feature/dircommon/directory.h
@@ -94,6 +94,7 @@ int parse_http_command(const char *headers,
 char *http_get_header(const char *headers, const char *which);
 
 int connection_dir_is_encrypted(const dir_connection_t *conn);
+bool connection_dir_is_anonymous(const dir_connection_t *conn);
 int connection_dir_reached_eof(dir_connection_t *conn);
 int connection_dir_process_inbuf(dir_connection_t *conn);
 int connection_dir_finished_flushing(dir_connection_t *conn);
diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h
index 47b90ab009..814a325314 100644
--- a/src/feature/dircommon/vote_timing_st.h
+++ b/src/feature/dircommon/vote_timing_st.h
@@ -20,5 +20,5 @@ struct vote_timing_t {
   int dist_delay;
 };
 
-#endif
+#endif /* !defined(VOTE_TIMING_ST_H) */
 
diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dircommon/voting_schedule.c
index 0a7476eda7..5576ec69f7 100644
--- a/src/feature/dircommon/voting_schedule.c
+++ b/src/feature/dircommon/voting_schedule.c
@@ -150,7 +150,7 @@ voting_schedule_get_next_valid_after_time(void)
   /* This is a safe guard in order to make sure that the voting schedule
    * static object is at least initialized. Using this function with a zeroed
    * voting schedule can lead to bugs. */
-  if (tor_mem_is_zero((const char *) &voting_schedule,
+  if (fast_mem_is_zero((const char *) &voting_schedule,
                       sizeof(voting_schedule))) {
     need_to_recalculate_voting_schedule = true;
     goto done; /* no need for next check if we have to recalculate anyway */
diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dircommon/voting_schedule.h
index bafd81184e..d78c7ee2da 100644
--- a/src/feature/dircommon/voting_schedule.h
+++ b/src/feature/dircommon/voting_schedule.h
@@ -61,5 +61,5 @@ time_t voting_schedule_get_start_of_next_interval(time_t now,
                                                   int offset);
 time_t voting_schedule_get_next_valid_after_time(void);
 
-#endif /* TOR_VOTING_SCHEDULE_H */
+#endif /* !defined(TOR_VOTING_SCHEDULE_H) */
 
diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c
index 1680bdbf30..8ba5a53981 100644
--- a/src/feature/dirparse/authcert_parse.c
+++ b/src/feature/dirparse/authcert_parse.c
@@ -24,7 +24,8 @@ static token_rule_t dir_key_certificate_table[] = {
 /** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
  * the first character after the certificate. */
 authority_cert_t *
-authority_cert_parse_from_string(const char *s, const char **end_of_string)
+authority_cert_parse_from_string(const char *s, size_t maxlen,
+                                 const char **end_of_string)
 {
   /** Reject any certificate at least this big; it is probably an overflow, an
    * attack, a bug, or some other nonsense. */
@@ -35,24 +36,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
   char digest[DIGEST_LEN];
   directory_token_t *tok;
   char fp_declared[DIGEST_LEN];
-  char *eos;
+  const char *eos;
   size_t len;
   int found;
   memarea_t *area = NULL;
+  const char *end_of_s = s + maxlen;
   const char *s_dup = s;
 
-  s = eat_whitespace(s);
-  eos = strstr(s, "\ndir-key-certification");
+  s = eat_whitespace_eos(s, end_of_s);
+  eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification");
   if (! eos) {
     log_warn(LD_DIR, "No signature found on key certificate");
     return NULL;
   }
-  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
+  eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n");
   if (! eos) {
     log_warn(LD_DIR, "No end-of-signature found on key certificate");
     return NULL;
   }
-  eos = strchr(eos+2, '\n');
+  eos = memchr(eos+2, '\n', end_of_s - (eos+2));
   tor_assert(eos);
   ++eos;
   len = eos - s;
@@ -69,7 +71,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
     log_warn(LD_DIR, "Error tokenizing key certificate");
     goto err;
   }
-  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
+  if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version",
                            "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
     goto err;
   tok = smartlist_get(tokens, 0);
diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h
index ca475ad0e3..800631c3de 100644
--- a/src/feature/dirparse/authcert_parse.h
+++ b/src/feature/dirparse/authcert_parse.h
@@ -13,6 +13,7 @@
 #define TOR_AUTHCERT_PARSE_H
 
 authority_cert_t *authority_cert_parse_from_string(const char *s,
+                                                   size_t maxlen,
                                                    const char **end_of_string);
 
 #endif /* !defined(TOR_AUTHCERT_PARSE_H) */
diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c
index 5a75af3994..4bb4db7821 100644
--- a/src/feature/dirparse/microdesc_parse.c
+++ b/src/feature/dirparse/microdesc_parse.c
@@ -18,6 +18,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/relay/router.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -32,7 +33,7 @@ static token_rule_t microdesc_token_table[] = {
   T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
   T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
   T0N("a",                     K_A,                GE(1),       NO_OBJ ),
-  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("family",                K_FAMILY,           CONCAT_ARGS, NO_OBJ ),
   T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
   T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
   A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
@@ -91,6 +92,190 @@ find_start_of_next_microdesc(const char *s, const char *eos)
 #undef NEXT_LINE
 }
 
+static inline int
+policy_is_reject_star_or_null(struct short_policy_t *policy)
+{
+  return !policy || short_policy_is_reject_star(policy);
+}
+
+/**
+ * Return a human-readable description of a given saved_location_t.
+ * Never returns NULL.
+ **/
+static const char *
+saved_location_to_string(saved_location_t where)
+{
+  const char *location;
+  switch (where) {
+    case SAVED_NOWHERE:
+      location = "download or generated string";
+      break;
+    case SAVED_IN_CACHE:
+      location = "cache";
+      break;
+    case SAVED_IN_JOURNAL:
+      location = "journal";
+      break;
+    default:
+      location = "unknown location";
+      break;
+  }
+  return location;
+}
+
+/**
+ * Given a microdescriptor stored in <b>where</b> which starts at <b>s</b>,
+ * which ends at <b>start_of_next_microdescriptor</b>, and which is located
+ * within a larger document beginning at <b>start</b>: Fill in the body,
+ * bodylen, bodylen, saved_location, off, and digest fields of <b>md</b> as
+ * appropriate.
+ *
+ * The body field will be an alias within <b>s</b> if <b>saved_location</b>
+ * is SAVED_IN_CACHE, and will be copied into body and nul-terminated
+ * otherwise.
+ **/
+static int
+microdesc_extract_body(microdesc_t *md,
+                       const char *start,
+                       const char *s, const char *start_of_next_microdesc,
+                       saved_location_t where)
+{
+  const bool copy_body = (where != SAVED_IN_CACHE);
+
+  const char *cp = tor_memstr(s, start_of_next_microdesc-s, "onion-key");
+
+  const bool no_onion_key = (cp == NULL);
+  if (no_onion_key) {
+    cp = s; /* So that we have *some* junk to put in the body */
+  }
+
+  md->bodylen = start_of_next_microdesc - cp;
+  md->saved_location = where;
+  if (copy_body)
+    md->body = tor_memdup_nulterm(cp, md->bodylen);
+  else
+    md->body = (char*)cp;
+  md->off = cp - start;
+
+  crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+
+  return no_onion_key ? -1 : 0;
+}
+
+/**
+ * Parse a microdescriptor which begins at <b>s</b> and ends at
+ * <b>start_of_next_microdesc.  Store its fields into <b>md</b>.  Use
+ * <b>where</b> for generating log information.  If <b>allow_annotations</b>
+ * is true, then one or more annotations may precede the microdescriptor body
+ * proper.  Use <b>area</b> for memory management, clearing it when done.
+ *
+ * On success, return 0; otherwise return -1.
+ **/
+static int
+microdesc_parse_fields(microdesc_t *md,
+                       memarea_t *area,
+                       const char *s, const char *start_of_next_microdesc,
+                       int allow_annotations,
+                       saved_location_t where)
+{
+  smartlist_t *tokens = smartlist_new();
+  int rv = -1;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+  directory_token_t *tok;
+
+  if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                      microdesc_token_table, flags)) {
+    log_warn(LD_DIR, "Unparseable microdescriptor found in %s",
+             saved_location_to_string(where));
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+    if (parse_iso_time(tok->args[0], &md->last_listed)) {
+      log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+      goto err;
+    }
+  }
+
+  tok = find_by_keyword(tokens, K_ONION_KEY);
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_DIR,
+             "Relay's onion key had invalid exponent.");
+    goto err;
+  }
+  md->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  md->onion_pkey_len = tok->object_size;
+  crypto_pk_free(tok->key);
+
+  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+    curve25519_public_key_t k;
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
+      goto err;
+    }
+    md->onion_curve25519_pkey =
+      tor_memdup(&k, sizeof(curve25519_public_key_t));
+  }
+
+  smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
+  if (id_lines) {
+    SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
+      tor_assert(t->n_args >= 2);
+      if (!strcmp(t->args[0], "ed25519")) {
+        if (md->ed25519_identity_pkey) {
+          log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        ed25519_public_key_t k;
+        if (ed25519_public_from_base64(&k, t->args[1])<0) {
+          log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
+      }
+    } SMARTLIST_FOREACH_END(t);
+    smartlist_free(id_lines);
+  }
+
+  {
+    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+    if (a_lines) {
+      find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
+      smartlist_free(a_lines);
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+    md->family = nodefamily_parse(tok->args[0],
+                                  NULL,
+                                  NF_WARN_MALFORMED);
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_P))) {
+    md->exit_policy = parse_short_policy(tok->args[0]);
+  }
+  if ((tok = find_opt_by_keyword(tokens, K_P6))) {
+    md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+  }
+
+  if (policy_is_reject_star_or_null(md->exit_policy) &&
+      policy_is_reject_star_or_null(md->ipv6_exit_policy)) {
+    md->policy_is_reject_star = 1;
+  }
+
+  rv = 0;
+ err:
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  memarea_clear(area);
+  smartlist_free(tokens);
+
+  return rv;
+}
+
 /** Parse as many microdescriptors as are found from the string starting at
  * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
  * annotations we recognize and ignore ones we don't.
@@ -108,16 +293,11 @@ microdescs_parse_from_string(const char *s, const char *eos,
                              saved_location_t where,
                              smartlist_t *invalid_digests_out)
 {
-  smartlist_t *tokens;
   smartlist_t *result;
   microdesc_t *md = NULL;
   memarea_t *area;
   const char *start = s;
   const char *start_of_next_microdesc;
-  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
-  const int copy_body = (where != SAVED_IN_CACHE);
-
-  directory_token_t *tok;
 
   if (!eos)
     eos = s + strlen(s);
@@ -125,143 +305,47 @@ microdescs_parse_from_string(const char *s, const char *eos,
   s = eat_whitespace_eos(s, eos);
   area = memarea_new();
   result = smartlist_new();
-  tokens = smartlist_new();
 
   while (s < eos) {
-    int okay = 0;
+   bool okay = false;
 
     start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
     if (!start_of_next_microdesc)
       start_of_next_microdesc = eos;
 
     md = tor_malloc_zero(sizeof(microdesc_t));
+    uint8_t md_digest[DIGEST256_LEN];
     {
-      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
-                                  "onion-key");
-      const int no_onion_key = (cp == NULL);
-      if (no_onion_key) {
-        cp = s; /* So that we have *some* junk to put in the body */
-      }
+      const bool body_not_found =
+        microdesc_extract_body(md, start, s,
+                               start_of_next_microdesc,
+                               where) < 0;
 
-      md->bodylen = start_of_next_microdesc - cp;
-      md->saved_location = where;
-      if (copy_body)
-        md->body = tor_memdup_nulterm(cp, md->bodylen);
-      else
-        md->body = (char*)cp;
-      md->off = cp - start;
-      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
-      if (no_onion_key) {
+      memcpy(md_digest, md->digest, DIGEST256_LEN);
+      if (body_not_found) {
         log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
         goto next;
       }
     }
 
-    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
-                        microdesc_token_table, flags)) {
-      log_warn(LD_DIR, "Unparseable microdescriptor");
-      goto next;
+    if (microdesc_parse_fields(md, area, s, start_of_next_microdesc,
+                               allow_annotations, where) == 0) {
+      smartlist_add(result, md);
+      md = NULL; // prevent free
+      okay = true;
     }
 
-    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
-      if (parse_iso_time(tok->args[0], &md->last_listed)) {
-        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
-        goto next;
-      }
-    }
-
-    tok = find_by_keyword(tokens, K_ONION_KEY);
-    if (!crypto_pk_public_exponent_ok(tok->key)) {
-      log_warn(LD_DIR,
-               "Relay's onion key had invalid exponent.");
-      goto next;
-    }
-    router_set_rsa_onion_pkey(tok->key, &md->onion_pkey,
-                              &md->onion_pkey_len);
-    crypto_pk_free(tok->key);
-
-    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
-      curve25519_public_key_t k;
-      tor_assert(tok->n_args >= 1);
-      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
-        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
-        goto next;
-      }
-      md->onion_curve25519_pkey =
-        tor_memdup(&k, sizeof(curve25519_public_key_t));
-    }
-
-    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
-    if (id_lines) {
-      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
-        tor_assert(t->n_args >= 2);
-        if (!strcmp(t->args[0], "ed25519")) {
-          if (md->ed25519_identity_pkey) {
-            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          ed25519_public_key_t k;
-          if (ed25519_public_from_base64(&k, t->args[1])<0) {
-            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
-        }
-      } SMARTLIST_FOREACH_END(t);
-      smartlist_free(id_lines);
-    }
-
-    {
-      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
-      if (a_lines) {
-        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
-        smartlist_free(a_lines);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
-      int i;
-      md->family = smartlist_new();
-      for (i=0;i<tok->n_args;++i) {
-        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
-          log_warn(LD_DIR, "Illegal nickname %s in family line",
-                   escaped(tok->args[i]));
-          goto next;
-        }
-        smartlist_add_strdup(md->family, tok->args[i]);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_P))) {
-      md->exit_policy = parse_short_policy(tok->args[0]);
-    }
-    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
-      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
-    }
-
-    smartlist_add(result, md);
-    okay = 1;
-
-    md = NULL;
   next:
     if (! okay && invalid_digests_out) {
       smartlist_add(invalid_digests_out,
-                    tor_memdup(md->digest, DIGEST256_LEN));
+                    tor_memdup(md_digest, DIGEST256_LEN));
     }
     microdesc_free(md);
     md = NULL;
-
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    memarea_clear(area);
-    smartlist_clear(tokens);
     s = start_of_next_microdesc;
   }
 
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
   memarea_drop_all(area);
-  smartlist_free(tokens);
 
   return result;
 }
diff --git a/src/feature/dirparse/microdesc_parse.h b/src/feature/dirparse/microdesc_parse.h
index 23a90084b1..95af85544a 100644
--- a/src/feature/dirparse/microdesc_parse.h
+++ b/src/feature/dirparse/microdesc_parse.h
@@ -17,4 +17,4 @@ smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
                                           saved_location_t where,
                                           smartlist_t *invalid_digests_out);
 
-#endif
+#endif /* !defined(TOR_MICRODESC_PARSE_H) */
diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c
index 109eebeb66..d5405e6464 100644
--- a/src/feature/dirparse/ns_parse.c
+++ b/src/feature/dirparse/ns_parse.c
@@ -151,10 +151,11 @@ static token_rule_t networkstatus_vote_footer_token_table[] = {
  * -1. */
 int
 router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              size_t len,
                                               const char **start_out,
                                               const char **end_out)
 {
-  return router_get_hash_impl_helper(s, strlen(s),
+  return router_get_hash_impl_helper(s, len,
                                      "network-status-version",
                                      "\ndirectory-signature",
                                      ' ', LOG_INFO,
@@ -166,12 +167,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s,
  * signed portion can be identified.  Return 0 on success, -1 on failure. */
 int
 router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                           const char *s)
+                                           const char *s, size_t len)
 {
   const char *start, *end;
-  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+  if (router_get_networkstatus_v3_signed_boundaries(s, len,
+                                                    &start, &end) < 0) {
     start = s;
-    end = s + strlen(s);
+    end = s + len;
   }
   tor_assert(start);
   tor_assert(end);
@@ -182,9 +184,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
 /** Set <b>digests</b> to all the digests of the consensus document in
  * <b>s</b> */
 int
-router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
+router_get_networkstatus_v3_hashes(const char *s, size_t len,
+                                   common_digests_t *digests)
 {
-  return router_get_hashes_impl(s,strlen(s),digests,
+  return router_get_hashes_impl(s, len, digests,
                                 "network-status-version",
                                 "\ndirectory-signature",
                                 ' ');
@@ -195,13 +198,13 @@ router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
  * return the start of the directory footer, or the next directory signature.
  * If none is found, return the end of the string. */
 static inline const char *
-find_start_of_next_routerstatus(const char *s)
+find_start_of_next_routerstatus(const char *s, const char *s_eos)
 {
   const char *eos, *footer, *sig;
-  if ((eos = strstr(s, "\nr ")))
+  if ((eos = tor_memstr(s, s_eos - s, "\nr ")))
     ++eos;
   else
-    eos = s + strlen(s);
+    eos = s_eos;
 
   footer = tor_memstr(s, eos-s, "\ndirectory-footer");
   sig = tor_memstr(s, eos-s, "\ndirectory-signature");
@@ -289,7 +292,8 @@ routerstatus_parse_guardfraction(const char *guardfraction_str,
  **/
 STATIC routerstatus_t *
 routerstatus_parse_entry_from_string(memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *s_eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
@@ -308,7 +312,7 @@ routerstatus_parse_entry_from_string(memarea_t *area,
     flav = FLAV_NS;
   tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
 
-  eos = find_start_of_next_routerstatus(*s);
+  eos = find_start_of_next_routerstatus(*s, s_eos);
 
   if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing router status");
@@ -430,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area,
         rs->is_hs_dir = 1;
       } else if (!strcmp(tok->args[i], "V2Dir")) {
         rs->is_v2_dir = 1;
+      } else if (!strcmp(tok->args[i], "StaleDesc")) {
+        rs->is_staledesc = 1;
       }
     }
     /* These are implied true by having been included in a consensus made
@@ -1051,7 +1057,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
 /** Parse a v3 networkstatus vote, opinion, or consensus (depending on
  * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
 networkstatus_t *
-networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
+networkstatus_parse_vote_from_string(const char *s,
+                                     size_t s_len,
+                                     const char **eos_out,
                                      networkstatus_type_t ns_type)
 {
   smartlist_t *tokens = smartlist_new();
@@ -1067,20 +1075,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   memarea_t *area = NULL, *rs_area = NULL;
   consensus_flavor_t flav = FLAV_NS;
   char *last_kwd=NULL;
+  const char *eos = s + s_len;
 
   tor_assert(s);
 
   if (eos_out)
     *eos_out = NULL;
 
-  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
-      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
+  if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed,
+                                                 s, s_len)<0) {
     log_warn(LD_DIR, "Unable to compute digest of network-status");
     goto err;
   }
 
   area = memarea_new();
-  end_of_header = find_start_of_next_routerstatus(s);
+  end_of_header = find_start_of_next_routerstatus(s, eos);
   if (tokenize_string(area, s, end_of_header, tokens,
                       (ns_type == NS_TYPE_CONSENSUS) ?
                       networkstatus_consensus_token_table :
@@ -1111,10 +1121,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   if (ns_type != NS_TYPE_CONSENSUS) {
     const char *end_of_cert = NULL;
-    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
+    if (!(cert = tor_memstr(s, end_of_header - s,
+                            "\ndir-key-certificate-version")))
       goto err;
     ++cert;
-    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
+    ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert,
+                                                &end_of_cert);
     if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
       goto err;
   }
@@ -1424,10 +1436,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   s = end_of_header;
   ns->routerstatus_list = smartlist_new();
 
-  while (!strcmpstart(s, "r ")) {
+  while (eos - s >= 2 && fast_memeq(s, "r ", 2)) {
     if (ns->type != NS_TYPE_CONSENSUS) {
       vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
+      if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns,
                                                rs, 0, 0)) {
         smartlist_add(ns->routerstatus_list, rs);
       } else {
@@ -1435,7 +1447,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
       }
     } else {
       routerstatus_t *rs;
-      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
+      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos,
+                                                     rs_tokens,
                                                      NULL, NULL,
                                                      ns->consensus_method,
                                                      flav))) {
@@ -1465,7 +1478,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
                             vrs) {
       if (! vrs->has_ed25519_listing ||
-          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
+          fast_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
         continue;
       if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
         log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
@@ -1480,10 +1493,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   /* Parse footer; check signature. */
   footer_tokens = smartlist_new();
-  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
+  if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version ")))
     ++end_of_footer;
   else
-    end_of_footer = s + strlen(s);
+    end_of_footer = eos;
   if (tokenize_string(area,s, end_of_footer, footer_tokens,
                       networkstatus_vote_footer_token_table, 0)) {
     log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h
index 10a6f9cefc..0cf2cc88d0 100644
--- a/src/feature/dirparse/ns_parse.h
+++ b/src/feature/dirparse/ns_parse.h
@@ -12,18 +12,19 @@
 #ifndef TOR_NS_PARSE_H
 #define TOR_NS_PARSE_H
 
-int router_get_networkstatus_v3_hashes(const char *s,
+int router_get_networkstatus_v3_hashes(const char *s, size_t len,
                                        common_digests_t *digests);
-int router_get_networkstatus_v3_signed_boundaries(const char *s,
+int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len,
                                                   const char **start_out,
                                                   const char **end_out);
 int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                               const char *s);
+                                               const char *s, size_t len);
 int compare_vote_routerstatus_entries(const void **_a, const void **_b);
 
 int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
 enum networkstatus_type_t;
 networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
+                                           size_t len,
                                            const char **eos_out,
                                            enum networkstatus_type_t ns_type);
 
@@ -35,11 +36,12 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
 struct memarea_t;
 STATIC routerstatus_t *routerstatus_parse_entry_from_string(
                                      struct memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
                                      consensus_flavor_t flav);
-#endif
+#endif /* defined(NS_PARSE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NS_PARSE_H) */
diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c
index 1664a77bbe..c22ed186b8 100644
--- a/src/feature/dirparse/parsecommon.c
+++ b/src/feature/dirparse/parsecommon.c
@@ -15,6 +15,7 @@
 #include "lib/string/printf.h"
 #include "lib/memarea/memarea.h"
 #include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/ctime/di_ops.h"
 
 #include <string.h>
 
@@ -169,7 +170,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok,
   char *cp = mem;
   int j = 0;
   char *args[MAX_ARGS];
-  memset(args, 0, sizeof(args));
   while (*cp) {
     if (j == MAX_ARGS)
       return -1;
@@ -251,6 +251,16 @@ token_check_object(memarea_t *area, const char *kwd,
   return tok;
 }
 
+/** Return true iff the <b>memlen</b>-byte chunk of memory at
+ * <b>memlen</b> is the same length as <b>token</b>, and their
+ * contents are equal. */
+static bool
+mem_eq_token(const void *mem, size_t memlen, const char *token)
+{
+  size_t len = strlen(token);
+  return memlen == len && fast_memeq(mem, token, len);
+}
+
 /** Helper function: read the next token from *s, advance *s to the end of the
  * token, and return the parsed token.  Parse *<b>s</b> according to the list
  * of tokens in <b>table</b>.
@@ -266,7 +276,7 @@ get_next_token(memarea_t *area,
    * attack, a bug, or some other nonsense. */
 #define MAX_LINE_LENGTH (128*1024)
 
-  const char *next, *eol, *obstart;
+  const char *next, *eol;
   size_t obname_len;
   int i;
   directory_token_t *tok;
@@ -290,7 +300,7 @@ get_next_token(memarea_t *area,
 
   next = find_whitespace_eos(*s, eol);
 
-  if (!strcmp_len(*s, "opt", next-*s)) {
+  if (mem_eq_token(*s, next-*s, "opt")) {
     /* Skip past an "opt" at the start of the line. */
     *s = eat_whitespace_eos_no_nl(next, eol);
     next = find_whitespace_eos(*s, eol);
@@ -301,7 +311,7 @@ get_next_token(memarea_t *area,
   /* Search the table for the appropriate entry.  (I tried a binary search
    * instead, but it wasn't any faster.) */
   for (i = 0; table[i].t ; ++i) {
-    if (!strcmp_len(*s, table[i].t, next-*s)) {
+    if (mem_eq_token(*s, next-*s, table[i].t)) {
       /* We've found the keyword. */
       kwd = table[i].t;
       tok->tp = table[i].v;
@@ -352,9 +362,8 @@ get_next_token(memarea_t *area,
   if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
     goto check_object;
 
-  obstart = *s; /* Set obstart to start of object spec */
   if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
-      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      !mem_eq_token(eol-5, 5, "-----") ||   /* nuls or invalid endings */
       (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
     RET_ERR("Malformed object: bad begin line");
   }
@@ -373,8 +382,8 @@ get_next_token(memarea_t *area,
     eol = eos;
   /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
   if ((size_t)(eol-next) != 9+obname_len+5 ||
-      strcmp_len(next+9, tok->object_type, obname_len) ||
-      strcmp_len(eol-5, "-----", 5)) {
+      !mem_eq_token(next+9, obname_len, tok->object_type) ||
+      !mem_eq_token(eol-5, 5, "-----")) {
     tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
              tok->object_type);
     ebuf[sizeof(ebuf)-1] = '\0';
@@ -383,28 +392,32 @@ get_next_token(memarea_t *area,
   if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
     RET_ERR("Couldn't parse object: missing footer or object much too big.");
 
+  {
+    int r;
+    size_t maxsize = base64_decode_maxsize(next-*s);
+    tok->object_body = ALLOC(maxsize);
+    r = base64_decode(tok->object_body, maxsize, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+
   if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
     if (o_syn != NEED_KEY && o_syn != NEED_KEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected public key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size);
+    if (! tok->key)
       RET_ERR("Couldn't parse public key.");
   } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
     if (o_syn != NEED_SKEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected private key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_private_key1024_from_string(tok->key,
-                                                   obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode_private(tok->object_body,
+                                             tok->object_size,
+                                             1024);
+    if (! tok->key)
       RET_ERR("Couldn't parse private key.");
-  } else { /* If it's something else, try to base64-decode it */
-    int r;
-    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
-    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
-    if (r<0)
-      RET_ERR("Malformed object: bad base64-encoded data");
-    tok->object_size = r;
   }
   *s = eol;
 
diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c
index e44fbf77f9..f78c46f186 100644
--- a/src/feature/dirparse/routerparse.c
+++ b/src/feature/dirparse/routerparse.c
@@ -591,8 +591,8 @@ router_parse_entry_from_string(const char *s, const char *end,
              "Relay's onion key had invalid exponent.");
     goto err;
   }
-  router_set_rsa_onion_pkey(tok->key, &router->onion_pkey,
-                            &router->onion_pkey_len);
+  router->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  router->onion_pkey_len = tok->object_size;
   crypto_pk_free(tok->key);
 
   if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
diff --git a/src/feature/dirparse/sigcommon.h b/src/feature/dirparse/sigcommon.h
index fdd8e839a9..b6b34e8f62 100644
--- a/src/feature/dirparse/sigcommon.h
+++ b/src/feature/dirparse/sigcommon.h
@@ -43,6 +43,6 @@ MOCK_DECL(STATIC int, signed_digest_equals,
 MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
                            const char *start, size_t len,
                            digest_algorithm_t alg));
-#endif
+#endif /* defined(SIGCOMMON_PRIVATE) */
 
 #endif /* !defined(TOR_SIGCOMMON_H) */
diff --git a/src/feature/dirparse/signing.h b/src/feature/dirparse/signing.h
index 2e3699baf8..8b119b4eb2 100644
--- a/src/feature/dirparse/signing.h
+++ b/src/feature/dirparse/signing.h
@@ -20,4 +20,4 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
                                    const char *digest,
                                    size_t digest_len,
                                    crypto_pk_t *private_key);
-#endif
+#endif /* !defined(TOR_SIGNING_H) */
diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h
index 853fe8cb0f..36c6b5a1ec 100644
--- a/src/feature/dirparse/unparseable.h
+++ b/src/feature/dirparse/unparseable.h
@@ -26,7 +26,7 @@ void dump_desc_init(void);
   log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
             name, (unsigned long)alloc, (unsigned long)used);     \
   STMT_END
-#else /* !(defined(DEBUG_AREA_ALLOC)) */
+#else /* !defined(DEBUG_AREA_ALLOC) */
 #define DUMP_AREA(a,name) STMT_NIL
 #endif /* defined(DEBUG_AREA_ALLOC) */
 
@@ -51,6 +51,6 @@ EXTERN(struct smartlist_t *, descs_dumped)
 MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
     (const char *dirname, const char *f));
 STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-#endif
+#endif /* defined(UNPARSEABLE_PRIVATE) */
 
 #endif /* !defined(TOR_UNPARSEABLE_H) */
diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c
index 09932c97ac..f7847d9a16 100644
--- a/src/feature/hibernate/hibernate.c
+++ b/src/feature/hibernate/hibernate.c
@@ -35,8 +35,9 @@ hibernating, phase 2:
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "feature/hibernate/hibernate.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/relay/router.h"
@@ -56,7 +57,7 @@ hibernating, phase 2:
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-# endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
@@ -66,8 +67,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL;
 /** If are hibernating, when do we plan to wake up? Set to 0 if we
  * aren't hibernating. */
 static time_t hibernate_end_time = 0;
-/** If we are shutting down, when do we plan finally exit? Set to 0 if
- * we aren't shutting down. */
+/** If we are shutting down, when do we plan to finally exit? Set to 0 if we
+ * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed
+ * to happen from mainloop_schedule_shutdown() now.) */
 static time_t shutdown_time = 0;
 
 /** A timed event that we'll use when it's time to wake up from
@@ -560,7 +562,7 @@ time_to_record_bandwidth_usage(time_t now)
   /* Note every 600 sec */
 #define NOTE_INTERVAL (600)
   /* Or every 20 megabytes */
-#define NOTE_BYTES 20*(1024*1024)
+#define NOTE_BYTES (20*1024*1024)
   static uint64_t last_read_bytes_noted = 0;
   static uint64_t last_written_bytes_noted = 0;
   static time_t last_time_noted = 0;
@@ -813,7 +815,7 @@ hibernate_soft_limit_reached(void)
    * We want to stop accepting connections when ALL of the following are true:
    *   - We expect to use up the remaining bytes in under 3 hours
    *   - We have used up 95% of our bytes.
-   *   - We have less than 500MB of bytes left.
+   *   - We have less than 500MBytes left.
    */
   uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT);
   if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) {
@@ -831,8 +833,6 @@ hibernate_soft_limit_reached(void)
   return get_accounting_bytes() >= soft_limit;
 }
 
-#define TOR_USEC_PER_SEC (1000000)
-
 /** Called when we get a SIGINT, or when bandwidth soft limit is
  * reached. Puts us into "loose hibernation": we don't accept new
  * connections, but we continue handling old ones. */
@@ -867,7 +867,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
     log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new "
                "connections, and will shut down in %d seconds. Interrupt "
                "again to exit now.", options->ShutdownWaitLength);
-    shutdown_time = time(NULL) + options->ShutdownWaitLength;
+    /* We add an arbitrary delay here so that even if something goes wrong
+     * with the mainloop shutdown code, we can still shutdown from
+     * consider_hibernation() if we call it... but so that the
+     * mainloop_schedule_shutdown() mechanism will be the first one called.
+     */
+    shutdown_time = time(NULL) + options->ShutdownWaitLength + 5;
+    mainloop_schedule_shutdown(options->ShutdownWaitLength);
 #ifdef HAVE_SYSTEMD
     /* tell systemd that we may need more than the default 90 seconds to shut
      * down so they don't kill us. add some extra time to actually finish
@@ -887,7 +893,7 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
      */
     sd_notifyf(0, "EXTEND_TIMEOUT_USEC=%" PRIu64,
             ((uint64_t)(options->ShutdownWaitLength) + 30) * TOR_USEC_PER_SEC);
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
   } else { /* soft limit reached */
     hibernate_end_time = interval_end_time;
   }
@@ -1096,11 +1102,12 @@ consider_hibernation(time_t now)
   hibernate_state_t prev_state = hibernate_state;
 
   /* If we're in 'exiting' mode, then we just shut down after the interval
-   * elapses. */
+   * elapses.  The mainloop was supposed to catch this via
+   * mainloop_schedule_shutdown(), but apparently it didn't. */
   if (hibernate_state == HIBERNATE_STATE_EXITING) {
     tor_assert(shutdown_time);
     if (shutdown_time <= now) {
-      log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+      log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting.");
       tor_shutdown_event_loop_and_exit(0);
     }
     return; /* if exiting soon, don't worry about bandwidth limits */
@@ -1112,7 +1119,7 @@ consider_hibernation(time_t now)
     if (hibernate_end_time > now && accounting_enabled) {
       /* If we're hibernating, don't wake up until it's time, regardless of
        * whether we're in a new interval. */
-      return ;
+      return;
     } else {
       hibernate_end_time_elapsed(now);
     }
@@ -1240,8 +1247,6 @@ on_hibernate_state_change(hibernate_state_t prev_state)
   if (prev_state != HIBERNATE_STATE_INITIAL) {
     rescan_periodic_events(get_options());
   }
-
-  reschedule_per_second_timer();
 }
 
 /** Free all resources held by the accounting module */
diff --git a/src/feature/hibernate/hibernate.h b/src/feature/hibernate/hibernate.h
index 3309ef0ce3..2e245f6ab1 100644
--- a/src/feature/hibernate/hibernate.h
+++ b/src/feature/hibernate/hibernate.h
@@ -32,6 +32,7 @@ int getinfo_helper_accounting(control_connection_t *conn,
                               const char **errmsg);
 uint64_t get_accounting_max_total(void);
 void accounting_free_all(void);
+bool accounting_tor_is_dormant(void);
 
 #ifdef HIBERNATE_PRIVATE
 /** Possible values of hibernate_state */
diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c
index 05f9940ae6..9817113b23 100644
--- a/src/feature/hs/hs_cache.c
+++ b/src/feature/hs/hs_cache.c
@@ -710,6 +710,11 @@ cache_clean_v3_as_client(time_t now)
     MAP_DEL_CURRENT(key);
     entry_size = cache_get_client_entry_size(entry);
     bytes_removed += entry_size;
+    /* We just removed an old descriptor. We need to close all intro circuits
+     * so we don't have leftovers that can be selected while lacking a
+     * descriptor. We leave the rendezvous circuits opened because they could
+     * be in use. */
+    hs_client_close_intro_circuits_from_desc(entry->desc);
     /* Entry is not in the cache anymore, destroy it. */
     cache_client_desc_free(entry);
     /* Update our OOM. We didn't use the remove() function because we are in
diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c
index 613ffe7260..df59f73c1b 100644
--- a/src/feature/hs/hs_cell.c
+++ b/src/feature/hs/hs_cell.c
@@ -473,10 +473,132 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
   }
 }
 
+/* Build and add to the given DoS cell extension the given parameter type and
+ * value. */
+static void
+build_establish_intro_dos_param(trn_cell_extension_dos_t *dos_ext,
+                                uint8_t param_type, uint64_t param_value)
+{
+  trn_cell_extension_dos_param_t *dos_param =
+    trn_cell_extension_dos_param_new();
+
+  /* Extra safety. We should never send an unknown parameter type. */
+  tor_assert(param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC ||
+             param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+
+  trn_cell_extension_dos_param_set_type(dos_param, param_type);
+  trn_cell_extension_dos_param_set_value(dos_param, param_value);
+  trn_cell_extension_dos_add_params(dos_ext, dos_param);
+
+  /* Not freeing the trunnel object because it is now owned by dos_ext. */
+}
+
+/* Build the DoS defense cell extension and put it in the given extensions
+ * object. Return 0 on success, -1 on failure.  (Right now, failure is only
+ * possible if there is a bug.) */
+static int
+build_establish_intro_dos_extension(const hs_service_config_t *service_config,
+                                    trn_cell_extension_t *extensions)
+{
+  ssize_t ret;
+  size_t dos_ext_encoded_len;
+  uint8_t *field_array;
+  trn_cell_extension_field_t *field = NULL;
+  trn_cell_extension_dos_t *dos_ext = NULL;
+
+  tor_assert(service_config);
+  tor_assert(extensions);
+
+  /* We are creating a cell extension field of the type DoS. */
+  field = trn_cell_extension_field_new();
+  trn_cell_extension_field_set_field_type(field,
+                                          TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+
+  /* Build DoS extension field. We will put in two parameters. */
+  dos_ext = trn_cell_extension_dos_new();
+  trn_cell_extension_dos_set_n_params(dos_ext, 2);
+
+  /* Build DoS parameter INTRO2 rate per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC,
+                                  service_config->intro_dos_rate_per_sec);
+  /* Build DoS parameter INTRO2 burst per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC,
+                                  service_config->intro_dos_burst_per_sec);
+
+  /* Set the field with the encoded DoS extension. */
+  ret = trn_cell_extension_dos_encoded_len(dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  dos_ext_encoded_len = ret;
+  /* Set length field and the field array size length. */
+  trn_cell_extension_field_set_field_len(field, dos_ext_encoded_len);
+  trn_cell_extension_field_setlen_field(field, dos_ext_encoded_len);
+  /* Encode the DoS extension into the cell extension field. */
+  field_array = trn_cell_extension_field_getarray_field(field);
+  ret = trn_cell_extension_dos_encode(field_array,
+                 trn_cell_extension_field_getlen_field(field), dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  tor_assert(ret == (ssize_t) dos_ext_encoded_len);
+
+  /* Finally, encode field into the cell extension. */
+  trn_cell_extension_add_fields(extensions, field);
+
+  /* We've just add an extension field to the cell extensions so increment the
+   * total number. */
+  trn_cell_extension_set_num(extensions,
+                             trn_cell_extension_get_num(extensions) + 1);
+
+  /* Cleanup. DoS extension has been encoded at this point. */
+  trn_cell_extension_dos_free(dos_ext);
+
+  return 0;
+
+ err:
+  trn_cell_extension_field_free(field);
+  trn_cell_extension_dos_free(dos_ext);
+  return -1;
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
+/* Allocate and build all the ESTABLISH_INTRO cell extension. The given
+ * extensions pointer is always set to a valid cell extension object. */
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip)
+{
+  int ret;
+  trn_cell_extension_t *extensions;
+
+  tor_assert(service_config);
+  tor_assert(ip);
+
+  extensions = trn_cell_extension_new();
+  trn_cell_extension_set_num(extensions, 0);
+
+  /* If the defense has been enabled service side (by the operator with a
+   * torrc option) and the intro point does support it. */
+  if (service_config->has_dos_defense_enabled &&
+      ip->support_intro2_dos_defense) {
+    /* This function takes care to increment the number of extensions. */
+    ret = build_establish_intro_dos_extension(service_config, extensions);
+    if (ret < 0) {
+      /* Return no extensions on error. */
+      goto end;
+    }
+  }
+
+ end:
+  return extensions;
+}
+
 /* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
  * object. The encoded cell is put in cell_out that MUST at least be of the
  * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else
@@ -484,15 +606,17 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
  * legacy cell creation. */
 ssize_t
 hs_cell_build_establish_intro(const char *circ_nonce,
+                              const hs_service_config_t *service_config,
                               const hs_service_intro_point_t *ip,
                               uint8_t *cell_out)
 {
   ssize_t cell_len = -1;
   uint16_t sig_len = ED25519_SIG_LEN;
-  trn_cell_extension_t *ext;
   trn_cell_establish_intro_t *cell = NULL;
+  trn_cell_extension_t *extensions;
 
   tor_assert(circ_nonce);
+  tor_assert(service_config);
   tor_assert(ip);
 
   /* Quickly handle the legacy IP. */
@@ -505,11 +629,12 @@ hs_cell_build_establish_intro(const char *circ_nonce,
     goto done;
   }
 
+  /* Build the extensions, if any. */
+  extensions = build_establish_intro_extensions(service_config, ip);
+
   /* Set extension data. None used here. */
-  ext = trn_cell_extension_new();
-  trn_cell_extension_set_num(ext, 0);
   cell = trn_cell_establish_intro_new();
-  trn_cell_establish_intro_set_extensions(cell, ext);
+  trn_cell_establish_intro_set_extensions(cell, extensions);
   /* Set signature size. Array is then allocated in the cell. We need to do
    * this early so we can use trunnel API to get the signature length. */
   trn_cell_establish_intro_set_sig_len(cell, sig_len);
@@ -758,7 +883,14 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
        idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) {
     link_specifier_t *lspec =
       trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx);
-    smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec));
+    if (BUG(!lspec)) {
+      goto done;
+    }
+    link_specifier_t *lspec_dup = link_specifier_dup(lspec);
+    if (BUG(!lspec_dup)) {
+      goto done;
+    }
+    smartlist_add(data->link_specifiers, lspec_dup);
   }
 
   /* Success. */
@@ -949,4 +1081,3 @@ hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
   /* The data object has no ownership of any members. */
   memwipe(data, 0, sizeof(hs_cell_introduce1_data_t));
 }
-
diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h
index 9569de535e..864b6fda5f 100644
--- a/src/feature/hs/hs_cell.h
+++ b/src/feature/hs/hs_cell.h
@@ -79,6 +79,7 @@ typedef struct hs_cell_introduce2_data_t {
 
 /* Build cell API. */
 ssize_t hs_cell_build_establish_intro(const char *circ_nonce,
+                                      const hs_service_config_t *config,
                                       const hs_service_intro_point_t *ip,
                                       uint8_t *cell_out);
 ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
@@ -105,5 +106,15 @@ int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
 /* Util API. */
 void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data);
 
+#ifdef TOR_UNIT_TESTS
+
+#include "trunnel/hs/cell_common.h"
+
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
 #endif /* !defined(TOR_HS_CELL_H) */
 
diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c
index 8acfcbd65b..5e213b5aba 100644
--- a/src/feature/hs/hs_circuit.c
+++ b/src/feature/hs/hs_circuit.c
@@ -15,6 +15,7 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuit.h"
@@ -89,7 +90,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
   cpath = tor_malloc_zero(sizeof(crypt_path_t));
   cpath->magic = CRYPT_PATH_MAGIC;
 
-  if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys),
+  if (cpath_init_circuit_crypto(cpath, (char*)keys, sizeof(keys),
                                 is_service_side, 1) < 0) {
     tor_free(cpath);
     goto err;
@@ -126,7 +127,7 @@ create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
     goto err;
   }
   /* ... and set up cpath. */
-  if (circuit_init_cpath_crypto(hop,
+  if (cpath_init_circuit_crypto(hop,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 0, 0) < 0)
     goto err;
@@ -177,7 +178,7 @@ finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
   circ->hs_circ_has_timed_out = 0;
 
   /* Append the hop to the cpath of this circuit */
-  onion_append_to_cpath(&circ->cpath, hop);
+  cpath_extend_linked_list(&circ->cpath, hop);
 
   /* In legacy code, 'pending_final_cpath' points to the final hop we just
    * appended to the cpath. We set the original pointer to NULL so that we
@@ -258,8 +259,7 @@ create_rp_circuit_identifier(const hs_service_t *service,
   tor_assert(server_pk);
   tor_assert(keys);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_RENDEZVOUS);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */
   memcpy(ident->rendezvous_cookie, rendezvous_cookie,
          sizeof(ident->rendezvous_cookie));
@@ -293,8 +293,7 @@ create_intro_circuit_identifier(const hs_service_t *service,
   tor_assert(service);
   tor_assert(ip);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_INTRO);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey);
 
   return ident;
@@ -318,7 +317,7 @@ send_establish_intro(const hs_service_t *service,
 
   /* Encode establish intro cell. */
   cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce,
-                                           ip, payload);
+                                           &service->config, ip, payload);
   if (cell_len < 0) {
     log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s "
                       "on circuit %u. Closing circuit.",
@@ -388,10 +387,7 @@ launch_rendezvous_point_circuit(const hs_service_t *service,
                                         &data->onion_pk,
                                         service->config.is_single_onion);
   if (info == NULL) {
-    /* We are done here, we can't extend to the rendezvous point.
-     * If you're running an IPv6-only v3 single onion service on 0.3.2 or with
-     * 0.3.2 clients, and somehow disable the option check, it will fail here.
-     */
+    /* We are done here, we can't extend to the rendezvous point. */
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Not enough info to open a circuit to a rendezvous point for "
            "%s service %s.",
@@ -569,81 +565,6 @@ retry_service_rendezvous_point(const origin_circuit_t *circ)
   return;
 }
 
-/* Add all possible link specifiers in node to lspecs:
- *  - legacy ID is mandatory thus MUST be present in node;
- *  - include ed25519 link specifier if present in the node, and the node
- *    supports ed25519 link authentication, even if its link versions are not
- *    compatible with us;
- *  - include IPv4 link specifier, if the primary address is not IPv4, log a
- *    BUG() warning, and return an empty smartlist;
- *  - include IPv6 link specifier if present in the node. */
-static void
-get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
-{
-  link_specifier_t *ls;
-  tor_addr_port_t ap;
-
-  tor_assert(node);
-  tor_assert(lspecs);
-
-  /* Get the relay's IPv4 address. */
-  node_get_prim_orport(node, &ap);
-
-  /* We expect the node's primary address to be a valid IPv4 address.
-   * This conforms to the protocol, which requires either an IPv4 or IPv6
-   * address (or both). */
-  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
-      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
-    return;
-  }
-
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_IPV4);
-  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
-  link_specifier_set_un_ipv4_port(ls, ap.port);
-  /* Four bytes IPv4 and two bytes port. */
-  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
-                            sizeof(ap.port));
-  smartlist_add(lspecs, ls);
-
-  /* Legacy ID is mandatory and will always be present in node. */
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
-  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
-         link_specifier_getlen_un_legacy_id(ls));
-  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
-  smartlist_add(lspecs, ls);
-
-  /* ed25519 ID is only included if the node has it, and the node declares a
-     protocol version that supports ed25519 link authentication, even if that
-     link version is not compatible with us. (We are sending the ed25519 key
-     to another tor, which may support different link versions.) */
-  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
-      node_supports_ed25519_link_authentication(node, 0)) {
-    ls = link_specifier_new();
-    link_specifier_set_ls_type(ls, LS_ED25519_ID);
-    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
-           link_specifier_getlen_un_ed25519_id(ls));
-    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
-    smartlist_add(lspecs, ls);
-  }
-
-  /* Check for IPv6. If so, include it as well. */
-  if (node_has_ipv6_orport(node)) {
-    ls = link_specifier_new();
-    node_get_pref_ipv6_orport(node, &ap);
-    link_specifier_set_ls_type(ls, LS_IPV6);
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
-    smartlist_add(lspecs, ls);
-  }
-}
-
 /* Using the given descriptor intro point ip, the node of the
  * rendezvous point rp_node and the service's subcredential, populate the
  * already allocated intro1_data object with the needed key material and link
@@ -666,10 +587,9 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip,
   tor_assert(subcredential);
   tor_assert(intro1_data);
 
-  /* Build the link specifiers from the extend information of the rendezvous
-   * circuit that we've picked previously. */
-  rp_lspecs = smartlist_new();
-  get_lspecs_from_node(rp_node, rp_lspecs);
+  /* Build the link specifiers from the node at the end of the rendezvous
+   * circuit that we opened for this introduction. */
+  rp_lspecs = node_get_link_specifier_smartlist(rp_node, 0);
   if (smartlist_len(rp_lspecs) == 0) {
     /* We can't rendezvous without link specifiers. */
     smartlist_free(rp_lspecs);
@@ -1060,9 +980,7 @@ hs_circ_handle_introduce2(const hs_service_t *service,
   ret = 0;
 
  done:
-  SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec,
-                    link_specifier_free(lspec));
-  smartlist_free(data.link_specifiers);
+  link_specifier_smartlist_free(data.link_specifiers);
   memwipe(&data, 0, sizeof(data));
   return ret;
 }
diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c
index 5480d5eb84..e34f564fb4 100644
--- a/src/feature/hs/hs_circuitmap.c
+++ b/src/feature/hs/hs_circuitmap.c
@@ -272,6 +272,33 @@ hs_circuitmap_get_or_circuit(hs_token_type_t type,
 
 /**** Public relay-side getters: */
 
+/* Public function: Return v2 and v3 introduction circuit to this relay.
+ * Always return a newly allocated list for which it is the caller's
+ * responsability to free it. */
+smartlist_t *
+hs_circuitmap_get_all_intro_circ_relay_side(void)
+{
+  circuit_t **iter;
+  smartlist_t *circuit_list = smartlist_new();
+
+  HT_FOREACH(iter, hs_circuitmap_ht, the_hs_circuitmap) {
+    circuit_t *circ = *iter;
+
+    /* An origin circuit or purpose is wrong or the hs token is not set to be
+     * a v2 or v3 intro relay side type, we ignore the circuit. Else, we have
+     * a match so add it to our list. */
+    if (CIRCUIT_IS_ORIGIN(circ) ||
+        circ->purpose != CIRCUIT_PURPOSE_INTRO_POINT ||
+        (circ->hs_token->type != HS_TOKEN_INTRO_V3_RELAY_SIDE &&
+         circ->hs_token->type != HS_TOKEN_INTRO_V2_RELAY_SIDE)) {
+      continue;
+    }
+    smartlist_add(circuit_list, circ);
+  }
+
+  return circuit_list;
+}
+
 /* Public function: Return a v3 introduction circuit to this relay with
  * <b>auth_key</b>. Return NULL if no such circuit is found in the
  * circuitmap. */
diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h
index c1bbb1ff1c..eac8230bbf 100644
--- a/src/feature/hs/hs_circuitmap.h
+++ b/src/feature/hs/hs_circuitmap.h
@@ -34,6 +34,8 @@ void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
 void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
                                          const ed25519_public_key_t *auth_key);
 
+smartlist_t *hs_circuitmap_get_all_intro_circ_relay_side(void);
+
 /** Public service-side API: */
 
 struct origin_circuit_t *
diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c
index c65f857419..8b63375939 100644
--- a/src/feature/hs/hs_client.c
+++ b/src/feature/hs/hs_client.c
@@ -167,9 +167,7 @@ purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
    * some point and we don't care about those anymore. */
   hs_build_blinded_pubkey(identity_pk, NULL, 0,
                           hs_get_time_period_num(0), &blinded_pk);
-  if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   /* Purge last hidden service request from cache for this blinded key. */
   hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk);
 }
@@ -356,7 +354,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   ed25519_public_key_t blinded_pubkey;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   hs_ident_dir_conn_t hs_conn_dir_ident;
-  int retval;
 
   tor_assert(hsdir);
   tor_assert(onion_identity_pk);
@@ -365,10 +362,7 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return HS_CLIENT_FETCH_ERROR;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Copy onion pk to a dir_ident so that we attach it to the dir conn */
   hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey,
@@ -407,7 +401,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
 STATIC routerstatus_t *
 pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 {
-  int retval;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   uint64_t current_time_period = hs_get_time_period_num(0);
   smartlist_t *responsible_hsdirs = NULL;
@@ -420,10 +413,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return NULL;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Get responsible hsdirs of service for this time period */
   responsible_hsdirs = smartlist_new();
@@ -436,7 +426,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 
   /* Pick an HSDir from the responsible ones. The ownership of
    * responsible_hsdirs is given to this function so no need to free it. */
-  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey);
+  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey, NULL);
 
   return hsdir_rs;
 }
@@ -461,6 +451,24 @@ fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk))
   return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs);
 }
 
+/* With a given <b>onion_identity_pk</b>, fetch its descriptor. If
+ * <b>hsdirs</b> is specified, use the directory servers specified in the list.
+ * Else, use a random server. */
+void
+hs_client_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  if (hsdirs != NULL) {
+    SMARTLIST_FOREACH_BEGIN(hsdirs, const routerstatus_t *, hsdir) {
+      directory_launch_v3_desc_fetch(onion_identity_pk, hsdir);
+    } SMARTLIST_FOREACH_END(hsdir);
+  } else {
+    fetch_v3_desc(onion_identity_pk);
+  }
+}
+
 /* Make sure that the given v3 origin circuit circ is a valid correct
  * introduction circuit. This will BUG() on any problems and hard assert if
  * the anonymity of the circuit is not ok. Return 0 on success else -1 where
@@ -530,13 +538,15 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id,
   SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                           hs_desc_intro_point_t *, ip) {
     SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                            const hs_desc_link_specifier_t *, lspec) {
+                            const link_specifier_t *, lspec) {
       /* Not all tor node have an ed25519 identity key so we still rely on the
        * legacy identity digest. */
-      if (lspec->type != LS_LEGACY_ID) {
+      if (link_specifier_get_ls_type(lspec) != LS_LEGACY_ID) {
         continue;
       }
-      if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) {
+      if (fast_memneq(legacy_id,
+                      link_specifier_getconstarray_un_legacy_id(lspec),
+                      DIGEST_LEN)) {
         break;
       }
       /* Found it. */
@@ -698,7 +708,7 @@ setup_intro_circ_auth_key(origin_circuit_t *circ)
   }
 
   /* Reaching this point means we didn't find any intro point for this circuit
-   * which is not suppose to happen. */
+   * which is not supposed to happen. */
   tor_assert_nonfatal_unreached();
 
  end:
@@ -764,24 +774,13 @@ STATIC extend_info_t *
 desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip)
 {
   extend_info_t *ei;
-  smartlist_t *lspecs = smartlist_new();
 
   tor_assert(ip);
 
-  /* We first encode the descriptor link specifiers into the binary
-   * representation which is a trunnel object. */
-  SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                          const hs_desc_link_specifier_t *, desc_lspec) {
-    link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec);
-    smartlist_add(lspecs, lspec);
-  } SMARTLIST_FOREACH_END(desc_lspec);
-
   /* Explicitly put the direct connection option to 0 because this is client
    * side and there is no such thing as a non anonymous client. */
-  ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0);
+  ei = hs_get_extend_info_from_lspecs(ip->link_specifiers, &ip->onion_key, 0);
 
-  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls));
-  smartlist_free(lspecs);
   return ei;
 }
 
@@ -1552,7 +1551,10 @@ parse_auth_file_content(const char *client_key_str)
   auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
   if (base32_decode((char *) auth->enc_seckey.secret_key,
                     sizeof(auth->enc_seckey.secret_key),
-                    seckey_b32, strlen(seckey_b32)) < 0) {
+                    seckey_b32, strlen(seckey_b32)) !=
+      sizeof(auth->enc_seckey.secret_key)) {
+    log_warn(LD_REND, "Client authorization encoded base32 private key "
+                      "can't be decoded: %s", seckey_b32);
     goto err;
   }
   strncpy(auth->onion_address, onion_address, HS_SERVICE_ADDR_LEN_BASE32);
diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h
index dadfa024b8..96a96755fd 100644
--- a/src/feature/hs/hs_client.h
+++ b/src/feature/hs/hs_client.h
@@ -44,6 +44,10 @@ typedef struct hs_client_service_authorization_t {
 void hs_client_note_connection_attempt_succeeded(
                                        const edge_connection_t *conn);
 
+void hs_client_launch_v3_desc_fetch(
+                               const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs);
+
 int hs_client_decode_descriptor(
                      const char *desc_str,
                      const ed25519_public_key_t *service_identity_pk,
diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c
index ebe49f09a5..036d23a6b0 100644
--- a/src/feature/hs/hs_common.c
+++ b/src/feature/hs/hs_common.c
@@ -21,6 +21,7 @@
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_service.h"
 #include "feature/hs_common/shared_random_client.h"
@@ -30,6 +31,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendservice.h"
+#include "feature/relay/routermode.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
@@ -84,7 +86,7 @@ set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
   return 0;
 }
 
-#else /* !(defined(HAVE_SYS_UN_H)) */
+#else /* !defined(HAVE_SYS_UN_H) */
 
 static int
 set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
@@ -926,7 +928,8 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out,
   }
 
   /* Decode address so we can extract needed fields. */
-  if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) {
+  if (base32_decode(decoded, sizeof(decoded), address, strlen(address))
+      != sizeof(decoded)) {
     log_warn(LD_REND, "Service address %s can't be decoded.",
              escaped_safe_str(address));
     goto invalid;
@@ -940,7 +943,7 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out,
   return -1;
 }
 
-/* Validate a given onion address. The length, the base32 decoding and
+/* Validate a given onion address. The length, the base32 decoding, and
  * checksum are validated. Return 1 if valid else 0. */
 int
 hs_address_is_valid(const char *address)
@@ -955,7 +958,7 @@ hs_address_is_valid(const char *address)
     goto invalid;
   }
 
-  /* Get the checksum it's suppose to be and compare it with what we have
+  /* Get the checksum it's supposed to be and compare it with what we have
    * encoded in the address. */
   build_hs_checksum(&service_pubkey, version, target_checksum);
   if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) {
@@ -983,7 +986,7 @@ hs_address_is_valid(const char *address)
  * The returned address is base32 encoded and put in addr_out. The caller MUST
  * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long.
  *
- * Format is as follow:
+ * Format is as follows:
  *     base32(PUBKEY || CHECKSUM || VERSION)
  *     CHECKSUM = H(".onion checksum" || PUBKEY || VERSION)
  * */
@@ -1009,24 +1012,6 @@ hs_build_address(const ed25519_public_key_t *key, uint8_t version,
   tor_assert(hs_address_is_valid(addr_out));
 }
 
-/* Return a newly allocated copy of lspec. */
-link_specifier_t *
-hs_link_specifier_dup(const link_specifier_t *lspec)
-{
-  link_specifier_t *result = link_specifier_new();
-  memcpy(result, lspec, sizeof(*result));
-  /* The unrecognized field is a dynamic array so make sure to copy its
-   * content and not the pointer. */
-  link_specifier_setlen_un_unrecognized(
-                  result, link_specifier_getlen_un_unrecognized(lspec));
-  if (link_specifier_getlen_un_unrecognized(result)) {
-    memcpy(link_specifier_getarray_un_unrecognized(result),
-           link_specifier_getconstarray_un_unrecognized(lspec),
-           link_specifier_getlen_un_unrecognized(result));
-  }
-  return result;
-}
-
 /* From a given ed25519 public key pk and an optional secret, compute a
  * blinded public key and put it in blinded_pk_out. This is only useful to
  * the client side because the client only has access to the identity public
@@ -1042,7 +1027,7 @@ hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
 
   tor_assert(pk);
   tor_assert(blinded_pk_out);
-  tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
 
   build_blinded_key_param(pk, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1067,8 +1052,8 @@ hs_build_blinded_keypair(const ed25519_keypair_t *kp,
   tor_assert(kp);
   tor_assert(blinded_kp_out);
   /* Extra safety. A zeroed key is bad. */
-  tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
-  tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
 
   build_blinded_key_param(&kp->pubkey, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1300,15 +1285,15 @@ node_has_hsdir_index(const node_t *node)
 
   /* At this point, since the node has a desc, this node must also have an
    * hsdir index. If not, something went wrong, so BUG out. */
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.fetch,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_first,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_second,
                           DIGEST256_LEN))) {
     return 0;
   }
@@ -1606,20 +1591,25 @@ hs_purge_last_hid_serv_requests(void)
 /** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the
  *  one that we should use to fetch a descriptor right now. Take into account
  *  previous failed attempts at fetching this descriptor from HSDirs using the
- *  string identifier <b>req_key_str</b>.
+ *  string identifier <b>req_key_str</b>. We return whether we are rate limited
+ *  into *<b>is_rate_limited_out</b> if it is not NULL.
  *
  *  Steals ownership of <b>responsible_dirs</b>.
  *
  *  Return the routerstatus of the chosen HSDir if successful, otherwise return
  *  NULL if no HSDirs are worth trying right now. */
 routerstatus_t *
-hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
+hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str,
+              bool *is_rate_limited_out)
 {
   smartlist_t *usable_responsible_dirs = smartlist_new();
   const or_options_t *options = get_options();
   routerstatus_t *hs_dir;
   time_t now = time(NULL);
   int excluded_some;
+  bool rate_limited = false;
+  int rate_limited_count = 0;
+  int responsible_dirs_count = smartlist_len(responsible_dirs);
 
   tor_assert(req_key_str);
 
@@ -1639,6 +1629,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     if (last + hs_hsdir_requery_period(options) >= now ||
         !node || !node_has_preferred_descriptor(node, 0)) {
       SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
+      rate_limited_count++;
       continue;
     }
     if (!routerset_contains_node(options->ExcludeNodes, node)) {
@@ -1646,6 +1637,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     }
   } SMARTLIST_FOREACH_END(dir);
 
+  if (rate_limited_count > 0 || responsible_dirs_count > 0) {
+    rate_limited = rate_limited_count == responsible_dirs_count;
+  }
+
   excluded_some =
     smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
 
@@ -1657,9 +1652,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
   smartlist_free(responsible_dirs);
   smartlist_free(usable_responsible_dirs);
   if (!hs_dir) {
+    const char *warn_str = (rate_limited) ? "we are rate limited." :
+                              "we requested them all recently without success";
     log_info(LD_REND, "Could not pick one of the responsible hidden "
-                      "service directories, because we requested them all "
-                      "recently without success.");
+                      "service directories, because %s.", warn_str);
     if (options->StrictNodes && excluded_some) {
       log_warn(LD_REND, "Could not pick a hidden service directory for the "
                "requested hidden service: they are all either down or "
@@ -1671,17 +1667,23 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1);
   }
 
+  if (is_rate_limited_out != NULL) {
+    *is_rate_limited_out = rate_limited;
+  }
+
   return hs_dir;
 }
 
-/* From a list of link specifier, an onion key and if we are requesting a
- * direct connection (ex: single onion service), return a newly allocated
- * extend_info_t object. This function always returns an extend info with
- * an IPv4 address, or NULL.
+/* Given a list of link specifiers lspecs, a curve 25519 onion_key, and
+ * a direct connection boolean direct_conn (true for single onion services),
+ * return a newly allocated extend_info_t object.
+ *
+ * This function always returns an extend info with a valid IP address and
+ * ORPort, or NULL. If direct_conn is false, the IP address is always IPv4.
  *
  * It performs the following checks:
- *  if either IPv4 or legacy ID is missing, return NULL.
- *  if direct_conn, and we can't reach the IPv4 address, return NULL.
+ *  if there is no usable IP address, or legacy ID is missing, return NULL.
+ *  if direct_conn, and we can't reach any IP address, return NULL.
  */
 extend_info_t *
 hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
@@ -1690,21 +1692,40 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
 {
   int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0;
   char legacy_id[DIGEST_LEN] = {0};
-  uint16_t port_v4 = 0;
-  tor_addr_t addr_v4;
   ed25519_public_key_t ed25519_pk;
   extend_info_t *info = NULL;
+  tor_addr_port_t ap;
 
-  tor_assert(lspecs);
+  tor_addr_make_null(&ap.addr, AF_UNSPEC);
+  ap.port = 0;
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Specified link specifiers is null");
+    goto done;
+  }
+
+  if (onion_key == NULL) {
+    log_warn(LD_BUG, "Specified onion key is null");
+    goto done;
+  }
+
+  if (smartlist_len(lspecs) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND, "Empty link specifier list.");
+    /* Return NULL. */
+    goto done;
+  }
 
   SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
     switch (link_specifier_get_ls_type(ls)) {
     case LS_IPV4:
-      /* Skip if we already seen a v4. */
-      if (have_v4) continue;
-      tor_addr_from_ipv4h(&addr_v4,
+      /* Skip if we already seen a v4. If direct_conn is true, we skip this
+       * block because fascist_firewall_choose_address_ls() will set ap. If
+       * direct_conn is false, set ap to the first IPv4 address and port in
+       * the link specifiers.*/
+      if (have_v4 || direct_conn) continue;
+      tor_addr_from_ipv4h(&ap.addr,
                           link_specifier_get_un_ipv4_addr(ls));
-      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      ap.port = link_specifier_get_un_ipv4_port(ls);
       have_v4 = 1;
       break;
     case LS_LEGACY_ID:
@@ -1728,45 +1749,38 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
     }
   } SMARTLIST_FOREACH_END(ls);
 
-  /* Legacy ID is mandatory, and we require IPv4. */
-  if (!have_v4 || !have_legacy_id) {
+  /* Choose a preferred address first, but fall back to an allowed address. */
+  if (direct_conn)
+    fascist_firewall_choose_address_ls(lspecs, 0, &ap);
+
+  /* Legacy ID is mandatory, and we require an IP address. */
+  if (!tor_addr_port_is_valid_ap(&ap, 0)) {
+    /* If we're missing the IP address, log a warning and return NULL. */
+    log_info(LD_NET, "Unreachable or invalid IP address in link state");
     goto done;
   }
-
-  /* We know we have IPv4, because we just checked. */
-  if (!direct_conn) {
-    /* All clients can extend to any IPv4 via a 3-hop path. */
-    goto validate;
-  } else if (direct_conn &&
-             fascist_firewall_allows_address_addr(&addr_v4, port_v4,
-                                                  FIREWALL_OR_CONNECTION,
-                                                  0, 0)) {
-    /* Direct connection and we can reach it in IPv4 so go for it. */
-    goto validate;
-
-    /* We will add support for falling back to a 3-hop path in a later
-     * release. */
-  } else {
-    /* If we can't reach IPv4, return NULL. */
+  if (!have_legacy_id) {
+    /* If we're missing the legacy ID, log a warning and return NULL. */
+    log_warn(LD_PROTOCOL, "Missing Legacy ID in link state");
     goto done;
   }
 
-  /* We will add support for IPv6 in a later release. */
+  /* We will add support for falling back to a 3-hop path in a later
+   * release. */
 
- validate:
   /* We'll validate now that the address we've picked isn't a private one. If
-   * it is, are we allowing to extend to private address? */
-  if (!extend_info_addr_is_allowed(&addr_v4)) {
+   * it is, are we allowed to extend to private addresses? */
+  if (!extend_info_addr_is_allowed(&ap.addr)) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Requested address is private and we are not allowed to extend to "
-           "it: %s:%u", fmt_addr(&addr_v4), port_v4);
+           "it: %s:%u", fmt_addr(&ap.addr), ap.port);
     goto done;
   }
 
   /* We do have everything for which we think we can connect successfully. */
   info = extend_info_new(NULL, legacy_id,
                          (have_ed25519_id) ? &ed25519_pk : NULL, NULL,
-                         onion_key, &addr_v4, port_v4);
+                         onion_key, &ap.addr, ap.port);
  done:
   return info;
 }
@@ -1827,3 +1841,42 @@ hs_inc_rdv_stream_counter(origin_circuit_t *circ)
     tor_assert_nonfatal_unreached();
   }
 }
+
+/* Return a newly allocated link specifier object that is a copy of dst. */
+link_specifier_t *
+link_specifier_dup(const link_specifier_t *src)
+{
+  link_specifier_t *dup = NULL;
+  uint8_t *buf = NULL;
+
+  if (BUG(!src)) {
+    goto err;
+  }
+
+  ssize_t encoded_len_alloc = link_specifier_encoded_len(src);
+  if (BUG(encoded_len_alloc < 0)) {
+    goto err;
+  }
+
+  buf = tor_malloc_zero(encoded_len_alloc);
+  ssize_t encoded_len_data = link_specifier_encode(buf,
+                                                   encoded_len_alloc,
+                                                   src);
+  if (BUG(encoded_len_data < 0)) {
+    goto err;
+  }
+
+  ssize_t parsed_len = link_specifier_parse(&dup, buf, encoded_len_alloc);
+  if (BUG(parsed_len < 0)) {
+    goto err;
+  }
+
+  goto done;
+
+ err:
+  dup = NULL;
+
+ done:
+  tor_free(buf);
+  return dup;
+}
diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h
index a44505930a..3009780d90 100644
--- a/src/feature/hs/hs_common.h
+++ b/src/feature/hs/hs_common.h
@@ -217,8 +217,6 @@ uint64_t hs_get_time_period_num(time_t now);
 uint64_t hs_get_next_time_period_num(time_t now);
 time_t hs_get_start_time_of_next_time_period(time_t now);
 
-link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec);
-
 MOCK_DECL(int, hs_in_period_between_tp_and_srv,
           (const networkstatus_t *consensus, time_t now));
 
@@ -243,7 +241,8 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk,
                               int use_second_hsdir_index,
                               int for_fetching, smartlist_t *responsible_dirs);
 routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs,
-                              const char *req_key_str);
+                              const char *req_key_str,
+                              bool *is_rate_limited_out);
 
 time_t hs_hsdir_requery_period(const or_options_t *options);
 time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
@@ -262,6 +261,8 @@ extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
                           const struct curve25519_public_key_t *onion_key,
                           int direct_conn);
 
+link_specifier_t *link_specifier_dup(const link_specifier_t *src);
+
 #ifdef HS_COMMON_PRIVATE
 
 STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out);
diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c
index ee4499ef5b..3b6caaec6a 100644
--- a/src/feature/hs/hs_config.c
+++ b/src/feature/hs/hs_config.c
@@ -218,6 +218,9 @@ config_has_invalid_options(const config_line_t *line_,
 
   const char *opts_exclude_v2[] = {
     "HiddenServiceExportCircuitID",
+    "HiddenServiceEnableIntroDoSDefense",
+    "HiddenServiceEnableIntroDoSRatePerSec",
+    "HiddenServiceEnableIntroDoSBurstPerSec",
     NULL /* End marker. */
   };
 
@@ -250,6 +253,16 @@ config_has_invalid_options(const config_line_t *line_,
                             "version %" PRIu32 " of service in %s",
                  opt, service->config.version,
                  service->config.directory_path);
+
+        if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
+          /* Special case this v2 option so that we can offer alternatives.
+           * If more such special cases appear, it would be good to
+           * generalize the exception mechanism here. */
+          log_warn(LD_CONFIG, "For v3 onion service client authorization, "
+                   "please read the 'CLIENT AUTHORIZATION' section in the "
+                   "manual.");
+        }
+
         ret = 1;
         /* Continue the loop so we can find all possible options. */
         continue;
@@ -276,6 +289,15 @@ config_validate_service(const hs_service_config_t *config)
     goto invalid;
   }
 
+  /* DoS validation values. */
+  if (config->has_dos_defense_enabled &&
+      (config->intro_dos_burst_per_sec < config->intro_dos_rate_per_sec)) {
+    log_warn(LD_CONFIG, "Hidden service DoS defenses burst (%" PRIu32 ") can "
+                        "not be smaller than the rate value (%" PRIu32 ").",
+             config->intro_dos_burst_per_sec, config->intro_dos_rate_per_sec);
+    goto invalid;
+  }
+
   /* Valid. */
   return 0;
  invalid:
@@ -296,6 +318,8 @@ config_service_v3(const config_line_t *line_,
 {
   int have_num_ip = 0;
   bool export_circuit_id = false; /* just to detect duplicate options */
+  bool dos_enabled = false, dos_rate_per_sec = false;
+  bool dos_burst_per_sec = false;
   const char *dup_opt_seen = NULL;
   const config_line_t *line;
 
@@ -334,6 +358,52 @@ config_service_v3(const config_line_t *line_,
       export_circuit_id = true;
       continue;
     }
+    if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSDefense")) {
+      config->has_dos_defense_enabled =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                                           HS_CONFIG_V3_DOS_DEFENSE_DEFAULT,
+                                           1, &ok);
+      if (!ok || dos_enabled) {
+        if (dos_enabled) {
+          dup_opt_seen = line->key;
+        }
+        goto err;
+      }
+      dos_enabled = true;
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSRatePerSec")) {
+      config->intro_dos_rate_per_sec =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                              HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN,
+                              HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX, &ok);
+      if (!ok || dos_rate_per_sec) {
+        if (dos_rate_per_sec) {
+          dup_opt_seen = line->key;
+        }
+        goto err;
+      }
+      dos_rate_per_sec = true;
+      log_info(LD_REND, "Service INTRO2 DoS defenses rate set to: %" PRIu32,
+               config->intro_dos_rate_per_sec);
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSBurstPerSec")) {
+      config->intro_dos_burst_per_sec =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                              HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN,
+                              HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX, &ok);
+      if (!ok || dos_burst_per_sec) {
+        if (dos_burst_per_sec) {
+          dup_opt_seen = line->key;
+        }
+        goto err;
+      }
+      dos_burst_per_sec = true;
+      log_info(LD_REND, "Service INTRO2 DoS defenses burst set to: %" PRIu32,
+               config->intro_dos_burst_per_sec);
+      continue;
+    }
   }
 
   /* We do not load the key material for the service at this stage. This is
@@ -496,15 +566,6 @@ config_generic_service(const config_line_t *line_,
    * becomes a single onion service. */
   if (rend_service_non_anonymous_mode_enabled(options)) {
     config->is_single_onion = 1;
-    /* We will add support for IPv6-only v3 single onion services in a future
-     * Tor version. This won't catch "ReachableAddresses reject *4", but that
-     * option doesn't work anyway. */
-    if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) {
-      log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not "
-               "supported. Set HiddenServiceSingleHopMode 0 and "
-               "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1.");
-      goto err;
-    }
   }
 
   /* Success */
diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h
index 040e451f13..beefc7a613 100644
--- a/src/feature/hs/hs_config.h
+++ b/src/feature/hs/hs_config.h
@@ -15,6 +15,15 @@
 #define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535
 /* Maximum number of intro points per version 3 services. */
 #define HS_CONFIG_V3_MAX_INTRO_POINTS 20
+/* Default value for the introduction DoS defenses. The MIN/MAX are inclusive
+ * meaning they can be used as valid values. */
+#define HS_CONFIG_V3_DOS_DEFENSE_DEFAULT 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT 25
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX INT32_MAX
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT 200
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX INT32_MAX
 
 /* API */
 
diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c
index 9970fdd123..abb421345c 100644
--- a/src/feature/hs/hs_control.c
+++ b/src/feature/hs/hs_control.c
@@ -7,9 +7,10 @@
  **/
 
 #include "core/or/or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_control.h"
 #include "feature/hs/hs_descriptor.h"
@@ -73,10 +74,7 @@ hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
   tor_assert(reason);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk,
@@ -98,10 +96,7 @@ hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk,
@@ -122,9 +117,7 @@ hs_control_desc_event_created(const char *onion_address,
   tor_assert(blinded_pk);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   /* Version 3 doesn't use the replica number in its descriptor ID computation
    * so we pass negative value so the control port subsystem can ignore it. */
@@ -150,9 +143,7 @@ hs_control_desc_event_upload(const char *onion_address,
   tor_assert(hsdir_index);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   control_event_hs_descriptor_upload(onion_address, hsdir_id_digest,
                                      base64_blinded_pk,
@@ -195,10 +186,7 @@ hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hs_descriptor_content(onion_address, base64_blinded_pk,
@@ -259,3 +247,16 @@ hs_control_hspost_command(const char *body, const char *onion_address,
   smartlist_free(hsdirs);
   return ret;
 }
+
+/* With a given <b>onion_identity_pk</b>, fetch its descriptor, optionally
+ * using the list of directory servers given in <b>hsdirs</b>, or a random
+ * server if it is NULL. This function calls hs_client_launch_v3_desc_fetch().
+ */
+void
+hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                           const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  hs_client_launch_v3_desc_fetch(onion_identity_pk, hsdirs);
+}
diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h
index f7ab642652..b55e4c53c9 100644
--- a/src/feature/hs/hs_control.h
+++ b/src/feature/hs/hs_control.h
@@ -48,5 +48,9 @@ void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
 int hs_control_hspost_command(const char *body, const char *onion_address,
                               const smartlist_t *hsdirs_rs);
 
+/* Command "HSFETCH [...]" */
+void hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                                const smartlist_t *hsdirs);
+
 #endif /* !defined(TOR_HS_CONTROL_H) */
 
diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c
index b6abf14a11..924ab3115e 100644
--- a/src/feature/hs/hs_descriptor.c
+++ b/src/feature/hs/hs_descriptor.c
@@ -324,12 +324,11 @@ encode_link_specifiers(const smartlist_t *specs)
 
   link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
 
-  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+  SMARTLIST_FOREACH_BEGIN(specs, const link_specifier_t *,
                           spec) {
-    link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec);
-    if (ls) {
-      link_specifier_list_add_spec(lslist, ls);
-    }
+    link_specifier_t *ls = link_specifier_dup(spec);
+    tor_assert(ls);
+    link_specifier_list_add_spec(lslist, ls);
   } SMARTLIST_FOREACH_END(spec);
 
   {
@@ -404,9 +403,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "enc-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->enc_key);
   if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
     goto done;
   }
@@ -422,7 +419,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
 }
 
 /* Encode an introduction point onion key. Return a newly allocated string
- * with it. On failure, return NULL. */
+ * with it. Can not fail. */
 static char *
 encode_onion_key(const hs_desc_intro_point_t *ip)
 {
@@ -432,12 +429,9 @@ encode_onion_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "onion-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->onion_key);
   tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64);
 
- done:
   return encoded;
 }
 
@@ -684,7 +678,7 @@ get_auth_client_str(const hs_desc_authorized_client_t *client)
   char encrypted_cookie_b64[HS_DESC_ENCRYPED_COOKIE_LEN * 2];
 
 #define ASSERT_AND_BASE64(field) STMT_BEGIN                        \
-  tor_assert(!tor_mem_is_zero((char *) client->field,              \
+  tor_assert(!fast_mem_is_zero((char *) client->field,              \
                               sizeof(client->field)));             \
   ret = base64_encode_nopad(field##_b64, sizeof(field##_b64),      \
                             client->field, sizeof(client->field)); \
@@ -798,8 +792,8 @@ get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
 /* Create the middle layer of the descriptor, which includes the client auth
  * data and the encrypted inner layer (provided as a base64 string at
  * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
- * layer plaintext, or NULL if an error occurred. It's the responsibility of
- * the caller to free the returned string. */
+ * layer plaintext. It's the responsibility of the caller to free the returned
+ * string. Can not fail. */
 static char *
 get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
                                     const char *layer2_b64_ciphertext)
@@ -815,13 +809,10 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
     const curve25519_public_key_t *ephemeral_pubkey;
 
     ephemeral_pubkey = &desc->superencrypted_data.auth_ephemeral_pubkey;
-    tor_assert(!tor_mem_is_zero((char *) ephemeral_pubkey->public_key,
+    tor_assert(!fast_mem_is_zero((char *) ephemeral_pubkey->public_key,
                                 CURVE25519_PUBKEY_LEN));
 
-    if (curve25519_public_to_base64(ephemeral_key_base64,
-                                    ephemeral_pubkey) < 0) {
-      goto done;
-    }
+    curve25519_public_to_base64(ephemeral_key_base64, ephemeral_pubkey);
     smartlist_add_asprintf(lines, "%s %s\n",
                            str_desc_auth_key, ephemeral_key_base64);
 
@@ -846,7 +837,6 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
 
   layer1_str = smartlist_join_strings(lines, "", 0, NULL);
 
- done:
   /* We need to memwipe all lines because it contains the ephemeral key */
   SMARTLIST_FOREACH(lines, char *, a, memwipe(a, 0, strlen(a)));
   SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
@@ -1092,11 +1082,7 @@ desc_encode_v3(const hs_descriptor_t *desc,
       tor_free(encoded_str);
       goto err;
     }
-    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
-      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
-      tor_free(encoded_str);
-      goto err;
-    }
+    ed25519_signature_to_base64(ed_sig_b64, &sig);
     /* Create the signature line. */
     smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
   }
@@ -1190,52 +1176,22 @@ decode_link_specifiers(const char *encoded)
   results = smartlist_new();
 
   for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
-    hs_desc_link_specifier_t *hs_spec;
     link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
-    tor_assert(ls);
-
-    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
-    hs_spec->type = link_specifier_get_ls_type(ls);
-    switch (hs_spec->type) {
-    case LS_IPV4:
-      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
-                          link_specifier_get_un_ipv4_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
-      break;
-    case LS_IPV6:
-      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
-                               link_specifier_getarray_un_ipv6_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
-      break;
-    case LS_LEGACY_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
-                 sizeof(hs_spec->u.legacy_id));
-      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
-             sizeof(hs_spec->u.legacy_id));
-      break;
-    case LS_ED25519_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_ed25519_id(ls) ==
-                 sizeof(hs_spec->u.ed25519_id));
-      memcpy(hs_spec->u.ed25519_id,
-             link_specifier_getconstarray_un_ed25519_id(ls),
-             sizeof(hs_spec->u.ed25519_id));
-      break;
-    default:
-      tor_free(hs_spec);
+    if (BUG(!ls)) {
       goto err;
     }
-
-    smartlist_add(results, hs_spec);
+    link_specifier_t *ls_dup = link_specifier_dup(ls);
+    if (BUG(!ls_dup)) {
+      goto err;
+    }
+    smartlist_add(results, ls_dup);
   }
 
   goto done;
  err:
   if (results) {
-    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    SMARTLIST_FOREACH(results, link_specifier_t *, s,
+                      link_specifier_free(s));
     smartlist_free(results);
     results = NULL;
   }
@@ -1400,6 +1356,50 @@ encrypted_data_length_is_valid(size_t len)
   return 0;
 }
 
+/* Build the KEYS component for the authorized client computation. The format
+ * of the construction is:
+ *
+ *    SECRET_SEED = x25519(sk, pk)
+ *    KEYS = KDF(subcredential | SECRET_SEED, 40)
+ *
+ * Set the <b>keys_out</b> argument to point to the buffer containing the KEYS,
+ * and return the buffer's length. The caller should wipe and free its content
+ * once done with it. This function can't fail. */
+static size_t
+build_descriptor_cookie_keys(const uint8_t *subcredential,
+                             size_t subcredential_len,
+                             const curve25519_secret_key_t *sk,
+                             const curve25519_public_key_t *pk,
+                             uint8_t **keys_out)
+{
+  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
+  uint8_t *keystream;
+  size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN;
+  crypto_xof_t *xof;
+
+  tor_assert(subcredential);
+  tor_assert(sk);
+  tor_assert(pk);
+  tor_assert(keys_out);
+
+  keystream = tor_malloc_zero(keystream_len);
+
+  /* Calculate x25519(sk, pk) to get the secret seed. */
+  curve25519_handshake(secret_seed, sk, pk);
+
+  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, subcredential, subcredential_len);
+  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
+  crypto_xof_squeeze_bytes(xof, keystream, keystream_len);
+  crypto_xof_free(xof);
+
+  memwipe(secret_seed, 0, sizeof(secret_seed));
+
+  *keys_out = keystream;
+  return keystream_len;
+}
+
 /* Decrypt the descriptor cookie given the descriptor, the auth client,
  * and the client secret key. On sucess, return 0 and a newly allocated
  * descriptor cookie descriptor_cookie_out. On error or if the client id
@@ -1412,33 +1412,29 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
                           uint8_t **descriptor_cookie_out)
 {
   int ret = -1;
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key = NULL;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
   uint8_t *descriptor_cookie = NULL;
+  const uint8_t *cookie_key = NULL;
   crypto_cipher_t *cipher = NULL;
-  crypto_xof_t *xof = NULL;
 
   tor_assert(desc);
   tor_assert(client);
   tor_assert(client_auth_sk);
-  tor_assert(!tor_mem_is_zero(
+  tor_assert(!fast_mem_is_zero(
         (char *) &desc->superencrypted_data.auth_ephemeral_pubkey,
         sizeof(desc->superencrypted_data.auth_ephemeral_pubkey)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_sk,
+  tor_assert(!fast_mem_is_zero((char *) client_auth_sk,
                               sizeof(*client_auth_sk)));
-  tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN));
+  tor_assert(!fast_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN));
 
-  /* Calculate x25519(client_x, hs_Y) */
-  curve25519_handshake(secret_seed, client_auth_sk,
-                       &desc->superencrypted_data.auth_ephemeral_pubkey);
-
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN,
+                             client_auth_sk,
+                             &desc->superencrypted_data.auth_ephemeral_pubkey,
+                             &keystream);
+  tor_assert(keystream_length > 0);
 
   /* If the client id of auth client is not the same as the calculcated
    * client id, it means that this auth client is invaild according to the
@@ -1464,8 +1460,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
   if (cipher) {
     crypto_cipher_free(cipher);
   }
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
   return ret;
 }
 
@@ -1481,10 +1477,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
  */
 MOCK_IMPL(STATIC size_t,
 decrypt_desc_layer,(const hs_descriptor_t *desc,
-                    const uint8_t *encrypted_blob,
-                    size_t encrypted_blob_size,
                     const uint8_t *descriptor_cookie,
-                    int is_superencrypted_layer,
+                    bool is_superencrypted_layer,
                     char **decrypted_out))
 {
   uint8_t *decrypted = NULL;
@@ -1494,6 +1488,12 @@ decrypt_desc_layer,(const hs_descriptor_t *desc,
   uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
   const uint8_t *salt, *encrypted, *desc_mac;
   size_t encrypted_len, result_len = 0;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
 
   tor_assert(decrypted_out);
   tor_assert(desc);
@@ -1607,9 +1607,8 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out)
   tor_assert(decrypted_out);
 
   superencrypted_len = decrypt_desc_layer(desc,
-                                 desc->plaintext_data.superencrypted_blob,
-                                 desc->plaintext_data.superencrypted_blob_size,
-                                 NULL, 1, &superencrypted_plaintext);
+                                          NULL,
+                                          true, &superencrypted_plaintext);
 
   if (!superencrypted_len) {
     log_warn(LD_REND, "Decrypting superencrypted desc failed.");
@@ -1658,9 +1657,9 @@ desc_decrypt_encrypted(const hs_descriptor_t *desc,
   }
 
   encrypted_len = decrypt_desc_layer(desc,
-                                 desc->superencrypted_data.encrypted_blob,
-                                 desc->superencrypted_data.encrypted_blob_size,
-                                 descriptor_cookie, 0, &encrypted_plaintext);
+                                     descriptor_cookie,
+                                     false, &encrypted_plaintext);
+
   if (!encrypted_len) {
     goto err;
   }
@@ -1972,6 +1971,7 @@ decode_intro_points(const hs_descriptor_t *desc,
   SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
   smartlist_free(intro_points);
 }
+
 /* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
  * descriptor in encoded_desc validates the descriptor content. */
 STATIC int
@@ -2575,7 +2575,7 @@ hs_desc_decode_descriptor(const char *encoded,
 
   /* Subcredentials are not optional. */
   if (BUG(!subcredential ||
-          tor_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
+          fast_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
     log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!");
     goto err;
   }
@@ -2838,8 +2838,8 @@ hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
     return;
   }
   if (ip->link_specifiers) {
-    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
-                      ls, hs_desc_link_specifier_free(ls));
+    SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *,
+                      ls, link_specifier_free(ls));
     smartlist_free(ip->link_specifiers);
   }
   tor_cert_free(ip->auth_key_cert);
@@ -2878,38 +2878,33 @@ hs_desc_build_authorized_client(const uint8_t *subcredential,
                                 const uint8_t *descriptor_cookie,
                                 hs_desc_authorized_client_t *client_out)
 {
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
+  const uint8_t *cookie_key;
   crypto_cipher_t *cipher;
-  crypto_xof_t *xof;
 
   tor_assert(client_auth_pk);
   tor_assert(auth_ephemeral_sk);
   tor_assert(descriptor_cookie);
   tor_assert(client_out);
   tor_assert(subcredential);
-  tor_assert(!tor_mem_is_zero((char *) auth_ephemeral_sk,
+  tor_assert(!fast_mem_is_zero((char *) auth_ephemeral_sk,
                               sizeof(*auth_ephemeral_sk)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_pk,
+  tor_assert(!fast_mem_is_zero((char *) client_auth_pk,
                               sizeof(*client_auth_pk)));
-  tor_assert(!tor_mem_is_zero((char *) descriptor_cookie,
+  tor_assert(!fast_mem_is_zero((char *) descriptor_cookie,
                               HS_DESC_DESCRIPTOR_COOKIE_LEN));
-  tor_assert(!tor_mem_is_zero((char *) subcredential,
+  tor_assert(!fast_mem_is_zero((char *) subcredential,
                               DIGEST256_LEN));
 
-  /* Calculate x25519(hs_y, client_X) */
-  curve25519_handshake(secret_seed,
-                       auth_ephemeral_sk,
-                       client_auth_pk);
-
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(subcredential, DIGEST256_LEN,
+                                 auth_ephemeral_sk, client_auth_pk,
+                                 &keystream);
+  tor_assert(keystream_length > 0);
 
+  /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */
   memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN);
   cookie_key = keystream + HS_DESC_CLIENT_ID_LEN;
 
@@ -2924,8 +2919,8 @@ hs_desc_build_authorized_client(const uint8_t *subcredential,
                         (const char *) descriptor_cookie,
                         HS_DESC_DESCRIPTOR_COOKIE_LEN);
 
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
 
   crypto_cipher_free(cipher);
 }
@@ -2937,69 +2932,6 @@ hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client)
   tor_free(client);
 }
 
-/* Free the given descriptor link specifier. */
-void
-hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls)
-{
-  if (ls == NULL) {
-    return;
-  }
-  tor_free(ls);
-}
-
-/* Return a newly allocated descriptor link specifier using the given extend
- * info and requested type. Return NULL on error. */
-hs_desc_link_specifier_t *
-hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type)
-{
-  hs_desc_link_specifier_t *ls = NULL;
-
-  tor_assert(info);
-
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = type;
-  switch (ls->type) {
-  case LS_IPV4:
-    if (info->addr.family != AF_INET) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_IPV6:
-    if (info->addr.family != AF_INET6) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_LEGACY_ID:
-    /* Bug out if the identity digest is not set */
-    if (BUG(tor_mem_is_zero(info->identity_digest,
-                            sizeof(info->identity_digest)))) {
-      goto err;
-    }
-    memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id));
-    break;
-  case LS_ED25519_ID:
-    /* ed25519 keys are optional for intro points */
-    if (ed25519_public_key_is_zero(&info->ed_identity)) {
-      goto err;
-    }
-    memcpy(ls->u.ed25519_id, info->ed_identity.pubkey,
-           sizeof(ls->u.ed25519_id));
-    break;
-  default:
-    /* Unknown type is code flow error. */
-    tor_assert(0);
-  }
-
-  return ls;
- err:
-  tor_free(ls);
-  return NULL;
-}
-
 /* From the given descriptor, remove and free every introduction point. */
 void
 hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
@@ -3015,59 +2947,3 @@ hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
     smartlist_clear(ips);
   }
 }
-
-/* From a descriptor link specifier object spec, returned a newly allocated
- * link specifier object that is the encoded representation of spec. Return
- * NULL on error. */
-link_specifier_t *
-hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec)
-{
-  tor_assert(spec);
-
-  link_specifier_t *ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, spec->type);
-
-  switch (spec->type) {
-  case LS_IPV4:
-    link_specifier_set_un_ipv4_addr(ls,
-                                    tor_addr_to_ipv4h(&spec->u.ap.addr));
-    link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
-    /* Four bytes IPv4 and two bytes port. */
-    link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
-                                  sizeof(spec->u.ap.port));
-    break;
-  case LS_IPV6:
-  {
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
-    break;
-  }
-  case LS_LEGACY_ID:
-  {
-    size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
-    uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
-    memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
-    link_specifier_set_ls_len(ls, legacy_id_len);
-    break;
-  }
-  case LS_ED25519_ID:
-  {
-    size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls);
-    uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls);
-    memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len);
-    link_specifier_set_ls_len(ls, ed25519_id_len);
-    break;
-  }
-  default:
-    tor_assert_nonfatal_unreached();
-    link_specifier_free(ls);
-    ls = NULL;
-  }
-
-  return ls;
-}
diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h
index 04a8e16d63..0a843f4f3c 100644
--- a/src/feature/hs/hs_descriptor.h
+++ b/src/feature/hs/hs_descriptor.h
@@ -69,28 +69,10 @@ typedef enum {
   HS_DESC_AUTH_ED25519 = 1
 } hs_desc_auth_type_t;
 
-/* Link specifier object that contains information on how to extend to the
- * relay that is the address, port and handshake type. */
-typedef struct hs_desc_link_specifier_t {
-  /* Indicate the type of link specifier. See trunnel ed25519_cert
-   * specification. */
-  uint8_t type;
-
-  /* It must be one of these types, can't be more than one. */
-  union {
-    /* IP address and port of the relay use to extend. */
-    tor_addr_port_t ap;
-    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
-    uint8_t legacy_id[DIGEST_LEN];
-    /* ed25519 identity. A 32-byte key. */
-    uint8_t ed25519_id[ED25519_PUBKEY_LEN];
-  } u;
-} hs_desc_link_specifier_t;
-
 /* Introduction point information located in a descriptor. */
 typedef struct hs_desc_intro_point_t {
   /* Link specifier(s) which details how to extend to the relay. This list
-   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+   * contains link_specifier_t objects. It MUST have at least one. */
   smartlist_t *link_specifiers;
 
   /* Onion key of the introduction point used to extend to it for the ntor
@@ -261,12 +243,6 @@ void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc);
 #define hs_desc_encrypted_data_free(desc) \
   FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc))
 
-void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls);
-#define hs_desc_link_specifier_free(ls) \
-  FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls))
-
-hs_desc_link_specifier_t *hs_desc_link_specifier_new(
-                                  const extend_info_t *info, uint8_t type);
 void hs_descriptor_clear_intro_points(hs_descriptor_t *desc);
 
 MOCK_DECL(int,
@@ -299,10 +275,8 @@ void hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client);
   FREE_AND_NULL(hs_desc_authorized_client_t, \
                 hs_desc_authorized_client_free_, (client))
 
-link_specifier_t *hs_desc_lspec_to_trunnel(
-                                   const hs_desc_link_specifier_t *spec);
-
 hs_desc_authorized_client_t *hs_desc_build_fake_authorized_client(void);
+
 void hs_desc_build_authorized_client(const uint8_t *subcredential,
                                      const curve25519_public_key_t *
                                      client_auth_pk,
@@ -335,10 +309,8 @@ STATIC int desc_sig_is_valid(const char *b64_sig,
                              const char *encoded_desc, size_t encoded_len);
 
 MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc,
-                                             const uint8_t *encrypted_blob,
-                                             size_t encrypted_blob_size,
                                              const uint8_t *descriptor_cookie,
-                                             int is_superencrypted_layer,
+                                             bool is_superencrypted_layer,
                                              char **decrypted_out));
 
 #endif /* defined(HS_DESCRIPTOR_PRIVATE) */
diff --git a/src/feature/hs/hs_dos.c b/src/feature/hs/hs_dos.c
new file mode 100644
index 0000000000..19794e09d3
--- /dev/null
+++ b/src/feature/hs/hs_dos.c
@@ -0,0 +1,200 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.c
+ * \brief Implement denial of service mitigation for the onion service
+ *        subsystem.
+ *
+ * This module defenses:
+ *
+ * - Introduction Rate Limiting: If enabled by the consensus, an introduction
+ *   point will rate limit client introduction towards the service (INTRODUCE2
+ *   cells). It uses a token bucket model with a rate and burst per second.
+ *
+ *   Proposal 305 will expand this module by allowing an operator to define
+ *   these values into the ESTABLISH_INTRO cell. Not yet implemented.
+ **/
+
+#define HS_DOS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/routermode.h"
+
+#include "lib/evloop/token_bucket.h"
+
+#include "feature/hs/hs_dos.h"
+
+/* Default value of the allowed INTRODUCE2 cell rate per second. Above that
+ * value per second, the introduction is denied. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC 25
+
+/* Default value of the allowed INTRODUCE2 cell burst per second. This is the
+ * maximum value a token bucket has per second. We thus allow up to this value
+ * of INTRODUCE2 cell per second but the bucket is refilled by the rate value
+ * but never goes above that burst value. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC 200
+
+/* Default value of the consensus parameter enabling or disabling the
+ * introduction DoS defense. Disabled by default. */
+#define HS_DOS_INTRODUCE_ENABLED_DEFAULT 0
+
+/* Consensus parameters. The ESTABLISH_INTRO DoS cell extension have higher
+ * priority than these values. If no extension is sent, these are used only by
+ * the introduction point. */
+static uint32_t consensus_param_introduce_rate_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC;
+static uint32_t consensus_param_introduce_burst_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC;
+static uint32_t consensus_param_introduce_defense_enabled =
+  HS_DOS_INTRODUCE_ENABLED_DEFAULT;
+
+STATIC uint32_t
+get_intro2_enable_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSDefense",
+                                 HS_DOS_INTRODUCE_ENABLED_DEFAULT, 0, 1);
+}
+
+/* Return the parameter for the introduction rate per sec. */
+STATIC uint32_t
+get_intro2_rate_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSRatePerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/* Return the parameter for the introduction burst per sec. */
+STATIC uint32_t
+get_intro2_burst_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSBurstPerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/* Go over all introduction circuit relay side and adjust their rate/burst
+ * values using the global parameters. This is called right after the
+ * consensus parameters might have changed. */
+static void
+update_intro_circuits(void)
+{
+  /* Returns all HS version intro circuits. */
+  smartlist_t *intro_circs = hs_circuitmap_get_all_intro_circ_relay_side();
+
+  SMARTLIST_FOREACH_BEGIN(intro_circs, circuit_t *, circ) {
+    /* Defenses might have been enabled or disabled. */
+    TO_OR_CIRCUIT(circ)->introduce2_dos_defense_enabled =
+      consensus_param_introduce_defense_enabled;
+    /* Adjust the rate/burst value that might have changed. */
+    token_bucket_ctr_adjust(&TO_OR_CIRCUIT(circ)->introduce2_bucket,
+                            consensus_param_introduce_rate_per_sec,
+                            consensus_param_introduce_burst_per_sec);
+  } SMARTLIST_FOREACH_END(circ);
+
+  smartlist_free(intro_circs);
+}
+
+/* Set consensus parameters. */
+static void
+set_consensus_parameters(const networkstatus_t *ns)
+{
+  consensus_param_introduce_rate_per_sec =
+    get_intro2_rate_consensus_param(ns);
+  consensus_param_introduce_burst_per_sec =
+    get_intro2_burst_consensus_param(ns);
+  consensus_param_introduce_defense_enabled =
+    get_intro2_enable_consensus_param(ns);
+
+  /* The above might have changed which means we need to go through all
+   * introduction circuits (relay side) and update the token buckets. */
+  update_intro_circuits();
+}
+
+/*
+ * Public API.
+ */
+
+/* Initialize the INTRODUCE2 token bucket for the DoS defenses using the
+ * consensus/default values. We might get a cell extension that changes those
+ * later but if we don't, the default or consensus parameters are used. */
+void
+hs_dos_setup_default_intro2_defenses(or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  circ->introduce2_dos_defense_enabled =
+    consensus_param_introduce_defense_enabled;
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        consensus_param_introduce_rate_per_sec,
+                        consensus_param_introduce_burst_per_sec,
+                        (uint32_t) approx_time());
+}
+
+/* Called when the consensus has changed. We might have new consensus
+ * parameters to look at. */
+void
+hs_dos_consensus_has_changed(const networkstatus_t *ns)
+{
+  /* No point on updating these values if we are not a public relay that can
+   * be picked to be an introduction point. */
+  if (!public_server_mode(get_options())) {
+    return;
+  }
+
+  set_consensus_parameters(ns);
+}
+
+/* Return true iff an INTRODUCE2 cell can be sent on the given service
+ * introduction circuit. */
+bool
+hs_dos_can_send_intro2(or_circuit_t *s_intro_circ)
+{
+  tor_assert(s_intro_circ);
+
+  /* Allow to send the cell if the DoS defenses are disabled on the circuit.
+   * This can be set by the consensus, the ESTABLISH_INTRO cell extension or
+   * the hardcoded values in tor code. */
+  if (!s_intro_circ->introduce2_dos_defense_enabled) {
+    return true;
+  }
+
+  /* Should not happen but if so, scream loudly. */
+  if (BUG(TO_CIRCUIT(s_intro_circ)->purpose != CIRCUIT_PURPOSE_INTRO_POINT)) {
+    return false;
+  }
+
+  /* This is called just after we got a valid and parsed INTRODUCE1 cell. The
+   * service has been found and we have its introduction circuit.
+   *
+   * First, the INTRODUCE2 bucket will be refilled (if any). Then, decremented
+   * because we are about to send or not the cell we just got. Finally,
+   * evaluate if we can send it based on our token bucket state. */
+
+  /* Refill INTRODUCE2 bucket. */
+  token_bucket_ctr_refill(&s_intro_circ->introduce2_bucket,
+                          (uint32_t) approx_time());
+
+  /* Decrement the bucket for this valid INTRODUCE1 cell we just got. Don't
+   * underflow else we end up with a too big of a bucket. */
+  if (token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0) {
+    token_bucket_ctr_dec(&s_intro_circ->introduce2_bucket, 1);
+  }
+
+  /* Finally, we can send a new INTRODUCE2 if there are still tokens. */
+  return token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0;
+}
+
+/* Initialize the onion service Denial of Service subsystem. */
+void
+hs_dos_init(void)
+{
+  set_consensus_parameters(NULL);
+}
diff --git a/src/feature/hs/hs_dos.h b/src/feature/hs/hs_dos.h
new file mode 100644
index 0000000000..ccf4e27179
--- /dev/null
+++ b/src/feature/hs/hs_dos.h
@@ -0,0 +1,39 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.h
+ * \brief Header file containing denial of service defenses for the HS
+ *        subsystem for all versions.
+ **/
+
+#ifndef TOR_HS_DOS_H
+#define TOR_HS_DOS_H
+
+#include "core/or/or_circuit_st.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+/* Init */
+void hs_dos_init(void);
+
+/* Consensus. */
+void hs_dos_consensus_has_changed(const networkstatus_t *ns);
+
+/* Introduction Point. */
+bool hs_dos_can_send_intro2(or_circuit_t *s_intro_circ);
+void hs_dos_setup_default_intro2_defenses(or_circuit_t *circ);
+
+#ifdef HS_DOS_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC uint32_t get_intro2_enable_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_rate_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_burst_consensus_param(const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_DOS_PRIVATE) */
+
+#endif /* !defined(TOR_HS_DOS_H) */
diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c
index 8fd0013941..a00e55ec23 100644
--- a/src/feature/hs/hs_ident.c
+++ b/src/feature/hs/hs_ident.c
@@ -13,14 +13,10 @@
 /* Return a newly allocated circuit identifier. The given public key is copied
  * identity_pk into the identifier. */
 hs_ident_circuit_t *
-hs_ident_circuit_new(const ed25519_public_key_t *identity_pk,
-                     hs_ident_circuit_type_t circuit_type)
+hs_ident_circuit_new(const ed25519_public_key_t *identity_pk)
 {
-  tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO ||
-             circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS);
   hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
   ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
-  ident->circuit_type = circuit_type;
   return ident;
 }
 
diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h
index 8c46936a1e..82ca50f6b5 100644
--- a/src/feature/hs/hs_ident.h
+++ b/src/feature/hs/hs_ident.h
@@ -44,13 +44,6 @@ typedef struct hs_ident_circuit_t {
    * the one found in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* (All circuit) The type of circuit this identifier is attached to.
-   * Accessors of the fields in this object assert non fatal on this circuit
-   * type. In other words, if a rendezvous field is being accessed, the
-   * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is
-   * set when an object is initialized in its constructor. */
-  hs_ident_circuit_type_t circuit_type;
-
   /* (All circuit) Introduction point authentication key. It's also needed on
    * the rendezvous circuit for the ntor handshake. It's used as the unique key
    * of the introduction point so it should not be shared between multiple
@@ -120,8 +113,7 @@ typedef struct hs_ident_edge_conn_t {
 
 /* Circuit identifier API. */
 hs_ident_circuit_t *hs_ident_circuit_new(
-                             const ed25519_public_key_t *identity_pk,
-                             hs_ident_circuit_type_t circuit_type);
+                             const ed25519_public_key_t *identity_pk);
 void hs_ident_circuit_free_(hs_ident_circuit_t *ident);
 #define hs_ident_circuit_free(id) \
   FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id))
diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c
index 7717ed53d4..fe8486b1a6 100644
--- a/src/feature/hs/hs_intropoint.c
+++ b/src/feature/hs/hs_intropoint.c
@@ -10,6 +10,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/relay.h"
@@ -24,9 +25,11 @@
 #include "trunnel/hs/cell_introduce1.h"
 
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
 #include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
-#include "feature/hs/hs_common.h"
 
 #include "core/or/or_circuit_st.h"
 
@@ -179,6 +182,185 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ))
   return ret;
 }
 
+/* Validate the cell DoS extension parameters. Return true iff they've been
+ * bound check and can be used. Else return false. See proposal 305 for
+ * details and reasons about this validation. */
+STATIC bool
+cell_dos_extension_parameters_are_valid(uint64_t intro2_rate_per_sec,
+                                        uint64_t intro2_burst_per_sec)
+{
+  bool ret = false;
+
+  /* Check that received value is not below the minimum. Don't check if minimum
+     is set to 0, since the param is a positive value and gcc will complain. */
+#if HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0
+  if (intro2_rate_per_sec < HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too small. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_rate_per_sec > HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too big. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+
+  /* Check that received value is not below the minimum */
+#if HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0
+  if (intro2_burst_per_sec < HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too small. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_burst_per_sec > HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too big. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* In a rate limiting scenario, burst can never be smaller than the rate. At
+   * best it can be equal. */
+  if (intro2_burst_per_sec < intro2_rate_per_sec) {
+    log_info(LD_REND, "Intro point DoS defenses burst is smaller than rate. "
+                      "Rate: %" PRIu64 " vs Burst: %" PRIu64,
+             intro2_rate_per_sec, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* Passing validation. */
+  ret = true;
+
+ end:
+  return ret;
+}
+
+/* Parse the cell DoS extension and apply defenses on the given circuit if
+ * validation passes. If the cell extension is malformed or contains unusable
+ * values, the DoS defenses is disabled on the circuit. */
+static void
+handle_establish_intro_cell_dos_extension(
+                                const trn_cell_extension_field_t *field,
+                                or_circuit_t *circ)
+{
+  ssize_t ret;
+  uint64_t intro2_rate_per_sec = 0, intro2_burst_per_sec = 0;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  tor_assert(field);
+  tor_assert(circ);
+
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  if (ret < 0) {
+    goto end;
+  }
+
+  for (size_t i = 0; i < trn_cell_extension_dos_get_n_params(dos); i++) {
+    const trn_cell_extension_dos_param_t *param =
+      trn_cell_extension_dos_getconst_params(dos, i);
+    if (BUG(param == NULL)) {
+      goto end;
+    }
+
+    switch (trn_cell_extension_dos_param_get_type(param)) {
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC:
+      intro2_rate_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC:
+      intro2_burst_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    default:
+      goto end;
+    }
+  }
+
+  /* A value of 0 is valid in the sense that we accept it but we still disable
+   * the defenses so return false. */
+  if (intro2_rate_per_sec == 0 || intro2_burst_per_sec == 0) {
+    log_info(LD_REND, "Intro point DoS defenses parameter set to 0. "
+                      "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    circ->introduce2_dos_defense_enabled = 0;
+    goto end;
+  }
+
+  /* If invalid, we disable the defense on the circuit. */
+  if (!cell_dos_extension_parameters_are_valid(intro2_rate_per_sec,
+                                               intro2_burst_per_sec)) {
+    circ->introduce2_dos_defense_enabled = 0;
+    log_info(LD_REND, "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    goto end;
+  }
+
+  /* We passed validation, enable defenses and apply rate/burst. */
+  circ->introduce2_dos_defense_enabled = 1;
+
+  /* Initialize the INTRODUCE2 token bucket for the rate limiting. */
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        (uint32_t) intro2_rate_per_sec,
+                        (uint32_t) intro2_burst_per_sec,
+                        (uint32_t) approx_time());
+  log_info(LD_REND, "Intro point DoS defenses enabled. Rate is %" PRIu64
+                    " and Burst is %" PRIu64,
+           intro2_rate_per_sec, intro2_burst_per_sec);
+
+ end:
+  trn_cell_extension_dos_free(dos);
+  return;
+}
+
+/* Parse every cell extension in the given ESTABLISH_INTRO cell. */
+static void
+handle_establish_intro_cell_extensions(
+                            const trn_cell_establish_intro_t *parsed_cell,
+                            or_circuit_t *circ)
+{
+  const trn_cell_extension_t *extensions;
+
+  tor_assert(parsed_cell);
+  tor_assert(circ);
+
+  extensions = trn_cell_establish_intro_getconst_extensions(parsed_cell);
+  if (extensions == NULL) {
+    goto end;
+  }
+
+  /* Go over all extensions. */
+  for (size_t idx = 0; idx < trn_cell_extension_get_num(extensions); idx++) {
+    const trn_cell_extension_field_t *field =
+      trn_cell_extension_getconst_fields(extensions, idx);
+    if (BUG(field == NULL)) {
+      /* The number of extensions should match the number of fields. */
+      break;
+    }
+
+    switch (trn_cell_extension_field_get_field_type(field)) {
+    case TRUNNEL_CELL_EXTENSION_TYPE_DOS:
+      /* After this, the circuit should be set for DoS defenses. */
+      handle_establish_intro_cell_dos_extension(field, circ);
+      break;
+    default:
+      /* Unknown extension. Skip over. */
+      break;
+    }
+  }
+
+ end:
+  return;
+}
+
 /** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
  *  well-formed and passed our verifications. Perform appropriate actions to
  *  establish an intro point. */
@@ -191,6 +373,13 @@ handle_verified_establish_intro_cell(or_circuit_t *circ,
   get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
                          parsed_cell);
 
+  /* Setup INTRODUCE2 defenses on the circuit. Must be done before parsing the
+   * cell extension that can possibly change the defenses' values. */
+  hs_dos_setup_default_intro2_defenses(circ);
+
+  /* Handle cell extension if any. */
+  handle_establish_intro_cell_extensions(parsed_cell, circ);
+
   /* Then notify the hidden service that the intro point is established by
      sending an INTRO_ESTABLISHED cell */
   if (hs_intro_send_intro_established_cell(circ)) {
@@ -392,7 +581,7 @@ validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
    * safety net here. The legacy ID must be zeroes in this case. */
   legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
   legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
-  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+  if (BUG(!fast_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
     goto invalid;
   }
 
@@ -480,6 +669,20 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
     }
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(service_circ)) {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(5 * 60);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v3 cell due to DoS "
+                            "limitations. Sending NACK to client.");
+      tor_free(msg);
+    }
+    status = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+    goto send_ack;
+  }
+
   /* Relay the cell to the service on its intro circuit with an INTRODUCE2
    * cell which is the same exact payload. */
   if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
@@ -518,7 +721,7 @@ introduce1_cell_is_legacy(const uint8_t *request)
 
   /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
    * indicates a legacy cell (v2). */
-  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+  if (!fast_mem_is_zero((const char *) request, DIGEST_LEN)) {
     /* Legacy cell. */
     return 1;
   }
@@ -546,6 +749,14 @@ circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
     return 0;
   }
 
+  /* Disallow single hop client circuit. */
+  if (circ->p_chan && channel_is_client(circ->p_chan)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Single hop client was rejected while trying to introduce. "
+           "Closing circuit.");
+    return 0;
+  }
+
   return 1;
 }
 
@@ -602,8 +813,8 @@ hs_intropoint_clear(hs_intropoint_t *ip)
     return;
   }
   tor_cert_free(ip->auth_key_cert);
-  SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls,
-                    hs_desc_link_specifier_free(ls));
+  SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *, ls,
+                    link_specifier_free(ls));
   smartlist_free(ip->link_specifiers);
   memset(ip, 0, sizeof(hs_intropoint_t));
 }
diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h
index e82575f052..94ebf021e4 100644
--- a/src/feature/hs/hs_intropoint.h
+++ b/src/feature/hs/hs_intropoint.h
@@ -57,6 +57,9 @@ STATIC int handle_introduce1(or_circuit_t *client_circ,
                              const uint8_t *request, size_t request_len);
 STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
 STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+STATIC bool cell_dos_extension_parameters_are_valid(
+                                          uint64_t intro2_rate_per_sec,
+                                          uint64_t intro2_burst_per_sec);
 
 #endif /* defined(HS_INTROPOINT_PRIVATE) */
 
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
index 6d32cae86c..17ac4fa4a9 100644
--- a/src/feature/hs/hs_service.c
+++ b/src/feature/hs/hs_service.c
@@ -242,6 +242,9 @@ set_service_default_config(hs_service_config_t *c,
   c->is_single_onion = 0;
   c->dir_group_readable = 0;
   c->is_ephemeral = 0;
+  c->has_dos_defense_enabled = HS_CONFIG_V3_DOS_DEFENSE_DEFAULT;
+  c->intro_dos_rate_per_sec = HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT;
+  c->intro_dos_burst_per_sec = HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT;
 }
 
 /* From a service configuration object config, clear everything from it
@@ -280,9 +283,10 @@ describe_intro_point(const hs_service_intro_point_t *ip)
   const char *legacy_id = NULL;
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, lspec) {
-    if (lspec->type == LS_LEGACY_ID) {
-      legacy_id = (const char *) lspec->u.legacy_id;
+                          const link_specifier_t *, lspec) {
+    if (link_specifier_get_ls_type(lspec) == LS_LEGACY_ID) {
+      legacy_id = (const char *)
+        link_specifier_getconstarray_un_legacy_id(lspec);
       break;
     }
   } SMARTLIST_FOREACH_END(lspec);
@@ -426,23 +430,16 @@ service_intro_point_free_void(void *obj)
 }
 
 /* Return a newly allocated service intro point and fully initialized from the
- * given extend_info_t ei if non NULL.
- * If is_legacy is true, we also generate the legacy key.
- * If supports_ed25519_link_handshake_any is true, we add the relay's ed25519
- * key to the link specifiers.
+ * given node_t node, if non NULL.
  *
- * If ei is NULL, returns a hs_service_intro_point_t with an empty link
+ * If node is NULL, returns a hs_service_intro_point_t with an empty link
  * specifier list and no onion key. (This is used for testing.)
  * On any other error, NULL is returned.
  *
- * ei must be an extend_info_t containing an IPv4 address. (We will add supoort
- * for IPv6 in a later release.) When calling extend_info_from_node(), pass
- * 0 in for_direct_connection to make sure ei always has an IPv4 address. */
+ * node must be an node_t with an IPv4 address. */
 STATIC hs_service_intro_point_t *
-service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
-                        unsigned int supports_ed25519_link_handshake_any)
+service_intro_point_new(const node_t *node)
 {
-  hs_desc_link_specifier_t *ls;
   hs_service_intro_point_t *ip;
 
   ip = tor_malloc_zero(sizeof(*ip));
@@ -472,12 +469,17 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
   ip->replay_cache = replaycache_new(0, 0);
 
   /* Initialize the base object. We don't need the certificate object. */
-  ip->base.link_specifiers = smartlist_new();
+  ip->base.link_specifiers = node_get_link_specifier_smartlist(node, 0);
+
+  if (node == NULL) {
+    goto done;
+  }
 
   /* Generate the encryption key for this intro point. */
   curve25519_keypair_generate(&ip->enc_key_kp, 0);
-  /* Figure out if this chosen node supports v3 or is legacy only. */
-  if (is_legacy) {
+  /* Figure out if this chosen node supports v3 or is legacy only.
+   * NULL nodes are used in the unit tests. */
+  if (!node_supports_ed25519_hs_intro(node)) {
     ip->base.is_only_legacy = 1;
     /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */
     ip->legacy_key = crypto_pk_new();
@@ -490,40 +492,13 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
     }
   }
 
-  if (ei == NULL) {
-    goto done;
-  }
+  /* Flag if this intro point supports the INTRO2 dos defenses. */
+  ip->support_intro2_dos_defense =
+    node_supports_establish_intro_dos_extension(node);
 
-  /* We'll try to add all link specifiers. Legacy is mandatory.
-   * IPv4 or IPv6 is required, and we always send IPv4. */
-  ls = hs_desc_link_specifier_new(ei, LS_IPV4);
-  /* It is impossible to have an extend info object without a v4. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID);
-  /* It is impossible to have an extend info object without an identity
-   * digest. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  /* ed25519 identity key is optional for intro points. If the node supports
-   * ed25519 link authentication, we include it. */
-  if (supports_ed25519_link_handshake_any) {
-    ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID);
-    if (ls) {
-      smartlist_add(ip->base.link_specifiers, ls);
-    }
-  }
-
-  /* IPv6 is not supported in this release. */
-
-  /* Finally, copy onion key from the extend_info_t object. */
-  memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key));
+  /* Finally, copy onion key from the node. */
+  memcpy(&ip->onion_key, node_get_curve25519_onion_key(node),
+         sizeof(ip->onion_key));
 
  done:
   return ip;
@@ -656,16 +631,16 @@ get_objects_from_ident(const hs_ident_circuit_t *ident,
  * encountered in the link specifier list. Return NULL if it can't be found.
  *
  * The caller does NOT have ownership of the object, the intro point does. */
-static hs_desc_link_specifier_t *
+static link_specifier_t *
 get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
 {
-  hs_desc_link_specifier_t *lnk_spec = NULL;
+  link_specifier_t *lnk_spec = NULL;
 
   tor_assert(ip);
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          hs_desc_link_specifier_t *, ls) {
-    if (ls->type == type) {
+                          link_specifier_t *, ls) {
+    if (link_specifier_get_ls_type(ls) == type) {
       lnk_spec = ls;
       goto end;
     }
@@ -681,7 +656,7 @@ get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
 STATIC const node_t *
 get_node_from_intro_point(const hs_service_intro_point_t *ip)
 {
-  const hs_desc_link_specifier_t *ls;
+  const link_specifier_t *ls;
 
   tor_assert(ip);
 
@@ -690,7 +665,8 @@ get_node_from_intro_point(const hs_service_intro_point_t *ip)
     return NULL;
   }
   /* XXX In the future, we want to only use the ed25519 ID (#22173). */
-  return node_get_by_id((const char *) ls->u.legacy_id);
+  return node_get_by_id(
+    (const char *) link_specifier_getconstarray_un_legacy_id(ls));
 }
 
 /* Given a service intro point, return the extend_info_t for it. This can
@@ -1179,7 +1155,8 @@ parse_authorized_client(const char *client_key_str)
   client = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
   if (base32_decode((char *) client->client_pk.public_key,
                     sizeof(client->client_pk.public_key),
-                    pubkey_b32, strlen(pubkey_b32)) < 0) {
+                    pubkey_b32, strlen(pubkey_b32)) !=
+      sizeof(client->client_pk.public_key)) {
     log_warn(LD_REND, "Client authorization public key cannot be decoded: %s",
              pubkey_b32);
     goto err;
@@ -1261,7 +1238,7 @@ load_client_keys(hs_service_t *service)
     client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
 
     /* If we cannot read the file, continue with the next file. */
-    if (!client_key_str)  {
+    if (!client_key_str) {
       log_warn(LD_REND, "Client authorization file %s can't be read. "
                         "Corrupted or verify permission? Ignoring.",
                client_key_file_path);
@@ -1556,7 +1533,7 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
                              hs_service_descriptor_t *desc, time_t now)
 {
   time_t *time_of_failure, *prev_ptr;
-  const hs_desc_link_specifier_t *legacy_ls;
+  const link_specifier_t *legacy_ls;
 
   tor_assert(ip);
   tor_assert(desc);
@@ -1565,22 +1542,13 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
   *time_of_failure = now;
   legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
   tor_assert(legacy_ls);
-  prev_ptr = digestmap_set(desc->intro_points.failed_id,
-                           (const char *) legacy_ls->u.legacy_id,
-                           time_of_failure);
+  prev_ptr = digestmap_set(
+    desc->intro_points.failed_id,
+    (const char *) link_specifier_getconstarray_un_legacy_id(legacy_ls),
+    time_of_failure);
   tor_free(prev_ptr);
 }
 
-/* Copy the descriptor link specifier object from src to dst. */
-static void
-link_specifier_copy(hs_desc_link_specifier_t *dst,
-                    const hs_desc_link_specifier_t *src)
-{
-  tor_assert(dst);
-  tor_assert(src);
-  memcpy(dst, src, sizeof(hs_desc_link_specifier_t));
-}
-
 /* Using a given descriptor signing keypair signing_kp, a service intro point
  * object ip and the time now, setup the content of an already allocated
  * descriptor intro desc_ip.
@@ -1615,9 +1583,14 @@ setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
 
   /* Copy link specifier(s). */
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, ls) {
-    hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy));
-    link_specifier_copy(copy, ls);
+                          const link_specifier_t *, ls) {
+    if (BUG(!ls)) {
+      goto done;
+    }
+    link_specifier_t *copy = link_specifier_dup(ls);
+    if (BUG(!copy)) {
+      goto done;
+    }
     smartlist_add(desc_ip->link_specifiers, copy);
   } SMARTLIST_FOREACH_END(ls);
 
@@ -1789,7 +1762,7 @@ build_service_desc_superencrypted(const hs_service_t *service,
          sizeof(curve25519_public_key_t));
 
   /* Test that subcred is not zero because we might use it below */
-  if (BUG(tor_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) {
+  if (BUG(fast_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) {
     return -1;
   }
 
@@ -1855,9 +1828,9 @@ build_service_desc_plaintext(const hs_service_t *service,
 
   tor_assert(service);
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->blinded_kp,
                               sizeof(desc->blinded_kp)));
-  tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->signing_kp,
                               sizeof(desc->signing_kp)));
 
   /* Set the subcredential. */
@@ -1907,7 +1880,7 @@ build_service_desc_keys(const hs_service_t *service,
   ed25519_keypair_t kp;
 
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk,
+  tor_assert(!fast_mem_is_zero((char *) &service->keys.identity_pk,
              ED25519_PUBKEY_LEN));
 
   /* XXX: Support offline key feature (#18098). */
@@ -2116,7 +2089,6 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
 {
   const or_options_t *options = get_options();
   const node_t *node;
-  extend_info_t *info = NULL;
   hs_service_intro_point_t *ip = NULL;
   /* Normal 3-hop introduction point flags. */
   router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC;
@@ -2145,43 +2117,17 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
    * we don't want to use that node anymore. */
   smartlist_add(exclude_nodes, (void *) node);
 
-  /* We do this to ease our life but also this call makes appropriate checks
-   * of the node object such as validating ntor support for instance.
-   *
-   * We must provide an extend_info for clients to connect over a 3-hop path,
-   * so we don't pass direct_conn here. */
-  info = extend_info_from_node(node, 0);
-  if (BUG(info == NULL)) {
-    goto err;
-  }
-
-  /* Let's do a basic sanity check here so that we don't end up advertising the
-   * ed25519 identity key of relays that don't actually support the link
-   * protocol */
-  if (!node_supports_ed25519_link_authentication(node, 0)) {
-    tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity));
-  } else {
-    /* Make sure we *do* have an ed key if we support the link authentication.
-     * Sending an empty key would result in a failure to extend. */
-    tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity));
-  }
+  /* Create our objects and populate them with the node information. */
+  ip = service_intro_point_new(node);
 
-  /* Create our objects and populate them with the node information.
-   * We don't care if the intro's link auth is compatible with us, because
-   * we are sending the ed25519 key to a remote client via the descriptor. */
-  ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node),
-                               node_supports_ed25519_link_authentication(node,
-                                                                         0));
   if (ip == NULL) {
     goto err;
   }
 
-  log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info));
-  extend_info_free(info);
+  log_info(LD_REND, "Picked intro point: %s", node_describe(node));
   return ip;
  err:
   service_intro_point_free(ip);
-  extend_info_free(info);
   return NULL;
 }
 
@@ -2387,15 +2333,70 @@ intro_point_should_expire(const hs_service_intro_point_t *ip,
   return 1;
 }
 
-/* Go over the given set of intro points for each service and remove any
- * invalid ones. The conditions for removal are:
+/* Return true iff we should remove the intro point ip from its service.
  *
- *    - The node doesn't exists anymore (not in consensus)
- *                          OR
- *    - The intro point maximum circuit retry count has been reached and no
- *      circuit can be found associated with it.
- *                          OR
- *    - The intro point has expired and we should pick a new one.
+ * We remove an intro point from the service descriptor list if one of
+ * these criteria is met:
+ *    - It has expired (either in INTRO2 count or in time).
+ *    - No node was found (fell off the consensus).
+ *    - We are over the maximum amount of retries.
+ *
+ * If an established or pending circuit is found for the given ip object, this
+ * return false indicating it should not be removed. */
+static bool
+should_remove_intro_point(hs_service_intro_point_t *ip, time_t now)
+{
+  bool ret = false;
+
+  tor_assert(ip);
+
+  /* Any one of the following needs to be True to furfill the criteria to
+   * remove an intro point. */
+  bool has_no_retries = (ip->circuit_retries >
+                         MAX_INTRO_POINT_CIRCUIT_RETRIES);
+  bool has_no_node = (get_node_from_intro_point(ip) == NULL);
+  bool has_expired = intro_point_should_expire(ip, now);
+
+  /* If the node fell off the consensus or the IP has expired, we have to
+   * remove it now. */
+  if (has_no_node || has_expired) {
+    ret = true;
+    goto end;
+  }
+
+  /* Pass this point, even though we might be over the retry limit, we check
+   * if a circuit (established or pending) exists. In that case, we should not
+   * remove it because it might simply be valid and opened at the previous
+   * scheduled event for the last retry. */
+
+  /* Did we established already? */
+  if (ip->circuit_established) {
+    goto end;
+  }
+  /* Do we simply have an existing circuit regardless of its state? */
+  if (hs_circ_service_get_intro_circ(ip)) {
+    goto end;
+  }
+
+  /* Getting here means we have _no_ circuits so then return if we have any
+   * remaining retries. */
+  ret = has_no_retries;
+
+ end:
+  /* Meaningful log in case we are about to remove the IP. */
+  if (ret) {
+    log_info(LD_REND, "Intro point %s%s (retried: %u times). "
+                      "Removing it.",
+             describe_intro_point(ip),
+             has_expired ? " has expired" :
+               (has_no_node) ?  " fell off the consensus" : "",
+             ip->circuit_retries);
+  }
+  return ret;
+}
+
+/* Go over the given set of intro points for each service and remove any
+ * invalid ones.
  *
  * If an intro point is removed, the circuit (if any) is immediately close.
  * If a circuit can't be found, the intro point is kept if it hasn't reached
@@ -2420,21 +2421,7 @@ cleanup_intro_points(hs_service_t *service, time_t now)
      * valid and remove any of them that aren't. */
     DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
                                 hs_service_intro_point_t *, ip) {
-      const node_t *node = get_node_from_intro_point(ip);
-      int has_expired = intro_point_should_expire(ip, now);
-
-      /* We cleanup an intro point if it has expired or if we do not know the
-       * node_t anymore (removed from our latest consensus) or if we've
-       * reached the maximum number of retry with a non existing circuit. */
-      if (has_expired || node == NULL ||
-          ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
-        log_info(LD_REND, "Intro point %s%s (retried: %u times). "
-                          "Removing it.",
-                 describe_intro_point(ip),
-                 has_expired ? " has expired" :
-                    (node == NULL) ?  " fell off the consensus" : "",
-                 ip->circuit_retries);
-
+      if (should_remove_intro_point(ip, now)) {
         /* We've retried too many times, remember it as a failed intro point
          * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */
         if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -2956,8 +2943,8 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now,
   /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */
   tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR);
 
-  log_info(LD_REND, "Encrypted revision counter %d to %ld",
-           (int) seconds_since_start_of_srv, (long int) rev_counter);
+  log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64,
+           (int) seconds_since_start_of_srv, rev_counter);
 
   hs_desc->desc->plaintext_data.revision_counter = rev_counter;
 }
@@ -3700,8 +3687,8 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk)
 }
 
 /* Return the number of service we have configured and usable. */
-unsigned int
-hs_service_get_num_services(void)
+MOCK_IMPL(unsigned int,
+hs_service_get_num_services,(void))
 {
   if (hs_service_map == NULL) {
     return 0;
diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h
index 5f43233ea1..c4bbb293bb 100644
--- a/src/feature/hs/hs_service.h
+++ b/src/feature/hs/hs_service.h
@@ -76,6 +76,10 @@ typedef struct hs_service_intro_point_t {
    * circuit associated with this intro point has received. This is used to
    * prevent replay attacks. */
   replaycache_t *replay_cache;
+
+  /* Support the INTRO2 DoS defense. If set, the DoS extension described by
+   * proposal 305 is sent. */
+  unsigned int support_intro2_dos_defense : 1;
 } hs_service_intro_point_t;
 
 /* Object handling introduction points of a service. */
@@ -241,6 +245,11 @@ typedef struct hs_service_config_t {
 
   /* Does this service export the circuit ID of its clients? */
   hs_circuit_id_protocol_t circuit_id_protocol;
+
+  /* DoS defenses. For the ESTABLISH_INTRO cell extension. */
+  unsigned int has_dos_defense_enabled : 1;
+  uint32_t intro_dos_rate_per_sec;
+  uint32_t intro_dos_burst_per_sec;
 } hs_service_config_t;
 
 /* Service state. */
@@ -310,7 +319,7 @@ hs_service_t *hs_service_new(const or_options_t *options);
 void hs_service_free_(hs_service_t *service);
 #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s))
 
-unsigned int hs_service_get_num_services(void);
+MOCK_DECL(unsigned int, hs_service_get_num_services,(void));
 void hs_service_stage_services(const smartlist_t *service_list);
 int hs_service_load_all_keys(void);
 int hs_service_get_version_from_key(const hs_service_t *service);
@@ -361,7 +370,7 @@ STATIC hs_service_t *get_first_service(void);
 STATIC hs_service_intro_point_t *service_intro_point_find_by_ident(
                                          const hs_service_t *service,
                                          const hs_ident_circuit_t *ident);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /* Service accessors. */
 STATIC hs_service_t *find_service(hs_service_ht *map,
@@ -369,10 +378,7 @@ STATIC hs_service_t *find_service(hs_service_ht *map,
 STATIC void remove_service(hs_service_ht *map, hs_service_t *service);
 STATIC int register_service(hs_service_ht *map, hs_service_t *service);
 /* Service introduction point functions. */
-STATIC hs_service_intro_point_t *service_intro_point_new(
-                            const extend_info_t *ei,
-                            unsigned int is_legacy,
-                            unsigned int supports_ed25519_link_handshake_any);
+STATIC hs_service_intro_point_t *service_intro_point_new(const node_t *node);
 STATIC void service_intro_point_free_(hs_service_intro_point_t *ip);
 #define service_intro_point_free(ip)                            \
   FREE_AND_NULL(hs_service_intro_point_t,             \
diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h
index d89440faca..6700eca15b 100644
--- a/src/feature/hs/hs_stats.h
+++ b/src/feature/hs/hs_stats.h
@@ -6,9 +6,13 @@
  * \brief Header file for hs_stats.c
  **/
 
+#ifndef TOR_HS_STATS_H
+#define TOR_HS_STATS_H
+
 void hs_stats_note_introduce2_cell(int is_hsv3);
 uint32_t hs_stats_get_n_introduce2_v3_cells(void);
 uint32_t hs_stats_get_n_introduce2_v2_cells(void);
 void hs_stats_note_service_rendezvous_launch(void);
 uint32_t hs_stats_get_n_rendezvous_launches(void);
 
+#endif /* !defined(TOR_HS_STATS_H) */
diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h
index 7d4116d8bb..6c86c02f47 100644
--- a/src/feature/hs/hsdir_index_st.h
+++ b/src/feature/hs/hsdir_index_st.h
@@ -20,5 +20,5 @@ struct hsdir_index_t {
   uint8_t store_second[DIGEST256_LEN];
 };
 
-#endif
+#endif /* !defined(HSDIR_INDEX_ST_H) */
 
diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h
index 95fe2c65ab..c90c52cfea 100644
--- a/src/feature/hs_common/shared_random_client.h
+++ b/src/feature/hs_common/shared_random_client.h
@@ -44,5 +44,5 @@ time_t get_start_time_of_current_round(void);
 
 #endif /* TOR_UNIT_TESTS */
 
-#endif /* TOR_SHARED_RANDOM_CLIENT_H */
+#endif /* !defined(TOR_SHARED_RANDOM_CLIENT_H) */
 
diff --git a/src/feature/keymgt/loadkey.h b/src/feature/keymgt/loadkey.h
index 8beee57a20..0a5af0b804 100644
--- a/src/feature/keymgt/loadkey.h
+++ b/src/feature/keymgt/loadkey.h
@@ -52,4 +52,4 @@ int read_encrypted_secret_key(ed25519_secret_key_t *out,
 int write_encrypted_secret_key(const ed25519_secret_key_t *out,
                                const char *fname);
 
-#endif
+#endif /* !defined(TOR_LOADKEY_H) */
diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c
index 7a065662a7..9fc3b62525 100644
--- a/src/feature/nodelist/authcert.c
+++ b/src/feature/nodelist/authcert.c
@@ -380,7 +380,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
   int added_trusted_cert = 0;
 
   for (s = contents; *s; s = eos) {
-    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s),
+                                                              &eos);
     cert_list_t *cl;
     if (!cert) {
       failure_code = -1;
diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h
index 2effdb06e6..071293f9ee 100644
--- a/src/feature/nodelist/authcert.h
+++ b/src/feature/nodelist/authcert.h
@@ -57,4 +57,4 @@ MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
 
 void authcert_free_all(void);
 
-#endif
+#endif /* !defined(TOR_AUTHCERT_H) */
diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h
index 68a84bc452..bf9b690c24 100644
--- a/src/feature/nodelist/authority_cert_st.h
+++ b/src/feature/nodelist/authority_cert_st.h
@@ -28,5 +28,5 @@ struct authority_cert_t {
   uint16_t dir_port;
 };
 
-#endif
+#endif /* !defined(AUTHORITY_CERT_ST_H) */
 
diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h
index b04a1abc7d..4d1378cdfa 100644
--- a/src/feature/nodelist/desc_store_st.h
+++ b/src/feature/nodelist/desc_store_st.h
@@ -36,4 +36,4 @@ struct desc_store_t {
   size_t bytes_dropped;
 };
 
-#endif
+#endif /* !defined(DESC_STORE_ST_H) */
diff --git a/src/feature/nodelist/describe.c b/src/feature/nodelist/describe.c
index 5c376408c0..1e46837685 100644
--- a/src/feature/nodelist/describe.c
+++ b/src/feature/nodelist/describe.c
@@ -9,66 +9,108 @@
  * \brief Format short descriptions of relays.
  */
 
+#define DESCRIBE_PRIVATE
+
 #include "core/or/or.h"
 #include "feature/nodelist/describe.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "core/or/extend_info_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
-
-/**
- * Longest allowed output of format_node_description, plus 1 character for
- * NUL.  This allows space for:
- * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
- * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
- * plus a terminating NUL.
- */
-#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+#include "feature/nodelist/microdesc_st.h"
 
 /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
  * hold a human-readable description of a node with identity digest
- * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
- * and address <b>addr</b> or <b>addr32h</b>.
+ * <b>id_digest</b>, nickname <b>nickname</b>, and addresses <b>addr32h</b> and
+ * <b>addr</b>.
  *
  * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
- * NULL.  The <b>addr32h</b> field is optional and may be set to 0.
+ * NULL or the null address.  The <b>addr32h</b> field is optional and may be
+ * set to 0.
  *
  * Return a pointer to the front of <b>buf</b>.
+ * If buf is NULL, return a string constant describing the error.
  */
-static const char *
+STATIC const char *
 format_node_description(char *buf,
                         const char *id_digest,
-                        int is_named,
                         const char *nickname,
                         const tor_addr_t *addr,
                         uint32_t addr32h)
 {
-  char *cp;
+  size_t rv = 0;
+  bool has_addr = addr && !tor_addr_is_null(addr);
 
   if (!buf)
     return "<NULL BUFFER>";
 
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
-  cp = buf+1+HEX_DIGEST_LEN;
+  memset(buf, 0, NODE_DESC_BUF_LEN);
+
+  if (!id_digest) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<NULL ID DIGEST>", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    return buf;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", NODE_DESC_BUF_LEN);
+  tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  id_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   if (nickname) {
-    buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
-    strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
-    cp += strlen(cp);
+    rv = strlcat(buf, "~", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    rv = strlcat(buf, nickname, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
-  if (addr32h || addr) {
-    memcpy(cp, " at ", 4);
-    cp += 4;
-    if (addr) {
-      tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
-    } else {
-      struct in_addr in;
-      in.s_addr = htonl(addr32h);
-      tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
-    }
+  if (addr32h || has_addr) {
+    rv = strlcat(buf, " at ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
+  if (addr32h) {
+    int ntoa_rv = 0;
+    char ipv4_addr_str[INET_NTOA_BUF_LEN];
+    memset(ipv4_addr_str, 0, sizeof(ipv4_addr_str));
+    struct in_addr in;
+    memset(&in, 0, sizeof(in));
+
+    in.s_addr = htonl(addr32h);
+    ntoa_rv = tor_inet_ntoa(&in, ipv4_addr_str, sizeof(ipv4_addr_str));
+    tor_assert_nonfatal(ntoa_rv >= 0);
+
+    rv = strlcat(buf, ipv4_addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  /* Both addresses are valid */
+  if (addr32h && has_addr) {
+    rv = strlcat(buf, " and ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  if (has_addr) {
+    const char *str_rv = NULL;
+    char addr_str[TOR_ADDR_BUF_LEN];
+    memset(addr_str, 0, sizeof(addr_str));
+
+    str_rv = tor_addr_to_str(addr_str, addr, sizeof(addr_str), 1);
+    tor_assert_nonfatal(str_rv == addr_str);
+
+    rv = strlcat(buf, addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   return buf;
 }
 
@@ -84,11 +126,11 @@ router_describe(const routerinfo_t *ri)
 
   if (!ri)
     return "<null>";
+
   return format_node_description(buf,
                                  ri->cache_info.identity_digest,
-                                 0,
                                  ri->nickname,
-                                 NULL,
+                                 &ri->ipv6_addr,
                                  ri->addr);
 }
 
@@ -103,25 +145,33 @@ node_describe(const node_t *node)
   static char buf[NODE_DESC_BUF_LEN];
   const char *nickname = NULL;
   uint32_t addr32h = 0;
-  int is_named = 0;
+  const tor_addr_t *ipv6_addr = NULL;
 
   if (!node)
     return "<null>";
 
   if (node->rs) {
     nickname = node->rs->nickname;
-    is_named = node->rs->is_named;
     addr32h = node->rs->addr;
+    ipv6_addr = &node->rs->ipv6_addr;
+    /* Support consensus versions less than 28, when IPv6 addresses were in
+     * microdescs. This code can be removed when 0.2.9 is no longer supported,
+     * and the MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC macro is removed. */
+    if (node->md && tor_addr_is_null(ipv6_addr)) {
+      ipv6_addr = &node->md->ipv6_addr;
+    }
   } else if (node->ri) {
     nickname = node->ri->nickname;
     addr32h = node->ri->addr;
+    ipv6_addr = &node->ri->ipv6_addr;
+  } else {
+    return "<null rs and ri>";
   }
 
   return format_node_description(buf,
                                  node->identity,
-                                 is_named,
                                  nickname,
-                                 NULL,
+                                 ipv6_addr,
                                  addr32h);
 }
 
@@ -137,11 +187,11 @@ routerstatus_describe(const routerstatus_t *rs)
 
   if (!rs)
     return "<null>";
+
   return format_node_description(buf,
                                  rs->identity_digest,
-                                 rs->is_named,
                                  rs->nickname,
-                                 NULL,
+                                 &rs->ipv6_addr,
                                  rs->addr);
 }
 
@@ -157,9 +207,9 @@ extend_info_describe(const extend_info_t *ei)
 
   if (!ei)
     return "<null>";
+
   return format_node_description(buf,
                                  ei->identity_digest,
-                                 0,
                                  ei->nickname,
                                  &ei->addr,
                                  0);
@@ -175,9 +225,39 @@ extend_info_describe(const extend_info_t *ei)
 void
 router_get_verbose_nickname(char *buf, const routerinfo_t *router)
 {
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
-                DIGEST_LEN);
-  buf[1+HEX_DIGEST_LEN] = '~';
-  strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
+  size_t rv = 0;
+
+  if (!buf)
+    return;
+
+  memset(buf, 0, MAX_VERBOSE_NICKNAME_LEN+1);
+
+  if (!router) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<null>", MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+    return;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  router->cache_info.identity_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+  }
+
+  rv = strlcat(buf, "~", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  rv = strlcat(buf, router->nickname, MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
 }
diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h
index 018af6470e..6c0d6dc48d 100644
--- a/src/feature/nodelist/describe.h
+++ b/src/feature/nodelist/describe.h
@@ -22,4 +22,36 @@ const char *node_describe(const struct node_t *node);
 const char *router_describe(const struct routerinfo_t *ri);
 const char *routerstatus_describe(const struct routerstatus_t *ri);
 
-#endif
+void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
+
+#if defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS)
+
+/**
+ * Longest allowed output for an IPv4 address "255.255.255.255", with NO
+ * terminating NUL.
+ */
+#define IPV4_BUF_LEN_NO_NUL 15
+
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL.  This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " 255.255.255.255 and [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN \
+  (MAX_VERBOSE_NICKNAME_LEN+4+IPV4_BUF_LEN_NO_NUL+5+TOR_ADDR_BUF_LEN)
+
+#endif /* defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS) */
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC const char *format_node_description(char *buf,
+                                           const char *id_digest,
+                                           const char *nickname,
+                                           const tor_addr_t *addr,
+                                           uint32_t addr32h);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_DESCRIBE_H) */
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
index 93baa6e4e0..ccbb378513 100644
--- a/src/feature/nodelist/dirlist.c
+++ b/src/feature/nodelist/dirlist.c
@@ -28,7 +28,7 @@
 
 #include "app/config/config.h"
 #include "core/or/policies.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/dirlist.h"
@@ -49,6 +49,37 @@ static smartlist_t *trusted_dir_servers = NULL;
  * and all fallback directory servers. */
 static smartlist_t *fallback_dir_servers = NULL;
 
+/** Helper: From a given trusted directory entry, add the v4 or/and v6 address
+ * to the nodelist address set. */
+static void
+add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir)
+{
+  tor_assert(dir);
+  tor_assert(dir->is_authority);
+
+  /* Add IPv4 and then IPv6 if applicable. */
+  nodelist_add_addr4_to_address_set(dir->addr);
+  if (!tor_addr_is_null(&dir->ipv6_addr)) {
+    nodelist_add_addr6_to_address_set(&dir->ipv6_addr);
+  }
+}
+
+/** Go over the trusted directory server list and add their address(es) to the
+ * nodelist address set. This is called everytime a new consensus is set. */
+MOCK_IMPL(void,
+dirlist_add_trusted_dir_addresses, (void))
+{
+  if (!trusted_dir_servers) {
+    return;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, const dir_server_t *, ent) {
+    if (ent->is_authority) {
+      add_trusted_dir_to_nodelist_addr_set(ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
 /** Return the number of directory authorities whose type matches some bit set
  * in <b>type</b>  */
 int
diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h
index 9fabd0a44a..c49162f1e9 100644
--- a/src/feature/nodelist/dirlist.h
+++ b/src/feature/nodelist/dirlist.h
@@ -44,4 +44,6 @@ void dir_server_add(dir_server_t *ent);
 void clear_dir_servers(void);
 void dirlist_free_all(void);
 
-#endif
+MOCK_DECL(void, dirlist_add_trusted_dir_addresses, (void));
+
+#endif /* !defined(TOR_DIRLIST_H) */
diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h
index 66e32c422f..ac2a803252 100644
--- a/src/feature/nodelist/document_signature_st.h
+++ b/src/feature/nodelist/document_signature_st.h
@@ -25,5 +25,5 @@ struct document_signature_t {
                                      * as good. */
 };
 
-#endif
+#endif /* !defined(DOCUMENT_SIGNATURE_ST_H) */
 
diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h
index c54277b05e..22c708f018 100644
--- a/src/feature/nodelist/extrainfo_st.h
+++ b/src/feature/nodelist/extrainfo_st.h
@@ -26,5 +26,5 @@ struct extrainfo_t {
   size_t pending_sig_len;
 };
 
-#endif
+#endif /* !defined(EXTRAINFO_ST_H) */
 
diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c
index 75cab7a0af..fea7cf4c65 100644
--- a/src/feature/nodelist/fmt_routerstatus.c
+++ b/src/feature/nodelist/fmt_routerstatus.c
@@ -14,55 +14,14 @@
 #include "core/or/or.h"
 #include "feature/nodelist/fmt_routerstatus.h"
 
-/* #include "lib/container/buffers.h" */
-/* #include "app/config/config.h" */
-/* #include "app/config/confparse.h" */
-/* #include "core/or/channel.h" */
-/* #include "core/or/channeltls.h" */
-/* #include "core/or/command.h" */
-/* #include "core/mainloop/connection.h" */
-/* #include "core/or/connection_or.h" */
-/* #include "feature/dircache/conscache.h" */
-/* #include "feature/dircache/consdiffmgr.h" */
-/* #include "feature/control/control.h" */
-/* #include "feature/dircache/directory.h" */
-/* #include "feature/dircache/dirserv.h" */
-/* #include "feature/hibernate/hibernate.h" */
-/* #include "feature/dirauth/keypin.h" */
-/* #include "core/mainloop/mainloop.h" */
-/* #include "feature/nodelist/microdesc.h" */
-/* #include "feature/nodelist/networkstatus.h" */
-/* #include "feature/nodelist/nodelist.h" */
 #include "core/or/policies.h"
-/* #include "core/or/protover.h" */
-/* #include "feature/stats/rephist.h" */
-/* #include "feature/relay/router.h" */
-/* #include "feature/nodelist/dirlist.h" */
 #include "feature/nodelist/routerlist.h"
-
-/* #include "feature/nodelist/routerparse.h" */
-/* #include "feature/nodelist/routerset.h" */
-/* #include "feature/nodelist/torcert.h" */
-/* #include "feature/dircommon/voting_schedule.h" */
-
 #include "feature/dirauth/dirvote.h"
 
-/* #include "feature/dircache/cached_dir_st.h" */
-/* #include "feature/dircommon/dir_connection_st.h" */
-/* #include "feature/nodelist/extrainfo_st.h" */
-/* #include "feature/nodelist/microdesc_st.h" */
-/* #include "feature/nodelist/node_st.h" */
 #include "feature/nodelist/routerinfo_st.h"
-/* #include "feature/nodelist/routerlist_st.h" */
-/* #include "core/or/tor_version_st.h" */
 #include "feature/nodelist/vote_routerstatus_st.h"
 
-/* #include "lib/compress/compress.h" */
-/* #include "lib/container/order.h" */
 #include "lib/crypt_ops/crypto_format.h"
-/* #include "lib/encoding/confline.h" */
-
-/* #include "lib/encoding/keyval.h" */
 
 /** Helper: write the router-status information in <b>rs</b> into a newly
  * allocated character buffer.  Use the same format as in network-status
@@ -135,7 +94,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     goto done;
 
   smartlist_add_asprintf(chunks,
-                   "s%s%s%s%s%s%s%s%s%s%s\n",
+                   "s%s%s%s%s%s%s%s%s%s%s%s\n",
                   /* These must stay in alphabetical order. */
                    rs->is_authority?" Authority":"",
                    rs->is_bad_exit?" BadExit":"",
@@ -145,6 +104,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                    rs->is_hs_dir?" HSDir":"",
                    rs->is_flagged_running?" Running":"",
                    rs->is_stable?" Stable":"",
+                   rs->is_staledesc?" StaleDesc":"",
                    rs->is_v2_dir?" V2Dir":"",
                    rs->is_valid?" Valid":"");
 
@@ -232,7 +192,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     }
 
     if (format == NS_V3_VOTE && vrs) {
-      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
+      if (fast_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
         smartlist_add_strdup(chunks, "id ed25519 none\n");
       } else {
         char ed_b64[BASE64_DIGEST256_LEN+1];
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
index dafaabb5e5..89ac0a2f83 100644
--- a/src/feature/nodelist/microdesc.c
+++ b/src/feature/nodelist/microdesc.c
@@ -23,6 +23,7 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -69,6 +70,8 @@ struct microdesc_cache_t {
 };
 
 static microdesc_cache_t *get_microdesc_cache_noload(void);
+static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                              const char *activity);
 
 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
 static inline unsigned int
@@ -110,8 +113,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest)
 
   /* If we have a reasonably live consensus, then most of our dirservers should
    * still be caching all the microdescriptors in it. Reasonably live
-   * consensuses are up to a day old. But microdescriptors expire 7 days after
-   * the last consensus that referenced them. */
+   * consensuses are up to a day old (or a day in the future). But
+   * microdescriptors expire 7 days after the last consensus that referenced
+   * them. */
   if (!networkstatus_get_reasonably_live_consensus(approx_time(),
                                                    FLAV_MICRODESC)) {
     return;
@@ -221,6 +225,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
   }
 
   md->off = tor_fd_getpos(fd);
+  warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
+                    "dumping a microdescriptor");
   written = write_all_to_fd(fd, md->body, md->bodylen);
   if (written != (ssize_t)md->bodylen) {
     written = written < 0 ? 0 : written;
@@ -480,6 +486,27 @@ microdesc_cache_clear(microdesc_cache_t *cache)
   cache->bytes_dropped = 0;
 }
 
+static void
+warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                  const char *activity)
+{
+  const char *nul_found = memchr(inp, 0, len);
+  if (BUG(nul_found)) {
+    log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
+             "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
+             activity, offset, (nul_found - inp), len);
+    const char *start_excerpt_at, *eos = inp + len;
+    if ((nul_found - inp) >= 16)
+      start_excerpt_at = nul_found - 16;
+    else
+      start_excerpt_at = inp;
+    size_t excerpt_len = MIN(32, eos - start_excerpt_at);
+    char tmp[65];
+    base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
+    log_warn(LD_BUG, "      surrounding string: %s", tmp);
+  }
+}
+
 /** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
  * for the first time.  Return 0 on success, -1 on failure. */
 int
@@ -497,6 +524,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
 
   mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
   if (mm) {
+    warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
     added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
                                     SAVED_IN_CACHE, 0, -1, NULL);
     if (added) {
@@ -508,7 +536,9 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   journal_content = read_file_to_str(cache->journal_fname,
                                      RFTS_IGNORE_MISSING, &st);
   if (journal_content) {
-    cache->journal_len = (size_t) st.st_size;
+    cache->journal_len = strlen(journal_content);
+    warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
+                      "reading microdesc journal");
     added = microdescs_add_to_cache(cache, journal_content,
                                     journal_content+st.st_size,
                                     SAVED_IN_JOURNAL, 0, -1, NULL);
@@ -544,8 +574,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
   size_t bytes_dropped = 0;
   time_t now = time(NULL);
 
-  /* If we don't know a live consensus, don't believe last_listed values: we
-   * might be starting up after being down for a while. */
+  /* If we don't know a reasonably live consensus, don't believe last_listed
+   * values: we might be starting up after being down for a while. */
   if (! force &&
       ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
       return;
@@ -884,10 +914,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno)
   if (md->body && md->saved_location != SAVED_IN_CACHE)
     tor_free(md->body);
 
-  if (md->family) {
-    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
-    smartlist_free(md->family);
-  }
+  nodefamily_free(md->family);
   short_policy_free(md->exit_policy);
   short_policy_free(md->ipv6_exit_policy);
 
@@ -943,7 +970,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
       continue;
     if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
       continue;
-    if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
+    if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
       continue;
     /* XXXX Also skip if we're a noncache and wouldn't use this router.
      * XXXX NM Microdesc
@@ -973,6 +1000,7 @@ update_microdesc_downloads(time_t now)
   if (directory_too_idle_to_fetch_descriptors(options, now))
     return;
 
+  /* Give up if we don't have a reasonably live consensus. */
   consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
   if (!consensus)
     return;
diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h
index bb8b23d664..e017c46c79 100644
--- a/src/feature/nodelist/microdesc_st.h
+++ b/src/feature/nodelist/microdesc_st.h
@@ -9,6 +9,7 @@
 
 struct curve25519_public_key_t;
 struct ed25519_public_key_t;
+struct nodefamily_t;
 struct short_policy_t;
 
 /** A microdescriptor is the smallest amount of information needed to build a
@@ -32,6 +33,8 @@ struct microdesc_t {
   unsigned int no_save : 1;
   /** If true, this microdesc has an entry in the microdesc_map */
   unsigned int held_in_map : 1;
+  /** True iff the exit policy for this router rejects everything. */
+  unsigned int policy_is_reject_star : 1;
   /** Reference count: how many node_ts have a reference to this microdesc? */
   unsigned int held_by_nodes;
 
@@ -69,12 +72,12 @@ struct microdesc_t {
   tor_addr_t ipv6_addr;
   /** As routerinfo_t.ipv6_orport */
   uint16_t ipv6_orport;
-  /** As routerinfo_t.family */
-  smartlist_t *family;
+  /** As routerinfo_t.family, with readable members parsed. */
+  struct nodefamily_t *family;
   /** IPv4 exit policy summary */
   struct short_policy_t *exit_policy;
   /** IPv6 exit policy summary */
   struct short_policy_t *ipv6_exit_policy;
 };
 
-#endif
+#endif /* !defined(MICRODESC_ST_H) */
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
index c74acd8b74..496bafb865 100644
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -44,6 +44,7 @@
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitstats.h"
@@ -57,7 +58,7 @@
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
 #include "feature/dircache/dirserv.h"
@@ -67,6 +68,7 @@
 #include "feature/dircommon/voting_schedule.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/fmt_routerstatus.h"
@@ -81,6 +83,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
+#include "feature/dirauth/dirauth_periodic.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
@@ -116,8 +119,6 @@ STATIC networkstatus_t *current_md_consensus = NULL;
 typedef struct consensus_waiting_for_certs_t {
   /** The consensus itself. */
   networkstatus_t *consensus;
-  /** The encoded version of the consensus, nul-terminated. */
-  char *body;
   /** When did we set the current value of consensus_waiting_for_certs?  If
    * this is too recent, we shouldn't try to fetch a new consensus for a
    * little while, to give ourselves time to get certificates for this one. */
@@ -179,6 +180,10 @@ static void update_consensus_bootstrap_multiple_downloads(
 static int networkstatus_check_required_protocols(const networkstatus_t *ns,
                                                   int client_mode,
                                                   char **warning_out);
+static int reload_consensus_from_file(const char *fname,
+                                      const char *flavor,
+                                      unsigned flags,
+                                      const char *source_dir);
 
 /** Forget that we've warned about anything networkstatus-related, so we will
  * give fresh warnings if the same behavior happens again. */
@@ -210,14 +215,11 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
-/**
- * Read and and return the cached consensus of type <b>flavorname</b>.  If
- * <b>unverified</b> is true, get the one we haven't verified. Return NULL if
- * the file isn't there. */
+/** Return the filename used to cache the consensus of a given flavor */
 static char *
-networkstatus_read_cached_consensus_impl(int flav,
-                                         const char *flavorname,
-                                         int unverified_consensus)
+networkstatus_get_cache_fname(int flav,
+                              const char *flavorname,
+                              int unverified_consensus)
 {
   char buf[128];
   const char *prefix;
@@ -232,21 +234,35 @@ networkstatus_read_cached_consensus_impl(int flav,
     tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
   }
 
-  char *filename = get_cachedir_fname(buf);
-  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  return get_cachedir_fname(buf);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static tor_mmap_t *
+networkstatus_map_cached_consensus_impl(int flav,
+                                        const char *flavorname,
+                                        int unverified_consensus)
+{
+  char *filename = networkstatus_get_cache_fname(flav,
+                                                 flavorname,
+                                                 unverified_consensus);
+  tor_mmap_t *result = tor_mmap_file(filename);
   tor_free(filename);
   return result;
 }
 
-/** Return a new string containing the current cached consensus of flavor
- * <b>flavorname</b>. */
-char *
-networkstatus_read_cached_consensus(const char *flavorname)
- {
+/** Map the file containing the current cached consensus of flavor
+ * <b>flavorname</b> */
+tor_mmap_t *
+networkstatus_map_cached_consensus(const char *flavorname)
+{
   int flav = networkstatus_parse_flavor_name(flavorname);
   if (flav < 0)
     return NULL;
-  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+  return networkstatus_map_cached_consensus_impl(flav, flavorname, 0);
 }
 
 /** Read every cached v3 consensus networkstatus from the disk. */
@@ -259,25 +275,15 @@ router_reload_consensus_networkstatus(void)
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
     const char *flavor = networkstatus_get_flavor_name(flav);
-    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
-        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
-                 flavor);
-      }
-      tor_free(s);
-    }
+    char *fname = networkstatus_get_cache_fname(flav, flavor, 0);
+    reload_consensus_from_file(fname, flavor, flags, NULL);
+    tor_free(fname);
 
-    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor,
-                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
-                                     NULL)) {
-        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
-                 "from cache", flavor);
-      }
-      tor_free(s);
-    }
+    fname = networkstatus_get_cache_fname(flav, flavor, 1);
+    reload_consensus_from_file(fname, flavor,
+                               flags | NSSET_WAS_WAITING_FOR_CERTS,
+                               NULL);
+    tor_free(fname);
   }
 
   update_certificate_downloads(time(NULL));
@@ -713,8 +719,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest)
 
 /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or
  * NULL if none was found. */
-const routerstatus_t *
-networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+MOCK_IMPL(const routerstatus_t *,
+networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest))
 {
   return networkstatus_vote_find_mutable_entry(ns, digest);
 }
@@ -1377,7 +1383,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor))
 }
 
 /** Return the most recent consensus that we have downloaded, or NULL if we
- * don't have one. */
+ * don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus,(void))
 {
@@ -1388,7 +1394,7 @@ networkstatus_get_latest_consensus,(void))
 }
 
 /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL
- * if we don't have one. */
+ * if we don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
 {
@@ -1422,10 +1428,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now)
   return (ns->valid_after <= now && now <= ns->valid_until);
 }
 
-/** Determine if <b>consensus</b> is valid or expired recently enough that
- * we can still use it.
+/** Determine if <b>consensus</b> is valid, or expired recently enough, or not
+ * too far in the future, so that we can still use it.
  *
- * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old or
+ * too new.
  */
 int
 networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
@@ -1434,29 +1441,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
   if (BUG(!consensus))
     return 0;
 
-  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+  return networkstatus_valid_after_is_reasonably_live(consensus->valid_after,
+                                                      now) &&
+         networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
                                                       now);
 }
 
+#define REASONABLY_LIVE_TIME (24*60*60)
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_after
+ * time, and checks to see if it is in the past, or not too far in the future.
+ */
+int
+networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                             time_t now)
+{
+  return (now >= valid_after - REASONABLY_LIVE_TIME);
+}
+
 /** As networkstatus_consensus_reasonably_live, but takes a valid_until
- * time rather than an entire consensus. */
+ * time, and checks to see if it is in the future, or not too far in the past.
+ */
 int
 networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                              time_t now)
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   return (now <= valid_until + REASONABLY_LIVE_TIME);
 }
 
 /** As networkstatus_get_live_consensus(), but is way more tolerant of expired
- * consensuses. */
+ *  and future consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_reasonably_live_consensus,(time_t now, int flavor))
 {
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
-      consensus->valid_after <= now &&
       networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
@@ -1655,6 +1675,7 @@ notify_before_networkstatus_changes(const networkstatus_t *old_c,
   notify_control_networkstatus_changed(old_c, new_c);
   dos_consensus_has_changed(new_c);
   relay_consensus_has_changed(new_c);
+  hs_dos_consensus_has_changed(new_c);
 }
 
 /* Called after a new consensus has been put in the global state. It is safe
@@ -1725,6 +1746,44 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
 #endif /* defined(TOR_UNIT_TESTS) */
 
 /**
+ * Helper: Read the current consensus of type <b>flavor</b> from
+ * <b>fname</b>.  Flags and return values are as for
+ * networkstatus_set_current_consensus().
+ **/
+static int
+reload_consensus_from_file(const char *fname,
+                           const char *flavor,
+                           unsigned flags,
+                           const char *source_dir)
+{
+  tor_mmap_t *map = tor_mmap_file(fname);
+  if (!map)
+    return 0;
+
+  int rv = networkstatus_set_current_consensus(map->data, map->size,
+                                               flavor, flags, source_dir);
+#ifdef _WIN32
+  if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) {
+    log_notice(LD_GENERAL, "Looks like the above failures are probably "
+               "because of a CRLF in consensus file %s; falling back to "
+               "read_file_to_string. Nothing to worry about: this file "
+               "was probably saved by an earlier version of Tor.",
+               escaped(fname));
+    char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+    rv = networkstatus_set_current_consensus(content, strlen(content),
+                                             flavor, flags, source_dir);
+    tor_free(content);
+  }
+#endif /* defined(_WIN32) */
+  if (rv < -1) {
+    log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s",
+             escaped(fname));
+  }
+  tor_munmap_file(map);
+  return rv;
+}
+
+/**
  * Helper for handle_missing_protocol_warning: handles either the
  * client case (if <b>is_client</b> is set) or the server case otherwise.
  */
@@ -1841,6 +1900,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor,
  */
 int
 networkstatus_set_current_consensus(const char *consensus,
+                                    size_t consensus_len,
                                     const char *flavor,
                                     unsigned flags,
                                     const char *source_dir)
@@ -1869,7 +1929,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   /* Make sure it's parseable. */
-  c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS);
+  c = networkstatus_parse_vote_from_string(consensus,
+                                           consensus_len,
+                                           NULL, NS_TYPE_CONSENSUS);
   if (!c) {
     log_warn(LD_DIR, "Unable to parse networkstatus consensus");
     result = -2;
@@ -1957,14 +2019,12 @@ networkstatus_set_current_consensus(const char *consensus,
           c->valid_after > current_valid_after) {
         waiting = &consensus_waiting_for_certs[flav];
         networkstatus_vote_free(waiting->consensus);
-        tor_free(waiting->body);
         waiting->consensus = c;
         free_consensus = 0;
-        waiting->body = tor_strdup(consensus);
         waiting->set_at = now;
         waiting->dl_failed = 0;
         if (!from_cache) {
-          write_str_to_file(unverified_fname, consensus, 0);
+          write_bytes_to_file(unverified_fname, consensus, consensus_len, 1);
         }
         if (dl_certs)
           authority_certs_fetch_missing(c, now, source_dir);
@@ -1976,9 +2036,9 @@ networkstatus_set_current_consensus(const char *consensus,
          * latest consensus. */
         if (was_waiting_for_certs && from_cache)
           if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
           }
       }
       goto done;
@@ -1991,9 +2051,9 @@ networkstatus_set_current_consensus(const char *consensus,
       }
       if (was_waiting_for_certs && (r < -1) && from_cache) {
         if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
         }
       }
       goto done;
@@ -2055,16 +2115,12 @@ networkstatus_set_current_consensus(const char *consensus,
       waiting->consensus->valid_after <= c->valid_after) {
     networkstatus_vote_free(waiting->consensus);
     waiting->consensus = NULL;
-    if (consensus != waiting->body)
-      tor_free(waiting->body);
-    else
-      waiting->body = NULL;
     waiting->set_at = 0;
     waiting->dl_failed = 0;
     if (unlink(unverified_fname) != 0) {
-      log_warn(LD_FS,
-               "Failed to unlink %s: %s",
-               unverified_fname, strerror(errno));
+      log_debug(LD_FS,
+                "Failed to unlink %s: %s",
+                unverified_fname, strerror(errno));
     }
   }
 
@@ -2082,7 +2138,6 @@ networkstatus_set_current_consensus(const char *consensus,
 
     nodelist_set_consensus(c);
 
-    /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
     update_consensus_networkstatus_fetch_time(now);
 
     /* Change the cell EWMA settings */
@@ -2095,6 +2150,7 @@ networkstatus_set_current_consensus(const char *consensus,
     circuit_build_times_new_consensus_params(
                                get_circuit_build_times_mutable(), c);
     channelpadding_new_consensus_params(c);
+    circpad_new_consensus_params(c);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
@@ -2108,17 +2164,18 @@ networkstatus_set_current_consensus(const char *consensus,
   if (we_want_to_fetch_flavor(options, flav)) {
     if (dir_server_mode(get_options())) {
       dirserv_set_cached_consensus_networkstatus(consensus,
+                                                 consensus_len,
                                                  flavor,
                                                  &c->digests,
                                                  c->digest_sha3_as_signed,
                                                  c->valid_after);
 
-      consdiffmgr_add_consensus(consensus, c);
+      consdiffmgr_add_consensus(consensus, consensus_len, c);
     }
   }
 
   if (!from_cache) {
-    write_str_to_file(consensus_fname, consensus, 0);
+    write_bytes_to_file(consensus_fname, consensus, consensus_len, 1);
   }
 
   warn_early_consensus(c, flavor, now);
@@ -2154,14 +2211,10 @@ networkstatus_note_certs_arrived(const char *source_dir)
     if (!waiting->consensus)
       continue;
     if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
-      char *waiting_body = waiting->body;
-      if (!networkstatus_set_current_consensus(
-                                 waiting_body,
-                                 flavor_name,
-                                 NSSET_WAS_WAITING_FOR_CERTS,
-                                 source_dir)) {
-        tor_free(waiting_body);
-      }
+      char *fname = networkstatus_get_cache_fname(i, flavor_name, 1);
+      reload_consensus_from_file(fname, flavor_name,
+                                 NSSET_WAS_WAITING_FOR_CERTS, source_dir);
+      tor_free(fname);
     }
   }
 }
@@ -2315,6 +2368,49 @@ networkstatus_getinfo_helper_single(const routerstatus_t *rs)
                                    NULL);
 }
 
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
+ *
+ * We assume that node-\>is_running has already been set, e.g. by
+ *   dirserv_set_router_is_running(ri, now);
+ */
+void
+set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                 const node_t *node,
+                                 const routerinfo_t *ri)
+{
+  memset(rs, 0, sizeof(routerstatus_t));
+
+  rs->is_authority =
+    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
+
+  /* Set by compute_performance_thresholds or from consensus */
+  rs->is_exit = node->is_exit;
+  rs->is_stable = node->is_stable;
+  rs->is_fast = node->is_fast;
+  rs->is_flagged_running = node->is_running;
+  rs->is_valid = node->is_valid;
+  rs->is_possible_guard = node->is_possible_guard;
+  rs->is_bad_exit = node->is_bad_exit;
+  rs->is_hs_dir = node->is_hs_dir;
+  rs->is_named = rs->is_unnamed = 0;
+
+  rs->published_on = ri->cache_info.published_on;
+  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
+  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  rs->addr = ri->addr;
+  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
+  rs->or_port = ri->or_port;
+  rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
+
+  tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
+  rs->ipv6_orport = ri->ipv6_orport;
+}
+
 /** Alloc and return a string describing routerstatuses for the most
  * recent info of each router we know about that is of purpose
  * <b>purpose_string</b>. Return NULL if unrecognized purpose.
@@ -2331,7 +2427,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   smartlist_t *statuses;
   const uint8_t purpose = router_purpose_from_string(purpose_string);
   routerstatus_t rs;
-  const int bridge_auth = authdir_mode_bridge(get_options());
 
   if (purpose == ROUTER_PURPOSE_UNKNOWN) {
     log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
@@ -2348,11 +2443,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
       continue;
     if (ri->purpose != purpose)
       continue;
-    /* TODO: modifying the running flag in a getinfo is a bad idea */
-    if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
-      dirserv_set_router_is_running(ri, now);
-    /* then generate and write out status lines for each of them */
-    set_routerstatus_from_routerinfo(&rs, node, ri, now, 0);
+    set_routerstatus_from_routerinfo(&rs, node, ri);
     smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs));
   } SMARTLIST_FOREACH_END(ri);
 
@@ -2362,41 +2453,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   return answer;
 }
 
-/** Write out router status entries for all our bridge descriptors. */
-void
-networkstatus_dump_bridge_status_to_file(time_t now)
-{
-  char *status = networkstatus_getinfo_by_purpose("bridge", now);
-  char *fname = NULL;
-  char *thresholds = NULL;
-  char *published_thresholds_and_status = NULL;
-  char published[ISO_TIME_LEN+1];
-  const routerinfo_t *me = router_get_my_routerinfo();
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *fingerprint_line = NULL;
-
-  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
-                                      fingerprint, 0) >= 0) {
-    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
-  } else {
-    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
-  }
-  format_iso_time(published, now);
-  dirserv_compute_bridge_flag_thresholds();
-  thresholds = dirserv_get_flag_thresholds_line();
-  tor_asprintf(&published_thresholds_and_status,
-               "published %s\nflag-thresholds %s\n%s%s",
-               published, thresholds, fingerprint_line ? fingerprint_line : "",
-               status);
-  fname = get_datadir_fname("networkstatus-bridges");
-  write_str_to_file(fname,published_thresholds_and_status,0);
-  tor_free(thresholds);
-  tor_free(published_thresholds_and_status);
-  tor_free(fname);
-  tor_free(status);
-  tor_free(fingerprint_line);
-}
-
 /* DOCDOC get_net_param_from_list */
 static int32_t
 get_net_param_from_list(smartlist_t *net_params, const char *param_name,
@@ -2668,6 +2724,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
   const char *required, *recommended;
   char *missing = NULL;
 
+  const bool consensus_postdates_this_release =
+    ns->valid_after >= tor_get_approx_release_date();
+
   tor_assert(warning_out);
 
   if (client_mode) {
@@ -2685,7 +2744,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
                  "%s on the Tor network. The missing protocols are: %s",
                  func, missing);
     tor_free(missing);
-    return 1;
+    return consensus_postdates_this_release ? 1 : 0;
   }
 
   if (! protover_all_supported(recommended, &missing)) {
@@ -2718,6 +2777,5 @@ networkstatus_free_all(void)
       networkstatus_vote_free(waiting->consensus);
       waiting->consensus = NULL;
     }
-    tor_free(waiting->body);
   }
 }
diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h
index 9e7b0f1bb0..600fd7fbd5 100644
--- a/src/feature/nodelist/networkstatus.h
+++ b/src/feature/nodelist/networkstatus.h
@@ -16,7 +16,7 @@
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
-char *networkstatus_read_cached_consensus(const char *flavorname);
+tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
 void routerstatus_free_(routerstatus_t *rs);
 #define routerstatus_free(rs) \
@@ -40,8 +40,9 @@ int compare_digest_to_routerstatus_entry(const void *_key,
                                          const void **_member);
 int compare_digest_to_vote_routerstatus_entry(const void *_key,
                                               const void **_member);
-const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns,
-                                              const char *digest);
+MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,(
+                                              networkstatus_t *ns,
+                                              const char *digest));
 routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns,
                                               const char *digest);
 int networkstatus_vote_find_entry_idx(networkstatus_t *ns,
@@ -87,6 +88,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
 int networkstatus_is_live(const networkstatus_t *ns, time_t now);
 int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
                                             time_t now);
+int networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                                 time_t now);
 int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                                  time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus,
@@ -106,6 +109,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options);
 #define NSSET_ACCEPT_OBSOLETE 8
 #define NSSET_REQUIRE_FLAVOR 16
 int networkstatus_set_current_consensus(const char *consensus,
+                                        size_t consensus_len,
                                         const char *flavor,
                                         unsigned flags,
                                         const char *source_dir);
@@ -118,7 +122,6 @@ void signed_descs_update_status_from_consensus_networkstatus(
 
 char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
 char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
-void networkstatus_dump_bridge_status_to_file(time_t now);
 MOCK_DECL(int32_t, networkstatus_get_param,
           (const networkstatus_t *ns, const char *param_name,
            int32_t default_val, int32_t min_val, int32_t max_val));
@@ -145,6 +148,10 @@ void vote_routerstatus_free_(vote_routerstatus_t *rs);
 #define vote_routerstatus_free(rs) \
   FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
 
+void set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                      const node_t *node,
+                                      const routerinfo_t *ri);
+
 #ifdef NETWORKSTATUS_PRIVATE
 #ifdef TOR_UNIT_TESTS
 STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
@@ -157,4 +164,3 @@ extern networkstatus_t *current_md_consensus;
 #endif /* defined(NETWORKSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_NETWORKSTATUS_H) */
-
diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h
index 677d8ed811..420c3d61e4 100644
--- a/src/feature/nodelist/networkstatus_sr_info_st.h
+++ b/src/feature/nodelist/networkstatus_sr_info_st.h
@@ -19,5 +19,5 @@ struct networkstatus_sr_info_t {
   smartlist_t *commits;
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_SR_INFO_ST_H) */
 
diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h
index 6160f12361..6e84c170d6 100644
--- a/src/feature/nodelist/networkstatus_st.h
+++ b/src/feature/nodelist/networkstatus_st.h
@@ -99,6 +99,9 @@ struct networkstatus_t {
 
   /** List of key=value strings from the headers of the bandwidth list file */
   smartlist_t *bw_file_headers;
+
+  /** A SHA256 digest of the bandwidth file used in a vote. */
+  uint8_t bw_file_digest256[DIGEST256_LEN];
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_ST_H) */
diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h
index 4037fcdeca..66af82a8e3 100644
--- a/src/feature/nodelist/networkstatus_voter_info_st.h
+++ b/src/feature/nodelist/networkstatus_voter_info_st.h
@@ -27,4 +27,4 @@ struct networkstatus_voter_info_t {
   smartlist_t *sigs;
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_VOTER_INFO_ST_H) */
diff --git a/src/feature/nodelist/nickname.h b/src/feature/nodelist/nickname.h
index 9bdc6b50e8..78db2a5f91 100644
--- a/src/feature/nodelist/nickname.h
+++ b/src/feature/nodelist/nickname.h
@@ -16,4 +16,4 @@ int is_legal_nickname(const char *s);
 int is_legal_nickname_or_hexdigest(const char *s);
 int is_legal_hexdigest(const char *s);
 
-#endif
+#endif /* !defined(TOR_NICKNAME_H) */
diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c
index e31abb247f..719b4b1b27 100644
--- a/src/feature/nodelist/node_select.c
+++ b/src/feature/nodelist/node_select.c
@@ -30,6 +30,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
+#include "lib/container/bitarray.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/math/fp.h"
 
@@ -585,6 +586,7 @@ compute_weighted_bandwidths(const smartlist_t *sl,
   }
 
   weight_scale = networkstatus_get_weight_scale_param(NULL);
+  tor_assert(weight_scale >= 1);
 
   if (rule == WEIGHT_FOR_GUARD) {
     Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
@@ -825,6 +827,58 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
   nodelist_add_node_and_family(sl, node);
 }
 
+/**
+ * Remove every node_t that appears in <b>excluded</b> from <b>sl</b>.
+ *
+ * Behaves like smartlist_subtract, but uses nodelist_idx values to deliver
+ * linear performance when smartlist_subtract would be quadratic.
+ **/
+static void
+nodelist_subtract(smartlist_t *sl, const smartlist_t *excluded)
+{
+  const smartlist_t *nodelist = nodelist_get_list();
+  const int nodelist_len = smartlist_len(nodelist);
+  bitarray_t *excluded_idx = bitarray_init_zero(nodelist_len);
+
+  /* We haven't used nodelist_idx in this way previously, so I'm going to be
+   * paranoid in this code, and check that nodelist_idx is correct for every
+   * node before we use it.  If we fail, we fall back to smartlist_subtract().
+   */
+
+  /* Set the excluded_idx bit corresponding to every excluded node...
+   */
+  SMARTLIST_FOREACH_BEGIN(excluded, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    bitarray_set(excluded_idx, idx);
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Then remove them from sl.
+   */
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    if (bitarray_is_set(excluded_idx, idx)) {
+      SMARTLIST_DEL_CURRENT(sl, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  bitarray_free(excluded_idx);
+  return;
+
+ internal_error:
+  log_warn(LD_BUG, "Internal error prevented us from using the fast method "
+           "for subtracting nodelists. Falling back to the quadratic way.");
+  smartlist_subtract(sl, excluded);
+  bitarray_free(excluded_idx);
+}
+
 /** Return a random running node from the nodelist. Never
  * pick a node that is in
  * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
@@ -859,6 +913,7 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
   const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
 
+  const smartlist_t *node_list = nodelist_get_list();
   smartlist_t *sl=smartlist_new(),
     *excludednodes=smartlist_new();
   const node_t *choice = NULL;
@@ -869,17 +924,17 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   rule = weight_for_exit ? WEIGHT_FOR_EXIT :
     (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
 
-  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+  SMARTLIST_FOREACH_BEGIN(node_list, const node_t *, node) {
     if (node_allows_single_hop_exits(node)) {
       /* Exclude relays that allow single hop exit circuits. This is an
        * obsolete option since 0.2.9.2-alpha and done by default in
        * 0.3.1.0-alpha. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     } else if (rendezvous_v3 &&
                !node_supports_v3_rendezvous_point(node)) {
       /* Exclude relays that do not support to rendezvous for a hidden service
        * version 3. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     }
   } SMARTLIST_FOREACH_END(node);
 
@@ -896,19 +951,11 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
            "We found %d running nodes.",
             smartlist_len(sl));
 
-  smartlist_subtract(sl,excludednodes);
-  log_debug(LD_CIRC,
-            "We removed %d excludednodes, leaving %d nodes.",
-            smartlist_len(excludednodes),
-            smartlist_len(sl));
-
   if (excludedsmartlist) {
-    smartlist_subtract(sl,excludedsmartlist);
-    log_debug(LD_CIRC,
-              "We removed %d excludedsmartlist, leaving %d nodes.",
-              smartlist_len(excludedsmartlist),
-              smartlist_len(sl));
+    smartlist_add_all(excludednodes, excludedsmartlist);
   }
+  nodelist_subtract(sl, excludednodes);
+
   if (excludedset) {
     routerset_subtract_nodes(sl,excludedset);
     log_debug(LD_CIRC,
diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h
index ed7450b92c..d8b4aca5c1 100644
--- a/src/feature/nodelist/node_select.h
+++ b/src/feature/nodelist/node_select.h
@@ -97,6 +97,6 @@ STATIC const routerstatus_t *router_pick_directory_server_impl(
                                            int *n_busy_out);
 STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
                                           int serverdesc, int microdesc);
-#endif
+#endif /* defined(NODE_SELECT_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NODE_SELECT_H) */
diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h
index 53ffde29e4..c63a535a19 100644
--- a/src/feature/nodelist/node_st.h
+++ b/src/feature/nodelist/node_st.h
@@ -99,4 +99,4 @@ struct node_t {
   struct hsdir_index_t hsdir_index;
 };
 
-#endif
+#endif /* !defined(NODE_ST_H) */
diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c
new file mode 100644
index 0000000000..2ec9d5fa40
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.c
@@ -0,0 +1,416 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.c
+ * \brief Code to manipulate encoded, reference-counted node families.  We
+ *  use these tricks to save space, since these families would otherwise
+ *  require a large number of tiny allocations.
+ **/
+
+#include "core/or/or.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
+#include "feature/nodelist/nodefamily_st.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/routerlist.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * Allocate and return a blank node family with space to hold <b>n_members</b>
+ * members.
+ */
+static nodefamily_t *
+nodefamily_alloc(int n_members)
+{
+  size_t alloc_len = offsetof(nodefamily_t, family_members) +
+    NODEFAMILY_ARRAY_SIZE(n_members);
+  nodefamily_t *nf = tor_malloc_zero(alloc_len);
+  nf->n_members = n_members;
+  return nf;
+}
+
+/**
+ * Hashtable hash implementation.
+ */
+static inline unsigned int
+nodefamily_hash(const nodefamily_t *nf)
+{
+  return (unsigned) siphash24g(nf->family_members,
+                               NODEFAMILY_ARRAY_SIZE(nf->n_members));
+}
+
+/**
+ * Hashtable equality implementation.
+ */
+static inline unsigned int
+nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b)
+{
+  return (a->n_members == b->n_members) &&
+    fast_memeq(a->family_members, b->family_members,
+               NODEFAMILY_ARRAY_SIZE(a->n_members));
+}
+
+static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families
+  = HT_INITIALIZER();
+
+HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             nodefamily_eq)
+HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             node_family_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/**
+ * Parse the family declaration in <b>s</b>, returning the canonical
+ * <b>nodefamily_t</b> for its members.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * If NF_WARN_MALFORMED is set in <b>flags</b>, warn about any
+ * elements that we can't parse.  (By default, we log at info.)
+ *
+ * If NF_REJECT_MALFORMED is set in <b>flags</b>, treat any unparseable
+ * elements as an error. (By default, we simply omit them.)
+ **/
+nodefamily_t *
+nodefamily_parse(const char *s, const uint8_t *rsa_id_self,
+                 unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Canonicalize the family list <b>s</b>, returning a newly allocated string.
+ *
+ * The canonicalization rules are fully specified in dir-spec.txt, but,
+ * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are
+ * truncated, nicknames are put into lowercase, unrecognized entries are left
+ * alone, and everything is sorted.
+ **/
+char *
+nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                        unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_t *result_members = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags,
+                                             result_members);
+
+  char *formatted = nodefamily_format(nf);
+  smartlist_split_string(result_members, formatted, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  smartlist_sort_strings(result_members);
+  char *combined = smartlist_join_strings(result_members, " ", 0, NULL);
+
+  nodefamily_free(nf);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp));
+  smartlist_free(result_members);
+  tor_free(formatted);
+
+  return combined;
+}
+
+/**
+ * qsort helper for encoded nodefamily elements.
+ **/
+static int
+compare_members(const void *a, const void *b)
+{
+  return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN);
+}
+
+/**
+ * Parse the member strings in <b>members</b>, returning their canonical
+ * <b>nodefamily_t</b>.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * The <b>flags</b> element is interpreted as in nodefamily_parse().
+ *
+ * If <b>unrecognized</b> is provided, fill it copies of any unrecognized
+ * members.  (Note that malformed $hexids are not considered unrecognized.)
+ **/
+nodefamily_t *
+nodefamily_from_members(const smartlist_t *members,
+                        const uint8_t *rsa_id_self,
+                        unsigned flags,
+                        smartlist_t *unrecognized_out)
+{
+  const int n_self = rsa_id_self ? 1 : 0;
+  int n_bad_elements = 0;
+  int n_members = smartlist_len(members) + n_self;
+  nodefamily_t *tmp = nodefamily_alloc(n_members);
+  uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0);
+
+  SMARTLIST_FOREACH_BEGIN(members, const char *, cp) {
+    bool bad_element = true;
+    if (is_legal_nickname(cp)) {
+      ptr[0] = NODEFAMILY_BY_NICKNAME;
+      tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname
+      memcpy(ptr+1, cp, strlen(cp));
+      tor_strlower((char*) ptr+1);
+      bad_element = false;
+    } else if (is_legal_hexdigest(cp)) {
+      char digest_buf[DIGEST_LEN];
+      char nn_buf[MAX_NICKNAME_LEN+1];
+      char nn_char=0;
+      if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) {
+        bad_element = false;
+        ptr[0] = NODEFAMILY_BY_RSA_ID;
+        memcpy(ptr+1, digest_buf, DIGEST_LEN);
+      }
+    } else {
+      if (unrecognized_out)
+        smartlist_add_strdup(unrecognized_out, cp);
+    }
+
+    if (bad_element) {
+      const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO;
+      log_fn(severity, LD_GENERAL,
+             "Bad element %s while parsing a node family.",
+             escaped(cp));
+      ++n_bad_elements;
+    } else {
+      ptr += NODEFAMILY_MEMBER_LEN;
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  if (n_bad_elements && (flags & NF_REJECT_MALFORMED))
+    goto err;
+
+  if (rsa_id_self) {
+    /* Add self. */
+    ptr[0] = NODEFAMILY_BY_RSA_ID;
+    memcpy(ptr+1, rsa_id_self, DIGEST_LEN);
+  }
+
+  n_members -= n_bad_elements;
+
+  /* Sort tmp into canonical order. */
+  qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN,
+        compare_members);
+
+  /* Remove duplicates. */
+  int i;
+  for (i = 0; i < n_members-1; ++i) {
+    uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i);
+    uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1);
+    if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) {
+      memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN);
+      --n_members;
+      --i;
+    }
+  }
+  int n_members_alloc = tmp->n_members;
+  tmp->n_members = n_members;
+
+  /* See if we already allocated this family. */
+  nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp);
+  if (found) {
+    /* If we did, great: incref it and return it. */
+    ++found->refcnt;
+    tor_free(tmp);
+    return found;
+  } else {
+    /* If not, insert it into the hashtable. */
+    if (n_members_alloc != n_members) {
+      /* Compact the family if needed */
+      nodefamily_t *tmp2 = nodefamily_alloc(n_members);
+      memcpy(tmp2->family_members, tmp->family_members,
+             n_members * NODEFAMILY_MEMBER_LEN);
+      tor_free(tmp);
+      tmp = tmp2;
+    }
+
+    tmp->refcnt = 1;
+    HT_INSERT(nodefamily_map, &the_node_families, tmp);
+    return tmp;
+  }
+
+ err:
+  tor_free(tmp);
+  return NULL;
+}
+
+/**
+ * Drop our reference to <b>family</b>, freeing it if there are no more
+ * references.
+ */
+void
+nodefamily_free_(nodefamily_t *family)
+{
+  if (family == NULL)
+    return;
+
+  --family->refcnt;
+
+  if (family->refcnt == 0) {
+    HT_REMOVE(nodefamily_map, &the_node_families, family);
+    tor_free(family);
+  }
+}
+
+/**
+ * Return true iff <b>family</b> contains the SHA1 RSA1024 identity
+ * <b>rsa_id</b>.
+ */
+bool
+nodefamily_contains_rsa_id(const nodefamily_t *family,
+                           const uint8_t *rsa_id)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    if (ptr[0] == NODEFAMILY_BY_RSA_ID &&
+        fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff <b>family</b> contains the nickname <b>name</b>.
+ */
+bool
+nodefamily_contains_nickname(const nodefamily_t *family,
+                             const char *name)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    // note that the strcasecmp() is safe because there is always at least one
+    // NUL in the encoded nickname, because all legal nicknames are less than
+    // DIGEST_LEN bytes long.
+    if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true if <b>family</b> contains the nickname or the RSA ID for
+ * <b>node</b>
+ **/
+bool
+nodefamily_contains_node(const nodefamily_t *family,
+                         const node_t *node)
+{
+  return
+    nodefamily_contains_nickname(family, node_get_nickname(node))
+    ||
+    nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node));
+}
+
+/**
+ * Look up every entry in <b>family</b>, and add add the corresponding
+ * node_t to <b>out</b>.
+ **/
+void
+nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                  smartlist_t *out)
+{
+  if (!family)
+    return;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    const node_t *node = NULL;
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED);
+        break;
+      case NODEFAMILY_BY_RSA_ID:
+        node = node_get_by_id((char*)ptr+1);
+        break;
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+    if (node)
+      smartlist_add(out, (void *)node);
+  }
+}
+
+/**
+ * Encode <b>family</b> as a space-separated string.
+ */
+char *
+nodefamily_format(const nodefamily_t *family)
+{
+  if (!family)
+    return tor_strdup("");
+
+  unsigned i;
+  smartlist_t *sl = smartlist_new();
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        smartlist_add_strdup(sl, (char*)ptr+1);
+        break;
+      case NODEFAMILY_BY_RSA_ID: {
+        char buf[HEX_DIGEST_LEN+2];
+        buf[0]='$';
+        base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN);
+        tor_strupper(buf);
+        smartlist_add_strdup(sl, buf);
+        break;
+      }
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+  }
+
+  char *result = smartlist_join_strings(sl, " ", 0, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Free all storage held in the nodefamily map.
+ **/
+void
+nodefamily_free_all(void)
+{
+  HT_CLEAR(nodefamily_map, &the_node_families);
+}
diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h
new file mode 100644
index 0000000000..31b71e77a0
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.h
+ * \brief Header file for nodefamily.c.
+ **/
+
+#ifndef TOR_NODEFAMILY_H
+#define TOR_NODEFAMILY_H
+
+#include "lib/malloc/malloc.h"
+#include <stdbool.h>
+
+typedef struct nodefamily_t nodefamily_t;
+struct node_t;
+struct smartlist_t;
+
+#define NF_WARN_MALFORMED    (1u<<0)
+#define NF_REJECT_MALFORMED  (1u<<1)
+
+nodefamily_t *nodefamily_parse(const char *s,
+                               const uint8_t *rsa_id_self,
+                               unsigned flags);
+nodefamily_t *nodefamily_from_members(const struct smartlist_t *members,
+                                      const uint8_t *rsa_id_self,
+                                      unsigned flags,
+                                      smartlist_t *unrecognized_out);
+void nodefamily_free_(nodefamily_t *family);
+#define nodefamily_free(family) \
+  FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family))
+
+bool nodefamily_contains_rsa_id(const nodefamily_t *family,
+                                const uint8_t *rsa_id);
+bool nodefamily_contains_nickname(const nodefamily_t *family,
+                                  const char *name);
+bool nodefamily_contains_node(const nodefamily_t *family,
+                              const struct node_t *node);
+void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                       struct smartlist_t *out);
+char *nodefamily_format(const nodefamily_t *family);
+char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                              unsigned flags);
+
+void nodefamily_free_all(void);
+
+#endif /* !defined(TOR_NODEFAMILY_H) */
diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h
new file mode 100644
index 0000000000..20390c9308
--- /dev/null
+++ b/src/feature/nodelist/nodefamily_st.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NODEFAMILY_ST_H
+#define TOR_NODEFAMILY_ST_H
+
+#include "orconfig.h"
+#include "ht.h"
+
+struct nodefamily_t {
+  /** Entry for this nodefamily_t within the hashtable. */
+  HT_ENTRY(nodefamily_t) ht_ent;
+  /** Reference count.  (The hashtable is not treated as a reference */
+  uint32_t refcnt;
+  /** Number of items encoded in <b>family_members</b>. */
+  uint32_t n_members;
+  /* A byte-array encoding the members of this family.  We encode each member
+   * as one byte to indicate whether it's a nickname or a fingerprint, plus
+   * DIGEST_LEN bytes of data.  The entries are lexically sorted.
+   */
+  uint8_t family_members[FLEXIBLE_ARRAY_MEMBER];
+};
+
+#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN)
+
+/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID.
+ */
+#define NODEFAMILY_BY_RSA_ID 0
+/** Tag byte, indicates that the following bytes are a NUL-padded nickname.
+ */
+#define NODEFAMILY_BY_NICKNAME 1
+
+/**
+ * Number of bytes to allocate in the array for a nodefamily_t with N members.
+ **/
+#define NODEFAMILY_ARRAY_SIZE(n) \
+  ((n) * NODEFAMILY_MEMBER_LEN)
+
+/**
+ * Pointer to the i'th member of <b>nf</b>, as encoded.
+ */
+#define NODEFAMILY_MEMBER_PTR(nf, i) \
+  (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN]))
+
+#endif /* !defined(TOR_NODEFAMILY_ST_H) */
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
index 99d7f746a8..9191173c3b 100644
--- a/src/feature/nodelist/nodelist.c
+++ b/src/feature/nodelist/nodelist.c
@@ -49,7 +49,7 @@
 #include "core/or/protover.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/hs/hs_client.h"
@@ -59,6 +59,7 @@
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
@@ -454,22 +455,43 @@ node_add_to_address_set(const node_t *node)
 
   if (node->rs) {
     if (node->rs->addr)
-      address_set_add_ipv4h(the_nodelist->node_addrs, node->rs->addr);
+      nodelist_add_addr4_to_address_set(node->rs->addr);
     if (!tor_addr_is_null(&node->rs->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->rs->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->rs->ipv6_addr);
   }
   if (node->ri) {
     if (node->ri->addr)
-      address_set_add_ipv4h(the_nodelist->node_addrs, node->ri->addr);
+      nodelist_add_addr4_to_address_set(node->ri->addr);
     if (!tor_addr_is_null(&node->ri->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->ri->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->ri->ipv6_addr);
   }
   if (node->md) {
     if (!tor_addr_is_null(&node->md->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->md->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->md->ipv6_addr);
   }
 }
 
+/** Add the given v4 address into the nodelist address set. */
+void
+nodelist_add_addr4_to_address_set(const uint32_t addr)
+{
+  if (!the_nodelist || !the_nodelist->node_addrs || addr == 0) {
+    return;
+  }
+  address_set_add_ipv4h(the_nodelist->node_addrs, addr);
+}
+
+/** Add the given v6 address into the nodelist address set. */
+void
+nodelist_add_addr6_to_address_set(const tor_addr_t *addr)
+{
+  if (BUG(!addr) || tor_addr_is_null(addr) || tor_addr_is_v4(addr) ||
+      !the_nodelist || !the_nodelist->node_addrs) {
+    return;
+  }
+  address_set_add(the_nodelist->node_addrs, addr);
+}
+
 /** Return true if <b>addr</b> is the address of some node in the nodelist.
  * If not, probably return false. */
 int
@@ -611,9 +633,12 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH(the_nodelist->nodes, node_t *, node,
                     node->rs = NULL);
 
-  /* Conservatively estimate that every node will have 2 addresses. */
-  const int estimated_addresses = smartlist_len(ns->routerstatus_list) *
-                                  get_estimated_address_per_node();
+  /* Conservatively estimate that every node will have 2 addresses (v4 and
+   * v6). Then we add the number of configured trusted authorities we have. */
+  int estimated_addresses = smartlist_len(ns->routerstatus_list) *
+                            get_estimated_address_per_node();
+  estimated_addresses += (get_n_authorities(V3_DIRINFO & BRIDGE_DIRINFO) *
+                          get_estimated_address_per_node());
   address_set_free(the_nodelist->node_addrs);
   the_nodelist->node_addrs = address_set_new(estimated_addresses);
 
@@ -664,6 +689,9 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
     node_add_to_address_set(node);
   } SMARTLIST_FOREACH_END(node);
+  /* Then, add all trusted configured directories. Some might not be in the
+   * consensus so make sure we know them. */
+  dirlist_add_trusted_dir_addresses();
 
   if (! authdir) {
     SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
@@ -943,7 +971,7 @@ nodelist_ensure_freshness(networkstatus_t *ns)
 /** Return a list of a node_t * for every node we know about.  The caller
  * MUST NOT modify the list. (You can set and clear flags in the nodes if
  * you must, but you must not add or remove nodes.) */
-MOCK_IMPL(smartlist_t *,
+MOCK_IMPL(const smartlist_t *,
 nodelist_get_list,(void))
 {
   init_nodelist();
@@ -1105,7 +1133,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
 /** Dummy object that should be unreturnable.  Used to ensure that
  * node_get_protover_summary_flags() always returns non-NULL. */
 static const protover_summary_flags_t zero_protover_flags = {
-  0,0,0,0,0,0,0
+  0,0,0,0,0,0,0,0,0
 };
 
 /** Return the protover_summary_flags for a given node. */
@@ -1165,6 +1193,17 @@ node_supports_ed25519_hs_intro(const node_t *node)
   return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
 }
 
+/** Return true iff <b>node</b> supports the DoS ESTABLISH_INTRO cell
+ * extenstion. */
+int
+node_supports_establish_intro_dos_extension(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->
+                           supports_establish_intro_dos_extension;
+}
+
 /** Return true iff <b>node</b> supports to be a rendezvous point for hidden
  * service version 3 (HSRend=2). */
 int
@@ -1188,6 +1227,102 @@ node_get_rsa_id_digest(const node_t *node)
   return (const uint8_t*)node->identity;
 }
 
+/* Returns a new smartlist with all possible link specifiers from node:
+ *  - legacy ID is mandatory thus MUST be present in node;
+ *  - include ed25519 link specifier if present in the node, and the node
+ *    supports ed25519 link authentication, and:
+ *    - if direct_conn is true, its link versions are compatible with us,
+ *    - if direct_conn is false, regardless of its link versions;
+ *  - include IPv4 link specifier, if the primary address is not IPv4, log a
+ *    BUG() warning, and return an empty smartlist;
+ *  - include IPv6 link specifier if present in the node.
+ *
+ * If node is NULL, returns an empty smartlist.
+ *
+ * The smartlist must be freed using link_specifier_smartlist_free(). */
+smartlist_t *
+node_get_link_specifier_smartlist(const node_t *node, bool direct_conn)
+{
+  link_specifier_t *ls;
+  tor_addr_port_t ap;
+  smartlist_t *lspecs = smartlist_new();
+
+  if (!node)
+    return lspecs;
+
+  /* Get the relay's IPv4 address. */
+  node_get_prim_orport(node, &ap);
+
+  /* We expect the node's primary address to be a valid IPv4 address.
+   * This conforms to the protocol, which requires either an IPv4 or IPv6
+   * address (or both). */
+  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
+      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
+    return lspecs;
+  }
+
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
+  link_specifier_set_un_ipv4_port(ls, ap.port);
+  /* Four bytes IPv4 and two bytes port. */
+  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
+                            sizeof(ap.port));
+  smartlist_add(lspecs, ls);
+
+  /* Legacy ID is mandatory and will always be present in node. */
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
+         link_specifier_getlen_un_legacy_id(ls));
+  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
+  smartlist_add(lspecs, ls);
+
+  /* ed25519 ID is only included if the node has it, and the node declares a
+   protocol version that supports ed25519 link authentication.
+   If direct_conn is true, we also require that the node's link version is
+   compatible with us. (Otherwise, we will be sending the ed25519 key
+   to another tor, which may support different link versions.) */
+  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
+      node_supports_ed25519_link_authentication(node, direct_conn)) {
+    ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, LS_ED25519_ID);
+    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
+           link_specifier_getlen_un_ed25519_id(ls));
+    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
+    smartlist_add(lspecs, ls);
+  }
+
+  /* Check for IPv6. If so, include it as well. */
+  if (node_has_ipv6_orport(node)) {
+    ls = link_specifier_new();
+    node_get_pref_ipv6_orport(node, &ap);
+    link_specifier_set_ls_type(ls, LS_IPV6);
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
+    smartlist_add(lspecs, ls);
+  }
+
+  return lspecs;
+}
+
+/* Free a link specifier list. */
+void
+link_specifier_smartlist_free_(smartlist_t *ls_list)
+{
+  if (!ls_list)
+    return;
+
+  SMARTLIST_FOREACH(ls_list, link_specifier_t *, lspec,
+                    link_specifier_free(lspec));
+  smartlist_free(ls_list);
+}
+
 /** Return the nickname of <b>node</b>, or NULL if we can't find one. */
 const char *
 node_get_nickname(const node_t *node)
@@ -1327,8 +1462,7 @@ node_exit_policy_rejects_all(const node_t *node)
   if (node->ri)
     return node->ri->policy_is_reject_star;
   else if (node->md)
-    return node->md->exit_policy == NULL ||
-      short_policy_is_reject_star(node->md->exit_policy);
+    return node->md->policy_is_reject_star;
   else
     return 1;
 }
@@ -1503,19 +1637,6 @@ node_is_me(const node_t *node)
   return router_digest_is_me(node->identity);
 }
 
-/** Return <b>node</b> declared family (as a list of names), or NULL if
- * the node didn't declare a family. */
-const smartlist_t *
-node_get_declared_family(const node_t *node)
-{
-  if (node->ri && node->ri->declared_family)
-    return node->ri->declared_family;
-  else if (node->md && node->md->family)
-    return node->md->family;
-  else
-    return NULL;
-}
-
 /* Does this node have a valid IPv6 address?
  * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for
  * checking specific ports. */
@@ -1775,7 +1896,7 @@ microdesc_has_curve25519_onion_key(const microdesc_t *md)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -1855,7 +1976,7 @@ node_set_country(node_t *node)
 void
 nodelist_refresh_countries(void)
 {
-  smartlist_t *nodes = nodelist_get_list();
+  const smartlist_t *nodes = nodelist_get_list();
   SMARTLIST_FOREACH(nodes, node_t *, node,
                     node_set_country(node));
 }
@@ -1866,6 +1987,9 @@ int
 addrs_in_same_network_family(const tor_addr_t *a1,
                              const tor_addr_t *a2)
 {
+  if (tor_addr_is_null(a1) || tor_addr_is_null(a2))
+    return 0;
+
   switch (tor_addr_family(a1)) {
     case AF_INET:
       return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
@@ -1881,7 +2005,7 @@ addrs_in_same_network_family(const tor_addr_t *a1,
  * (case-insensitive), or if <b>node's</b> identity key digest
  * matches a hexadecimal value stored in <b>nickname</b>.  Return
  * false otherwise. */
-static int
+STATIC int
 node_nickname_matches(const node_t *node, const char *nickname)
 {
   const char *n = node_get_nickname(node);
@@ -1893,7 +2017,7 @@ node_nickname_matches(const node_t *node, const char *nickname)
 }
 
 /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
-static inline int
+STATIC int
 node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
 {
   if (!lst) return 0;
@@ -1904,6 +2028,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
   return 0;
 }
 
+/** Return true iff n1's declared family contains n2. */
+STATIC int
+node_family_contains(const node_t *n1, const node_t *n2)
+{
+  if (n1->ri && n1->ri->declared_family) {
+    return node_in_nickname_smartlist(n1->ri->declared_family, n2);
+  } else if (n1->md) {
+    return nodefamily_contains_node(n1->md->family, n2);
+  } else {
+    return 0;
+  }
+}
+
+/**
+ * Return true iff <b>node</b> has declared a nonempty family.
+ **/
+STATIC bool
+node_has_declared_family(const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    return true;
+  }
+
+  if (node->md && node->md->family) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Add to <b>out</b> every node_t that is listed by <b>node</b> as being in
+ * its family.  (Note that these nodes are not in node's family unless they
+ * also agree that node is in their family.)
+ **/
+STATIC void
+node_lookup_declared_family(smartlist_t *out, const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) {
+      const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED);
+      if (n2) {
+        smartlist_add(out, (node_t *)n2);
+      }
+    } SMARTLIST_FOREACH_END(name);
+    return;
+  }
+
+  if (node->md && node->md->family) {
+    nodefamily_add_nodes_to_smartlist(node->md->family, out);
+  }
+}
+
 /** Return true iff r1 and r2 are in the same family, but not the same
  * router. */
 int
@@ -1916,19 +2095,20 @@ nodes_in_same_family(const node_t *node1, const node_t *node2)
     tor_addr_t a1, a2;
     node_get_addr(node1, &a1);
     node_get_addr(node2, &a2);
-    if (addrs_in_same_network_family(&a1, &a2))
+
+    tor_addr_port_t ap6_1, ap6_2;
+    node_get_pref_ipv6_orport(node1, &ap6_1);
+    node_get_pref_ipv6_orport(node2, &ap6_2);
+
+    if (addrs_in_same_network_family(&a1, &a2) ||
+        addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr))
       return 1;
   }
 
   /* Are they in the same family because the agree they are? */
-  {
-    const smartlist_t *f1, *f2;
-    f1 = node_get_declared_family(node1);
-    f2 = node_get_declared_family(node2);
-    if (f1 && f2 &&
-        node_in_nickname_smartlist(f1, node2) &&
-        node_in_nickname_smartlist(f2, node1))
-      return 1;
+  if (node_family_contains(node1, node2) &&
+      node_family_contains(node2, node1)) {
+    return 1;
   }
 
   /* Are they in the same family because the user says they are? */
@@ -1956,13 +2136,10 @@ void
 nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
 {
   const smartlist_t *all_nodes = nodelist_get_list();
-  const smartlist_t *declared_family;
   const or_options_t *options = get_options();
 
   tor_assert(node);
 
-  declared_family = node_get_declared_family(node);
-
   /* Let's make sure that we have the node itself, if it's a real node. */
   {
     const node_t *real_node = node_get_by_id(node->identity);
@@ -1973,35 +2150,35 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
   /* First, add any nodes with similar network addresses. */
   if (options->EnforceDistinctSubnets) {
     tor_addr_t node_addr;
+    tor_addr_port_t node_ap6;
     node_get_addr(node, &node_addr);
+    node_get_pref_ipv6_orport(node, &node_ap6);
 
     SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) {
       tor_addr_t a;
+      tor_addr_port_t ap6;
       node_get_addr(node2, &a);
-      if (addrs_in_same_network_family(&a, &node_addr))
+      node_get_pref_ipv6_orport(node2, &ap6);
+      if (addrs_in_same_network_family(&a, &node_addr) ||
+          addrs_in_same_network_family(&ap6.addr, &node_ap6.addr))
         smartlist_add(sl, (void*)node2);
     } SMARTLIST_FOREACH_END(node2);
   }
 
-  /* Now, add all nodes in the declared_family of this node, if they
+  /* Now, add all nodes in the declared family of this node, if they
    * also declare this node to be in their family. */
-  if (declared_family) {
+  if (node_has_declared_family(node)) {
+    smartlist_t *declared_family = smartlist_new();
+    node_lookup_declared_family(declared_family, node);
+
     /* Add every r such that router declares familyness with node, and node
      * declares familyhood with router. */
-    SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
-      const node_t *node2;
-      const smartlist_t *family2;
-      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
-        continue;
-      if (!(family2 = node_get_declared_family(node2)))
-        continue;
-      SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) {
-          if (node_nickname_matches(node, name2)) {
-            smartlist_add(sl, (void*)node2);
-            break;
-          }
-      } SMARTLIST_FOREACH_END(name2);
-    } SMARTLIST_FOREACH_END(name);
+    SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) {
+      if (node_family_contains(node2, node)) {
+        smartlist_add(sl, (void*)node2);
+      }
+    } SMARTLIST_FOREACH_END(node2);
+    smartlist_free(declared_family);
   }
 
   /* If the user declared any families locally, honor those too. */
@@ -2306,7 +2483,7 @@ compute_frac_paths_available(const networkstatus_t *consensus,
   const int authdir = authdir_mode_v3(options);
 
   count_usable_descriptors(num_present_out, num_usable_out,
-                           mid, consensus, now, NULL,
+                           mid, consensus, now, options->MiddleNodes,
                            USABLE_DESCRIPTOR_ALL);
   log_debug(LD_NET,
             "%s: %d present, %d usable",
@@ -2508,7 +2685,7 @@ count_loading_descriptors_progress(void)
   if (fraction > 1.0)
     return 0; /* it's not the number of descriptors holding us back */
   return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int)
-    (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 -
+    (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 -
                BOOTSTRAP_STATUS_LOADING_DESCRIPTORS));
 }
 
@@ -2595,7 +2772,7 @@ update_router_have_minimum_dir_info(void)
   /* If paths have just become available in this update. */
   if (res && !have_min_dir_info) {
     control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO");
-    control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+    control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
     log_info(LD_DIR,
              "We now have enough directory information to build circuits.");
   }
diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h
index c430f497d5..87cfa48e25 100644
--- a/src/feature/nodelist/nodelist.h
+++ b/src/feature/nodelist/nodelist.h
@@ -35,6 +35,8 @@ node_t *nodelist_add_microdesc(microdesc_t *md);
 void nodelist_set_consensus(networkstatus_t *ns);
 void nodelist_ensure_freshness(networkstatus_t *ns);
 int nodelist_probably_contains_address(const tor_addr_t *addr);
+void nodelist_add_addr4_to_address_set(const uint32_t addr);
+void nodelist_add_addr6_to_address_set(const tor_addr_t *addr);
 
 void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md);
 void nodelist_remove_routerinfo(routerinfo_t *ri);
@@ -68,7 +70,6 @@ const char *node_get_platform(const node_t *node);
 uint32_t node_get_prim_addr_ipv4h(const node_t *node);
 void node_get_address_string(const node_t *node, char *cp, size_t len);
 long node_get_declared_uptime(const node_t *node);
-const smartlist_t *node_get_declared_family(const node_t *node);
 const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
 int node_ed25519_id_matches(const node_t *node,
                             const struct ed25519_public_key_t *id);
@@ -77,7 +78,13 @@ int node_supports_ed25519_link_authentication(const node_t *node,
 int node_supports_v3_hsdir(const node_t *node);
 int node_supports_ed25519_hs_intro(const node_t *node);
 int node_supports_v3_rendezvous_point(const node_t *node);
+int node_supports_establish_intro_dos_extension(const node_t *node);
 const uint8_t *node_get_rsa_id_digest(const node_t *node);
+smartlist_t *node_get_link_specifier_smartlist(const node_t *node,
+                                               bool direct_conn);
+void link_specifier_smartlist_free_(smartlist_t *ls_list);
+#define link_specifier_smartlist_free(ls_list) \
+  FREE_AND_NULL(smartlist_t, link_specifier_smartlist_free_, (ls_list))
 
 int node_has_ipv6_addr(const node_t *node);
 int node_has_ipv6_orport(const node_t *node);
@@ -97,7 +104,7 @@ const struct curve25519_public_key_t *node_get_curve25519_onion_key(
                                   const node_t *node);
 crypto_pk_t *node_get_rsa_onion_key(const node_t *node);
 
-MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
+MOCK_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 /* Temporary during transition to multiple addresses.  */
 void node_get_addr(const node_t *node, tor_addr_t *addr_out);
@@ -155,10 +162,16 @@ int count_loading_descriptors_progress(void);
 
 #ifdef NODELIST_PRIVATE
 
+STATIC int node_nickname_matches(const node_t *node, const char *nickname);
+STATIC int node_in_nickname_smartlist(const smartlist_t *lst,
+                                      const node_t *node);
+STATIC int node_family_contains(const node_t *n1, const node_t *n2);
+STATIC bool node_has_declared_family(const node_t *node);
+STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node);
+
 #ifdef TOR_UNIT_TESTS
 
-STATIC void
-node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h
index bfa28c7754..8465060f93 100644
--- a/src/feature/nodelist/routerinfo.h
+++ b/src/feature/nodelist/routerinfo.h
@@ -17,11 +17,9 @@ void router_get_prim_orport(const routerinfo_t *router,
 int router_has_orport(const routerinfo_t *router,
                       const tor_addr_port_t *orport);
 
-void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
-
 smartlist_t *router_get_all_orports(const routerinfo_t *ri);
 
 const char *router_purpose_to_string(uint8_t p);
 uint8_t router_purpose_from_string(const char *s);
 
-#endif
+#endif /* !defined(TOR_ROUTERINFO_H) */
diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h
index 59656818c1..59fd56d0a0 100644
--- a/src/feature/nodelist/routerinfo_st.h
+++ b/src/feature/nodelist/routerinfo_st.h
@@ -112,4 +112,4 @@ struct routerinfo_t {
   uint8_t purpose;
 };
 
-#endif
+#endif /* !defined(ROUTERINFO_ST_H) */
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
index c7fa868929..b3cf3e2394 100644
--- a/src/feature/nodelist/routerlist.c
+++ b/src/feature/nodelist/routerlist.c
@@ -67,7 +67,7 @@
 #include "core/mainloop/mainloop.h"
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dirauth/reachability.h"
@@ -160,7 +160,7 @@ static time_t last_descriptor_download_attempted = 0;
  *
  * From time to time, we replace "cached-descriptors" with a new file
  * containing only the live, non-superseded descriptors, and clear
- * cached-routers.new.
+ * cached-descriptors.new.
  *
  * On startup, we read both files.
  */
@@ -1463,12 +1463,13 @@ router_descriptor_is_older_than,(const routerinfo_t *router, int seconds))
 }
 
 /** Add <b>router</b> to the routerlist, if we don't already have it.  Replace
- * older entries (if any) with the same key.  Note: Callers should not hold
- * their pointers to <b>router</b> if this function fails; <b>router</b>
- * will either be inserted into the routerlist or freed. Similarly, even
- * if this call succeeds, they should not hold their pointers to
- * <b>router</b> after subsequent calls with other routerinfo's -- they
- * might cause the original routerinfo to get freed.
+ * older entries (if any) with the same key.
+ *
+ * Note: Callers should not hold their pointers to <b>router</b> if this
+ * function fails; <b>router</b> will either be inserted into the routerlist or
+ * freed. Similarly, even if this call succeeds, they should not hold their
+ * pointers to <b>router</b> after subsequent calls with other routerinfo's --
+ * they might cause the original routerinfo to get freed.
  *
  * Returns the status for the operation. Might set *<b>msg</b> if it wants
  * the poster of the router to know something.
@@ -1930,6 +1931,8 @@ routerlist_remove_old_routers(void)
 void
 routerlist_descriptors_added(smartlist_t *sl, int from_cache)
 {
+  // XXXX use pubsub mechanism here.
+
   tor_assert(sl);
   control_event_descriptors_changed(sl);
   SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) {
@@ -1937,7 +1940,9 @@ routerlist_descriptors_added(smartlist_t *sl, int from_cache)
       learned_bridge_descriptor(ri, from_cache);
     if (ri->needs_retest_if_added) {
       ri->needs_retest_if_added = 0;
+#ifdef HAVE_MODULE_DIRAUTH
       dirserv_single_reachability_test(approx_time(), ri);
+#endif
     }
   } SMARTLIST_FOREACH_END(ri);
 }
@@ -2975,7 +2980,7 @@ routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey,
   digest256_matches = tor_memeq(ei->digest256,
                                 sd->extra_info_digest256, DIGEST256_LEN);
   digest256_matches |=
-    tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
+    fast_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
 
   /* The identity must match exactly to have been generated at the same time
    * by the same router. */
@@ -3059,7 +3064,7 @@ routerinfo_has_curve25519_onion_key(const routerinfo_t *ri)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -3227,6 +3232,8 @@ refresh_all_country_info(void)
     routerset_refresh_countries(options->EntryNodes);
   if (options->ExitNodes)
     routerset_refresh_countries(options->ExitNodes);
+  if (options->MiddleNodes)
+    routerset_refresh_countries(options->MiddleNodes);
   if (options->ExcludeNodes)
     routerset_refresh_countries(options->ExcludeNodes);
   if (options->ExcludeExitNodes)
diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h
index 5771ebb1ab..dc9203e015 100644
--- a/src/feature/nodelist/routerlist.h
+++ b/src/feature/nodelist/routerlist.h
@@ -37,9 +37,12 @@ typedef enum was_router_added_t {
   ROUTER_WAS_NOT_WANTED = -6,
   /* Router descriptor was rejected because it was older than
    * OLD_ROUTER_DESC_MAX_AGE. */
-  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
-  /* DOCDOC */
-  ROUTER_CERTS_EXPIRED = -8
+  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'ROUTER_IS_ALREADY_KNOWN' */
+  /* Some certs on this router are expired. */
+  ROUTER_CERTS_EXPIRED = -8,
+  /* We couldn't format the annotations for this router. This is a directory
+   * authority bug. */
+  ROUTER_AUTHDIR_BUG_ANNOTATIONS = -10
 } was_router_added_t;
 
 /** How long do we avoid using a directory server after it's given us a 503? */
diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h
index 7446ead3cb..10b919a1bf 100644
--- a/src/feature/nodelist/routerlist_st.h
+++ b/src/feature/nodelist/routerlist_st.h
@@ -36,5 +36,5 @@ struct routerlist_t {
   desc_store_t extrainfo_store;
 };
 
-#endif
+#endif /* !defined(ROUTERLIST_ST_H) */
 
diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c
index 55e2756959..9a205d39b7 100644
--- a/src/feature/nodelist/routerset.c
+++ b/src/feature/nodelist/routerset.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2001 Matej Pfajfar.
-n * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
@@ -34,6 +34,9 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/nickname.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
 #include "lib/geoip/geoip.h"
 
 #include "core/or/addr_policy_st.h"
@@ -41,6 +44,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
+#include "lib/confmgt/var_type_def_st.h"
 
 /** Return a new empty routerset. */
 routerset_t *
@@ -378,7 +382,7 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
   } else {
     /* We need to iterate over the routerlist to get all the ones of the
      * right kind. */
-    smartlist_t *nodes = nodelist_get_list();
+    const smartlist_t *nodes = nodelist_get_list();
     SMARTLIST_FOREACH(nodes, const node_t *, node, {
         if (running_only && !node->is_running)
           continue;
@@ -461,3 +465,111 @@ routerset_free_(routerset_t *routerset)
   bitarray_free(routerset->countries);
   tor_free(routerset);
 }
+
+/**
+ * config helper: parse a routerset-typed variable.
+ *
+ * Takes as input as a single line in <b>line</b>; writes its results into a
+ * routerset_t** passed as <b>target</b>.  On success return 0; on failure
+ * return -1 and store an error message into *<b>errmsg</b>.
+ **/
+/*
+ * Warning: For this type, the default value (NULL) and "" are sometimes
+ * considered different values.  That is generally risky, and best avoided for
+ * other types in the future.  For cases where we want the default to be "all
+ * routers" (like EntryNodes) we should add a new routerset value indicating
+ * "all routers" (see #31908)
+ */
+static int
+routerset_kv_parse(void *target, const config_line_t *line, char **errmsg,
+                  const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)target;
+  routerset_free(*p); // clear the old value, if any.
+  routerset_t *rs = routerset_new();
+  if (routerset_parse(rs, line->value, line->key) < 0) {
+    routerset_free(rs);
+    *errmsg = tor_strdup("Invalid router list.");
+    return -1;
+  } else {
+    if (routerset_is_empty(rs)) {
+      /* Represent empty sets as NULL. */
+      routerset_free(rs);
+    }
+    *p = rs;
+    return 0;
+  }
+}
+
+/**
+ * config helper: encode a routerset-typed variable.
+ *
+ * Return a newly allocated string containing the value of the
+ * routerset_t** passed as <b>value</b>.
+ */
+static char *
+routerset_encode(const void *value, const void *params)
+{
+  (void)params;
+  const routerset_t **p = (const routerset_t**)value;
+  return routerset_to_string(*p);
+}
+
+/**
+ * config helper: free and clear a routerset-typed variable.
+ *
+ * Clear the routerset_t** passed as <b>value</b>.
+ */
+static void
+routerset_clear(void *value, const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)value;
+  routerset_free(*p); // sets *p to NULL.
+}
+
+/**
+ * config helper: copy a routerset-typed variable.
+ *
+ * Takes it input from a routerset_t** in <b>src</b>; writes its output to a
+ * routerset_t** in <b>dest</b>.  Returns 0 on success, -1 on (impossible)
+ * failure.
+ **/
+static int
+routerset_copy(void *dest, const void *src, const void *params)
+{
+  (void)params;
+  routerset_t **output = (routerset_t**)dest;
+  const routerset_t *input = *(routerset_t**)src;
+  routerset_free(*output); // sets *output to NULL
+  if (! routerset_is_empty(input)) {
+    *output = routerset_new();
+    routerset_union(*output, input);
+  }
+  return 0;
+}
+
+/**
+ * Function table to implement a routerset_t-based configuration type.
+ **/
+static const var_type_fns_t routerset_type_fns = {
+  .kv_parse = routerset_kv_parse,
+  .encode = routerset_encode,
+  .clear = routerset_clear,
+  .copy = routerset_copy
+};
+
+/**
+ * Definition of a routerset_t-based configuration type.
+ *
+ * Values are mapped to and from strings using the format defined in
+ * routerset_parse(): nicknames, IP address patterns, and fingerprints--with
+ * optional space, separated by commas.
+ *
+ * Empty sets are represented as NULL.
+ **/
+const var_type_def_t ROUTERSET_type_defn = {
+  .name = "RouterList",
+  .fns = &routerset_type_fns
+};
diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h
index ca8b6fed93..f3bf4a1f7c 100644
--- a/src/feature/nodelist/routerset.h
+++ b/src/feature/nodelist/routerset.h
@@ -44,6 +44,9 @@ void routerset_free_(routerset_t *routerset);
 #define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
 int routerset_len(const routerset_t *set);
 
+struct var_type_def_t;
+extern const struct var_type_def_t ROUTERSET_type_defn;
+
 #ifdef ROUTERSET_PRIVATE
 #include "lib/container/bitarray.h"
 
diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h
index 288edf5982..46337c9e52 100644
--- a/src/feature/nodelist/routerstatus_st.h
+++ b/src/feature/nodelist/routerstatus_st.h
@@ -47,6 +47,8 @@ struct routerstatus_t {
   unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
                              * or it claims to accept tunnelled dir requests.
                              */
+  unsigned int is_staledesc:1; /** True iff the authorities think this router
+                                * should upload a new descriptor soon. */
 
   unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
   unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
@@ -76,5 +78,5 @@ struct routerstatus_t {
 
 };
 
-#endif
+#endif /* !defined(ROUTERSTATUS_ST_H) */
 
diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h
index bdcebf184a..64c28f7440 100644
--- a/src/feature/nodelist/signed_descriptor_st.h
+++ b/src/feature/nodelist/signed_descriptor_st.h
@@ -57,5 +57,5 @@ struct signed_descriptor_t {
   unsigned int send_unencrypted : 1;
 };
 
-#endif
+#endif /* !defined(SIGNED_DESCRIPTOR_ST_H) */
 
diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c
index b0197e9f13..270c14eb1c 100644
--- a/src/feature/nodelist/torcert.c
+++ b/src/feature/nodelist/torcert.c
@@ -74,7 +74,7 @@ tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
   tor_assert(real_len == alloc_len);
   tor_assert(real_len > ED25519_SIG_LEN);
   uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
-  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
+  tor_assert(fast_mem_is_zero((char*)sig, ED25519_SIG_LEN));
 
   ed25519_signature_t signature;
   if (ed25519_sign(&signature, encoded,
@@ -290,8 +290,8 @@ tor_cert_describe_signature_status(const tor_cert_t *cert)
 }
 
 /** Return a new copy of <b>cert</b> */
-tor_cert_t *
-tor_cert_dup(const tor_cert_t *cert)
+MOCK_IMPL(tor_cert_t *,
+tor_cert_dup,(const tor_cert_t *cert))
 {
   tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
   if (cert->encoded)
diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h
index 492275b514..03d5bdca93 100644
--- a/src/feature/nodelist/torcert.h
+++ b/src/feature/nodelist/torcert.h
@@ -71,7 +71,7 @@ int tor_cert_checksig(tor_cert_t *cert,
                       const ed25519_public_key_t *pubkey, time_t now);
 const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
 
-tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
+MOCK_DECL(tor_cert_t *,tor_cert_dup,(const tor_cert_t *cert));
 int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 
diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h
index 366754c166..0d909da260 100644
--- a/src/feature/nodelist/vote_routerstatus_st.h
+++ b/src/feature/nodelist/vote_routerstatus_st.h
@@ -38,4 +38,4 @@ struct vote_routerstatus_t {
   uint8_t ed25519_id[ED25519_PUBKEY_LEN];
 };
 
-#endif
+#endif /* !defined(VOTE_ROUTERSTATUS_ST_H) */
diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c
index e20a39482f..d62598d46f 100644
--- a/src/feature/relay/dns.c
+++ b/src/feature/relay/dns.c
@@ -59,7 +59,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
@@ -1360,6 +1360,42 @@ evdns_err_is_transient(int err)
   }
 }
 
+/**
+ * Return number of configured nameservers in <b>the_evdns_base</b>.
+ */
+size_t
+number_of_configured_nameservers(void)
+{
+  return evdns_base_count_nameservers(the_evdns_base);
+}
+
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+/**
+ * Return address of configured nameserver in <b>the_evdns_base</b>
+ * at index <b>idx</b>.
+ */
+tor_addr_t *
+configured_nameserver_address(const size_t idx)
+{
+ struct sockaddr_storage sa;
+ ev_socklen_t sa_len = sizeof(sa);
+
+ if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx,
+                                    (struct sockaddr *)&sa,
+                                    sa_len) > 0) {
+   tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t));
+   if (tor_addr_from_sockaddr(tor_addr,
+                              (const struct sockaddr *)&sa,
+                              NULL) == 0) {
+     return tor_addr;
+   }
+   tor_free(tor_addr);
+ }
+
+ return NULL;
+}
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
 /** Configure eventdns nameservers if force is true, or if the configuration
  * has changed since the last time we called this function, or if we failed on
  * our last attempt.  On Unix, this reads from /etc/resolv.conf or
@@ -1391,16 +1427,23 @@ configure_nameservers(int force)
   evdns_set_log_fn(evdns_log_cb);
   if (conf_fname) {
     log_debug(LD_FS, "stat()ing %s", conf_fname);
-    if (stat(sandbox_intern_string(conf_fname), &st)) {
+    int missing_resolv_conf = 0;
+    int stat_res = stat(sandbox_intern_string(conf_fname), &st);
+
+    if (stat_res) {
       log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s",
                conf_fname, strerror(errno));
-      goto err;
-    }
-    if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname)
+      missing_resolv_conf = 1;
+    } else if (!force && resolv_conf_fname &&
+               !strcmp(conf_fname,resolv_conf_fname)
         && st.st_mtime == resolv_conf_mtime) {
       log_info(LD_EXIT, "No change to '%s'", conf_fname);
       return 0;
     }
+
+    if (stat_res == 0 && st.st_size == 0)
+      missing_resolv_conf = 1;
+
     if (nameservers_configured) {
       evdns_base_search_clear(the_evdns_base);
       evdns_base_clear_nameservers_and_suspend(the_evdns_base);
@@ -1413,20 +1456,34 @@ configure_nameservers(int force)
           sandbox_intern_string("/etc/hosts"));
     }
 #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */
-    log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
-    if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
-        sandbox_intern_string(conf_fname)))) {
-      log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)",
-               conf_fname, conf_fname, r);
-      goto err;
-    }
-    if (evdns_base_count_nameservers(the_evdns_base) == 0) {
-      log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname);
-      goto err;
+
+    if (!missing_resolv_conf) {
+      log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
+      if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
+          sandbox_intern_string(conf_fname)))) {
+        log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers "
+                          "in '%s' (%d)", conf_fname, conf_fname, r);
+
+        if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" -
+          goto err; // in which case we expect libevent to add 127.0.0.1 as
+                    // fallback.
+      }
+      if (evdns_base_count_nameservers(the_evdns_base) == 0) {
+        log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.",
+                 conf_fname);
+      }
+
+      tor_free(resolv_conf_fname);
+      resolv_conf_fname = tor_strdup(conf_fname);
+      resolv_conf_mtime = st.st_mtime;
+    } else {
+      log_warn(LD_EXIT, "Could not read your DNS config from '%s' - "
+                        "please investigate your DNS configuration. "
+                        "This is possibly a problem. Meanwhile, falling"
+                        " back to local DNS at 127.0.0.1.", conf_fname);
+      evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1");
     }
-    tor_free(resolv_conf_fname);
-    resolv_conf_fname = tor_strdup(conf_fname);
-    resolv_conf_mtime = st.st_mtime;
+
     if (nameservers_configured)
       evdns_base_resume(the_evdns_base);
   }
diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h
index e4474cdf43..7b2a31a311 100644
--- a/src/feature/relay/dns.h
+++ b/src/feature/relay/dns.h
@@ -45,6 +45,11 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes);
 #ifdef DNS_PRIVATE
 #include "feature/relay/dns_structs.h"
 
+size_t number_of_configured_nameservers(void);
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+tor_addr_t *configured_nameserver_address(const size_t idx);
+#endif
+
 MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn,
 int is_resolve,or_circuit_t *oncirc, char **hostname_out,
 int *made_connection_pending_out, cached_resolve_t **resolve_out));
diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c
index 56c5bb96f5..c343d19b8d 100644
--- a/src/feature/relay/ext_orport.c
+++ b/src/feature/relay/ext_orport.c
@@ -20,7 +20,7 @@
 #include "core/or/or.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn)
 
   conn->base_.type = CONN_TYPE_OR;
   TO_CONN(conn)->state = 0; // set the state to a neutral value
-  control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0);
   connection_tls_start_handshake(conn, 1);
 }
 
@@ -659,4 +659,3 @@ ext_orport_free_all(void)
   if (ext_or_auth_cookie) /* Free the auth cookie */
     tor_free(ext_or_auth_cookie);
 }
-
diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c
index 696905cf5e..c37745cf33 100644
--- a/src/feature/relay/onion_queue.c
+++ b/src/feature/relay/onion_queue.c
@@ -212,10 +212,12 @@ num_ntors_per_tap(void)
 #define MIN_NUM_NTORS_PER_TAP 1
 #define MAX_NUM_NTORS_PER_TAP 100000
 
-  return networkstatus_get_param(NULL, "NumNTorsPerTAP",
-                                 DEFAULT_NUM_NTORS_PER_TAP,
-                                 MIN_NUM_NTORS_PER_TAP,
-                                 MAX_NUM_NTORS_PER_TAP);
+  int result = networkstatus_get_param(NULL, "NumNTorsPerTAP",
+                                       DEFAULT_NUM_NTORS_PER_TAP,
+                                       MIN_NUM_NTORS_PER_TAP,
+                                       MAX_NUM_NTORS_PER_TAP);
+  tor_assert(result > 0);
+  return result;
 }
 
 /** Choose which onion queue we'll pull from next. If one is empty choose
diff --git a/src/feature/relay/onion_queue.h b/src/feature/relay/onion_queue.h
index 0df921e057..cf478bc1a0 100644
--- a/src/feature/relay/onion_queue.h
+++ b/src/feature/relay/onion_queue.h
@@ -20,4 +20,4 @@ int onion_num_pending(uint16_t handshake_type);
 void onion_pending_remove(or_circuit_t *circ);
 void clear_pending_onions(void);
 
-#endif
+#endif /* !defined(TOR_ONION_QUEUE_H) */
diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c
new file mode 100644
index 0000000000..b48b495895
--- /dev/null
+++ b/src/feature/relay/relay_periodic.c
@@ -0,0 +1,308 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.c
+ * @brief Periodic functions for the relay subsytem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+#include "core/mainloop/cpuworker.h" // XXXX use a pubsub event.
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/circuituse.h" // XXXX move have_performed_bandwidth_test
+
+#include "feature/relay/dns.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/stats/predict_ports.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/control/control_events.h"
+
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: If we're a server and initializing dns failed, retry.
+ */
+static int
+retry_dns_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+#define RETRY_DNS_INTERVAL (10*60)
+  if (server_mode(options) && has_dns_init_failed())
+    dns_init();
+  return RETRY_DNS_INTERVAL;
+}
+
+DECLARE_EVENT(retry_dns, ROUTER, 0);
+
+static int dns_honesty_first_time = 1;
+
+/**
+ * Periodic event: if we're an exit, see if our DNS server is telling us
+ * obvious lies.
+ */
+static int
+check_dns_honesty_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  /* 9. and if we're an exit node, check whether our DNS is telling stories
+   * to us. */
+  if (net_is_disabled() ||
+      ! public_server_mode(options) ||
+      router_my_exit_policy_is_reject_star())
+    return PERIODIC_EVENT_NO_UPDATE;
+
+  if (dns_honesty_first_time) {
+    /* Don't launch right when we start */
+    dns_honesty_first_time = 0;
+    return crypto_rand_int_range(60, 180);
+  }
+
+  dns_launch_correctness_checks();
+  return 12*3600 + crypto_rand_int(12*3600);
+}
+
+DECLARE_EVENT(check_dns_honesty, RELAY, FL(NEED_NET));
+
+/* Periodic callback: rotate the onion keys after the period defined by the
+ * "onion-key-rotation-days" consensus parameter, shut down and restart all
+ * cpuworkers, and update our descriptor if necessary.
+ */
+static int
+rotate_onion_key_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_lifetime = get_onion_key_lifetime();
+    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
+    if (rotation_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL,"Rotating onion key.");
+    rotate_onion_key();
+    cpuworkers_rotate_keyinfo();
+    if (router_rebuild_descriptor(1)<0) {
+      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
+    }
+    if (advertised_server_mode() && !net_is_disabled())
+      router_upload_dir_desc_to_dirservers(0);
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(rotate_onion_key, ROUTER, 0);
+
+/** Periodic callback: consider rebuilding or and re-uploading our descriptor
+ * (if we've passed our internal checks). */
+static int
+check_descriptor_callback(time_t now, const or_options_t *options)
+{
+/** How often do we check whether part of our router info has changed in a
+ * way that would require an upload? That includes checking whether our IP
+ * address has changed. */
+#define CHECK_DESCRIPTOR_INTERVAL (60)
+
+  (void)options;
+
+  /* 2b. Once per minute, regenerate and upload the descriptor if the old
+   * one is inaccurate. */
+  if (!net_is_disabled()) {
+    check_descriptor_bandwidth_changed(now);
+    check_descriptor_ipaddress_changed(now);
+    mark_my_descriptor_dirty_if_too_old(now);
+    consider_publishable_server(0);
+  }
+
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_descriptor, ROUTER, FL(NEED_NET));
+
+static int dirport_reachability_count = 0;
+
+/**
+ * Periodic callback: check whether we're reachable (as a relay), and
+ * whether our bandwidth has changed enough that we need to
+ * publish a new descriptor.
+ */
+static int
+check_for_reachability_bw_callback(time_t now, const or_options_t *options)
+{
+  /* XXXX This whole thing was stuck in the middle of what is now
+   * XXXX check_descriptor_callback.  I'm not sure it's right. */
+
+  /* also, check religiously for reachability, if it's within the first
+   * 20 minutes of our uptime. */
+  if (server_mode(options) &&
+      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
+      !net_is_disabled()) {
+    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+      router_do_reachability_checks(1, dirport_reachability_count==0);
+      if (++dirport_reachability_count > 5)
+        dirport_reachability_count = 0;
+      return 1;
+    } else {
+      /* If we haven't checked for 12 hours and our bandwidth estimate is
+       * low, do another bandwidth test. This is especially important for
+       * bridges, since they might go long periods without much use. */
+      const routerinfo_t *me = router_get_my_routerinfo();
+      static int first_time = 1;
+      if (!first_time && me &&
+          me->bandwidthcapacity < me->bandwidthrate &&
+          me->bandwidthcapacity < 51200) {
+        reset_bandwidth_test();
+      }
+      first_time = 0;
+#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
+      return BANDWIDTH_RECHECK_INTERVAL;
+    }
+  }
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_for_reachability_bw, ROUTER, FL(NEED_NET));
+
+/**
+ * Callback: Send warnings if Tor doesn't find its ports reachable.
+ */
+static int
+reachability_warnings_callback(time_t now, const or_options_t *options)
+{
+  (void) now;
+
+  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
+  }
+
+  if (server_mode(options) &&
+      !net_is_disabled() &&
+      have_completed_a_circuit()) {
+    /* every 20 minutes, check and complain if necessary */
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me && !check_whether_orport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
+               "its ORPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->or_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
+                                  address, me->or_port);
+      tor_free(address);
+    }
+
+    if (me && !check_whether_dirport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,
+               "Your server (%s:%d) has not managed to confirm that its "
+               "DirPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->dir_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
+                                  address, me->dir_port);
+      tor_free(address);
+    }
+  }
+
+  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
+}
+
+DECLARE_EVENT(reachability_warnings, ROUTER, FL(NEED_NET));
+
+/* Periodic callback: Every 30 seconds, check whether it's time to make new
+ * Ed25519 subkeys.
+ */
+static int
+check_ed_keys_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    if (should_make_new_ed_keys(options, now)) {
+      int new_signing_key = load_ed_keys(options, now);
+      if (new_signing_key < 0 ||
+          generate_ed_link_cert(options, now, new_signing_key > 0)) {
+        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
+        tor_shutdown_event_loop_and_exit(1);
+      }
+    }
+    return 30;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_ed_keys, ROUTER, 0);
+
+/* Period callback: Check if our old onion keys are still valid after the
+ * period of time defined by the consensus parameter
+ * "onion-key-grace-period-days", otherwise expire them by setting them to
+ * NULL.
+ */
+static int
+check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_grace_period = get_onion_key_grace_period();
+    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
+    if (expiry_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL, "Expiring old onion keys.");
+    expire_old_onion_keys();
+    cpuworkers_rotate_keyinfo();
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_onion_keys_expiry_time, ROUTER, 0);
+
+void
+relay_register_periodic_events(void)
+{
+  periodic_events_register(&retry_dns_event);
+  periodic_events_register(&check_dns_honesty_event);
+  periodic_events_register(&rotate_onion_key_event);
+  periodic_events_register(&check_descriptor_event);
+  periodic_events_register(&check_for_reachability_bw_event);
+  periodic_events_register(&reachability_warnings_event);
+  periodic_events_register(&check_ed_keys_event);
+  periodic_events_register(&check_onion_keys_expiry_time_event);
+
+  dns_honesty_first_time = 1;
+  dirport_reachability_count = 0;
+}
+
+/**
+ * Update our schedule so that we'll check whether we need to update our
+ * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
+ * seconds.
+ */
+void
+reschedule_descriptor_update_check(void)
+{
+  periodic_event_reschedule(&check_descriptor_event);
+}
diff --git a/src/feature/relay/relay_periodic.h b/src/feature/relay/relay_periodic.h
new file mode 100644
index 0000000000..b6ea83c749
--- /dev/null
+++ b/src/feature/relay/relay_periodic.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.h
+ * @brief Header for feature/relay/relay_periodic.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+#define TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+
+void relay_register_periodic_events(void);
+void reschedule_descriptor_update_check(void);
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_PERIODIC_H) */
diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c
new file mode 100644
index 0000000000..106e88b2a5
--- /dev/null
+++ b/src/feature/relay/relay_sys.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.c
+ * @brief Subsystem definitions for the relay module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/onion_queue.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/relay_sys.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/router.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_relay_initialize(void)
+{
+  relay_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_relay_shutdown(void)
+{
+  dns_free_all();
+  ext_orport_free_all();
+  clear_pending_onions();
+  routerkeys_free_all();
+  router_free_all();
+}
+
+const struct subsys_fns_t sys_relay = {
+  .name = "relay",
+  .supported = true,
+  .level = 50,
+  .initialize = subsys_relay_initialize,
+  .shutdown = subsys_relay_shutdown,
+};
diff --git a/src/feature/relay/relay_sys.h b/src/feature/relay/relay_sys.h
new file mode 100644
index 0000000000..32e21d90d8
--- /dev/null
+++ b/src/feature/relay/relay_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.h
+ * @brief Header for feature/relay/relay_sys.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_SYS_H
+#define TOR_FEATURE_RELAY_RELAY_SYS_H
+
+extern const struct subsys_fns_t sys_relay;
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_SYS_H) */
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index e91550a78c..a46b522bd6 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -16,7 +16,7 @@
 #include "core/or/policies.h"
 #include "core/or/protover.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -30,6 +30,7 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
@@ -49,6 +50,7 @@
 #include "lib/encoding/confline.h"
 #include "lib/osinfo/uname.h"
 #include "lib/tls/tortls.h"
+#include "lib/version/torversion.h"
 
 #include "feature/dirauth/authmode.h"
 
@@ -58,6 +60,7 @@
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
@@ -149,6 +152,8 @@ routerinfo_err_to_string(int err)
       return "Cannot generate descriptor";
     case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
       return "Descriptor still rebuilding - not ready yet";
+    case TOR_ROUTERINFO_ERROR_INTERNAL_BUG:
+      return "Internal bug, see logs for details";
   }
 
   log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
@@ -191,8 +196,8 @@ set_onion_key(crypto_pk_t *k)
 
 /** Return the current onion key.  Requires that the onion key has been
  * loaded or generated. */
-crypto_pk_t *
-get_onion_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_onion_key,(void))
 {
   tor_assert(onionkey);
   return onionkey;
@@ -239,7 +244,7 @@ expire_old_onion_keys(void)
     lastonionkey = NULL;
   }
 
-  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+  /* We zero out the keypair. See the fast_mem_is_zero() check made in
    * construct_ntor_key_map() below. */
   memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
@@ -266,11 +271,12 @@ expire_old_onion_keys(void)
 
 /** Return the current secret onion key for the ntor handshake. Must only
  * be called from the main thread. */
-static const curve25519_keypair_t *
-get_current_curve25519_keypair(void)
+MOCK_IMPL(STATIC const struct curve25519_keypair_t *,
+get_current_curve25519_keypair,(void))
 {
   return &curve25519_onion_key;
 }
+
 /** Return a map from KEYID (the key itself) to keypairs for use in the ntor
  * handshake. Must only be called from the main thread. */
 di_digest256_map_t *
@@ -281,12 +287,12 @@ construct_ntor_key_map(void)
   const uint8_t *cur_pk = curve25519_onion_key.pubkey.public_key;
   const uint8_t *last_pk = last_curve25519_onion_key.pubkey.public_key;
 
-  if (!tor_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
+  if (!fast_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, cur_pk,
                     tor_memdup(&curve25519_onion_key,
                                sizeof(curve25519_keypair_t)));
   }
-  if (!tor_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
+  if (!fast_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
       tor_memneq(cur_pk, last_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, last_pk,
                     tor_memdup(&last_curve25519_onion_key,
@@ -337,6 +343,16 @@ set_server_identity_key(crypto_pk_t *k)
   }
 }
 
+#ifdef TOR_UNIT_TESTS
+/** Testing only -- set the server's RSA identity digest to
+ * be <b>digest</b> */
+void
+set_server_identity_key_digest_testing(const uint8_t *digest)
+{
+  memcpy(server_identitykey_digest, digest, DIGEST_LEN);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Make sure that we have set up our identity keys to match or not match as
  * appropriate, and die with an assertion if we have not. */
 static void
@@ -359,8 +375,8 @@ assert_identity_keys_ok(void)
 /** Returns the current server identity key; requires that the key has
  * been set, and that we are running as a Tor server.
  */
-crypto_pk_t *
-get_server_identity_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_server_identity_key,(void))
 {
   tor_assert(server_identitykey);
   tor_assert(server_mode(get_options()));
@@ -634,7 +650,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out,
                fname);
     goto done;
   }
-  parsed = authority_cert_parse_from_string(cert, &eos);
+  parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos);
   if (!parsed) {
     log_warn(LD_DIR, "Unable to parse certificate in %s", fname);
     goto done;
@@ -1031,7 +1047,7 @@ init_keys(void)
       return -1;
 
     keydir = get_keydir_fname("secret_onion_key_ntor.old");
-    if (tor_mem_is_zero((const char *)
+    if (fast_mem_is_zero((const char *)
                            last_curve25519_onion_key.pubkey.public_key,
                         CURVE25519_PUBKEY_LEN) &&
         file_status(keydir) == FN_FILE) {
@@ -1467,9 +1483,9 @@ static extrainfo_t *desc_extrainfo = NULL;
 static const char *desc_gen_reason = "uninitialized reason";
 /** Since when has our descriptor been "clean"?  0 if we need to regenerate it
  * now. */
-static time_t desc_clean_since = 0;
+STATIC time_t desc_clean_since = 0;
 /** Why did we mark the descriptor dirty? */
-static const char *desc_dirty_reason = "Tor just started";
+STATIC const char *desc_dirty_reason = "Tor just started";
 /** Boolean: do we need to regenerate the above? */
 static int desc_needs_upload = 0;
 
@@ -1688,10 +1704,6 @@ router_get_descriptor_gen_reason(void)
   return desc_gen_reason;
 }
 
-/** A list of nicknames that we've warned about including in our family
- * declaration verbatim rather than as digests. */
-static smartlist_t *warned_nonexistent_family = NULL;
-
 static int router_guess_address_from_dir_headers(uint32_t *guess);
 
 /** Make a current best guess at our address, either because
@@ -1804,26 +1816,159 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr)
                                                    CONN_TYPE_DIR_LISTENER);
 }
 
-/** Build a fresh routerinfo, signed server descriptor, and extra-info document
- * for this OR. Set r to the generated routerinfo, e to the generated
- * extra-info document. Return 0 on success, -1 on temporary error. Failure to
- * generate an extra-info document is not an error and is indicated by setting
- * e to NULL. Caller is responsible for freeing generated documents if 0 is
- * returned.
+/** A list of nicknames that we've warned about including in our family,
+ * for one reason or another. */
+static smartlist_t *warned_family = NULL;
+
+/**
+ * Return a new smartlist containing the family members configured in
+ * <b>options</b>.  Warn about invalid or missing entries.  Return NULL
+ * if this relay should not declare a family.
+ **/
+STATIC smartlist_t *
+get_my_declared_family(const or_options_t *options)
+{
+  if (!options->MyFamily)
+    return NULL;
+
+  if (options->BridgeRelay)
+    return NULL;
+
+  if (!warned_family)
+    warned_family = smartlist_new();
+
+  smartlist_t *declared_family = smartlist_new();
+  config_line_t *family;
+
+  /* First we try to get the whole family in the form of hexdigests. */
+  for (family = options->MyFamily; family; family = family->next) {
+    char *name = family->value;
+    const node_t *member;
+    if (options->Nickname && !strcasecmp(name, options->Nickname))
+      continue; /* Don't list ourself by nickname, that's redundant */
+    else
+      member = node_get_by_nickname(name, 0);
+
+    if (!member) {
+      /* This node doesn't seem to exist, so warn about it if it is not
+       * a hexdigest. */
+      int is_legal = is_legal_nickname_or_hexdigest(name);
+      if (!smartlist_contains_string(warned_family, name) &&
+          !is_legal_hexdigest(name)) {
+        if (is_legal)
+          log_warn(LD_CONFIG,
+                   "There is a router named %s in my declared family, but "
+                   "I have no descriptor for it. I'll use the nickname "
+                   "as is, but this may confuse clients. Please list it "
+                   "by identity digest instead.", escaped(name));
+        else
+          log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                   "family, but that isn't a legal digest or nickname. "
+                   "Skipping it.", escaped(name));
+        smartlist_add_strdup(warned_family, name);
+      }
+      if (is_legal) {
+        smartlist_add_strdup(declared_family, name);
+      }
+    } else {
+      /* List the node by digest. */
+      char *fp = tor_malloc(HEX_DIGEST_LEN+2);
+      fp[0] = '$';
+      base16_encode(fp+1,HEX_DIGEST_LEN+1,
+                    member->identity, DIGEST_LEN);
+      smartlist_add(declared_family, fp);
+
+      if (! is_legal_hexdigest(name) &&
+          !smartlist_contains_string(warned_family, name)) {
+        /* Warn if this node was not specified by hexdigest. */
+        log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                 "family, but it wasn't listed by digest. Please consider "
+                 "saying %s instead, if that's what you meant.",
+                 escaped(name), fp);
+        smartlist_add_strdup(warned_family, name);
+      }
+    }
+  }
+
+  /* Now declared_family should have the closest we can come to the
+   * identities that the user wanted.
+   *
+   * Unlike older versions of Tor, we _do_ include our own identity: this
+   * helps microdescriptor compression, and helps in-memory compression
+   * on clients. */
+  nodefamily_t *nf = nodefamily_from_members(declared_family,
+                                     router_get_my_id_digest(),
+                                     NF_WARN_MALFORMED,
+                                     NULL);
+  SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s));
+  smartlist_free(declared_family);
+  if (!nf) {
+    return NULL;
+  }
+
+  char *s = nodefamily_format(nf);
+  nodefamily_free(nf);
+
+  smartlist_t *result = smartlist_new();
+  smartlist_split_string(result, s, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  tor_free(s);
+
+  if (smartlist_len(result) == 1) {
+    /* This is a one-element list containing only ourself; instead return
+     * nothing */
+    const char *singleton = smartlist_get(result, 0);
+    bool is_me = false;
+    if (singleton[0] == '$') {
+      char d[DIGEST_LEN];
+      int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1));
+      if (n == DIGEST_LEN &&
+          fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) {
+        is_me = true;
+      }
+    }
+    if (!is_me) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Found a singleton family list with an element "
+               "that wasn't us! Element was %s", escaped(singleton));
+      // LCOV_EXCL_STOP
+    } else {
+      SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+      smartlist_free(result);
+      return NULL;
+    }
+  }
+
+  return result;
+}
+
+/** Allocate a fresh, unsigned routerinfo for this OR, without any of the
+ * fields that depend on the corresponding extrainfo.
+ *
+ * On success, set ri_out to the new routerinfo, and return 0.
+ * Caller is responsible for freeing the generated routerinfo.
+ *
+ * Returns a negative value and sets ri_out to NULL on temporary error.
  */
-int
-router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+MOCK_IMPL(STATIC int,
+router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out))
 {
-  routerinfo_t *ri;
-  extrainfo_t *ei;
+  routerinfo_t *ri = NULL;
   uint32_t addr;
   char platform[256];
   int hibernating = we_are_hibernating();
   const or_options_t *options = get_options();
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
+  if (BUG(!ri_out)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
 
   if (router_pick_published_address(options, &addr, 0) < 0) {
     log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
-    return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    result = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    goto err;
   }
 
   /* Log a message if the address in the descriptor doesn't match the ORPort
@@ -1880,8 +2025,8 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key());
   if (BUG(crypto_pk_get_digest(ri->identity_pkey,
                            ri->cache_info.identity_digest) < 0)) {
-    routerinfo_free(ri);
-    return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    result = TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    goto err;
   }
   ri->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
@@ -1918,134 +2063,260 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
     tor_free(p_tmp);
   }
 
-  if (options->MyFamily && ! options->BridgeRelay) {
-    if (!warned_nonexistent_family)
-      warned_nonexistent_family = smartlist_new();
-    ri->declared_family = smartlist_new();
-    config_line_t *family;
-    for (family = options->MyFamily; family; family = family->next) {
-       char *name = family->value;
-       const node_t *member;
-       if (!strcasecmp(name, options->Nickname))
-         continue; /* Don't list ourself, that's redundant */
-       else
-         member = node_get_by_nickname(name, 0);
-       if (!member) {
-         int is_legal = is_legal_nickname_or_hexdigest(name);
-         if (!smartlist_contains_string(warned_nonexistent_family, name) &&
-             !is_legal_hexdigest(name)) {
-           if (is_legal)
-             log_warn(LD_CONFIG,
-                      "I have no descriptor for the router named \"%s\" in my "
-                      "declared family; I'll use the nickname as is, but "
-                      "this may confuse clients.", name);
-           else
-             log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
-                      "declared family, but that isn't a legal nickname. "
-                      "Skipping it.", escaped(name));
-           smartlist_add_strdup(warned_nonexistent_family, name);
-         }
-         if (is_legal) {
-           smartlist_add_strdup(ri->declared_family, name);
-         }
-       } else if (router_digest_is_me(member->identity)) {
-         /* Don't list ourself in our own family; that's redundant */
-         /* XXX shouldn't be possible */
-       } else {
-         char *fp = tor_malloc(HEX_DIGEST_LEN+2);
-         fp[0] = '$';
-         base16_encode(fp+1,HEX_DIGEST_LEN+1,
-                       member->identity, DIGEST_LEN);
-         smartlist_add(ri->declared_family, fp);
-         if (smartlist_contains_string(warned_nonexistent_family, name))
-           smartlist_string_remove(warned_nonexistent_family, name);
-       }
-    }
+  ri->declared_family = get_my_declared_family(options);
 
-    /* remove duplicates from the list */
-    smartlist_sort_strings(ri->declared_family);
-    smartlist_uniq_strings(ri->declared_family);
+  if (options->BridgeRelay) {
+    ri->purpose = ROUTER_PURPOSE_BRIDGE;
+    /* Bridges shouldn't be able to send their descriptors unencrypted,
+     anyway, since they don't have a DirPort, and always connect to the
+     bridge authority anonymously.  But just in case they somehow think of
+     sending them on an unencrypted connection, don't allow them to try. */
+    ri->cache_info.send_unencrypted = 0;
+  } else {
+    ri->purpose = ROUTER_PURPOSE_GENERAL;
+    ri->cache_info.send_unencrypted = 1;
   }
 
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  *ri_out = NULL;
+  return result;
+
+ done:
+  *ri_out = ri;
+  return 0;
+}
+
+/** Allocate and return a fresh, unsigned extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * Uses options->Nickname to set the nickname, and options->BridgeRelay to set
+ * ei->cache_info.send_unencrypted.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+static extrainfo_t *
+router_build_fresh_unsigned_extrainfo(const routerinfo_t *ri)
+{
+  extrainfo_t *ei = NULL;
+  const or_options_t *options = get_options();
+
+  if (BUG(!ri))
+    return NULL;
+
   /* Now generate the extrainfo. */
   ei = tor_malloc_zero(sizeof(extrainfo_t));
   ei->cache_info.is_extrainfo = 1;
-  strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname));
+  strlcpy(ei->nickname, options->Nickname, sizeof(ei->nickname));
   ei->cache_info.published_on = ri->cache_info.published_on;
   ei->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
 
   memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest,
          DIGEST_LEN);
+
+  if (options->BridgeRelay) {
+    /* See note in router_build_fresh_routerinfo(). */
+    ei->cache_info.send_unencrypted = 0;
+  } else {
+    ei->cache_info.send_unencrypted = 1;
+  }
+
+  return ei;
+}
+
+/** Dump the extrainfo descriptor body for ei, sign it, and add the body and
+ * signature to ei->cache_info. Note that the extrainfo body is determined by
+ * ei, and some additional config and statistics state: see
+ * extrainfo_dump_to_string() for details.
+ *
+ * Return 0 on success, -1 on temporary error.
+ * If ei is NULL, logs a BUG() warning and returns -1.
+ * On error, ei->cache_info is not modified.
+ */
+static int
+router_dump_and_sign_extrainfo_descriptor_body(extrainfo_t *ei)
+{
+  if (BUG(!ei))
+    return -1;
+
   if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body,
                                ei, get_server_identity_key(),
                                get_master_signing_keypair()) < 0) {
     log_warn(LD_BUG, "Couldn't generate extra-info descriptor.");
-    extrainfo_free(ei);
-    ei = NULL;
-  } else {
-    ei->cache_info.signed_descriptor_len =
-      strlen(ei->cache_info.signed_descriptor_body);
-    router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
-                              ei->cache_info.signed_descriptor_len,
-                              ei->cache_info.signed_descriptor_digest);
-    crypto_digest256((char*) ei->digest256,
-                     ei->cache_info.signed_descriptor_body,
-                     ei->cache_info.signed_descriptor_len,
-                     DIGEST_SHA256);
+    return -1;
   }
 
-  /* Now finish the router descriptor. */
-  if (ei) {
-    memcpy(ri->cache_info.extra_info_digest,
-           ei->cache_info.signed_descriptor_digest,
-           DIGEST_LEN);
-    memcpy(ri->cache_info.extra_info_digest256,
-           ei->digest256,
-           DIGEST256_LEN);
-  } else {
-    /* ri was allocated with tor_malloc_zero, so there is no need to
-     * zero ri->cache_info.extra_info_digest here. */
+  ei->cache_info.signed_descriptor_len =
+    strlen(ei->cache_info.signed_descriptor_body);
+
+  router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
+                            ei->cache_info.signed_descriptor_len,
+                            ei->cache_info.signed_descriptor_digest);
+  crypto_digest256((char*) ei->digest256,
+                   ei->cache_info.signed_descriptor_body,
+                   ei->cache_info.signed_descriptor_len,
+                   DIGEST_SHA256);
+
+  return 0;
+}
+
+/** Allocate and return a fresh, signed extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+STATIC extrainfo_t *
+router_build_fresh_signed_extrainfo(const routerinfo_t *ri)
+{
+  int result = -1;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!ri))
+    return NULL;
+
+  ei = router_build_fresh_unsigned_extrainfo(ri);
+  /* router_build_fresh_unsigned_extrainfo() should not fail. */
+  if (BUG(!ei))
+    goto err;
+
+  result = router_dump_and_sign_extrainfo_descriptor_body(ei);
+  if (result < 0)
+    goto err;
+
+  goto done;
+
+ err:
+  extrainfo_free(ei);
+  return NULL;
+
+ done:
+  return ei;
+}
+
+/** Set the fields in ri that depend on ei.
+ *
+ * If ei is NULL, logs a BUG() warning and zeroes the relevant fields.
+ */
+STATIC void
+router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                        const extrainfo_t *ei)
+{
+  if (BUG(!ei)) {
+    /* Just to be safe, zero ri->cache_info.extra_info_digest here. */
+    memset(ri->cache_info.extra_info_digest, 0, DIGEST_LEN);
+    memset(ri->cache_info.extra_info_digest256, 0, DIGEST256_LEN);
+    return;
   }
+
+  /* Now finish the router descriptor. */
+  memcpy(ri->cache_info.extra_info_digest,
+         ei->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  memcpy(ri->cache_info.extra_info_digest256,
+         ei->digest256,
+         DIGEST256_LEN);
+}
+
+/** Dump the descriptor body for ri, sign it, and add the body and signature to
+ * ri->cache_info. Note that the descriptor body is determined by ri, and some
+ * additional config and state: see router_dump_router_to_string() for details.
+ *
+ * Return 0 on success, and a negative value on temporary error.
+ * If ri is NULL, logs a BUG() warning and returns a negative value.
+ * On error, ri->cache_info is not modified.
+ */
+STATIC int
+router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri)
+{
+  if (BUG(!ri))
+    return TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
   if (! (ri->cache_info.signed_descriptor_body =
           router_dump_router_to_string(ri, get_server_identity_key(),
                                        get_onion_key(),
                                        get_current_curve25519_keypair(),
                                        get_master_signing_keypair())) ) {
     log_warn(LD_BUG, "Couldn't generate router descriptor.");
-    routerinfo_free(ri);
-    extrainfo_free(ei);
     return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
   }
+
   ri->cache_info.signed_descriptor_len =
     strlen(ri->cache_info.signed_descriptor_body);
 
-  ri->purpose =
-    options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-  if (options->BridgeRelay) {
-    /* Bridges shouldn't be able to send their descriptors unencrypted,
-       anyway, since they don't have a DirPort, and always connect to the
-       bridge authority anonymously.  But just in case they somehow think of
-       sending them on an unencrypted connection, don't allow them to try. */
-    ri->cache_info.send_unencrypted = 0;
-    if (ei)
-      ei->cache_info.send_unencrypted = 0;
-  } else {
-    ri->cache_info.send_unencrypted = 1;
-    if (ei)
-      ei->cache_info.send_unencrypted = 1;
-  }
-
   router_get_router_hash(ri->cache_info.signed_descriptor_body,
                          strlen(ri->cache_info.signed_descriptor_body),
                          ri->cache_info.signed_descriptor_digest);
 
+  return 0;
+}
+
+/** Build a fresh routerinfo, signed server descriptor, and signed extrainfo
+ * document for this OR.
+ *
+ * Set r to the generated routerinfo, e to the generated extrainfo document.
+ * Failure to generate an extra-info document is not an error and is indicated
+ * by setting e to NULL.
+ * Return 0 on success, and a negative value on temporary error.
+ * Caller is responsible for freeing generated documents on success.
+ */
+int
+router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+{
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+  routerinfo_t *ri = NULL;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!r))
+    goto err;
+
+  if (BUG(!e))
+    goto err;
+
+  result = router_build_fresh_unsigned_routerinfo(&ri);
+  if (result < 0) {
+    goto err;
+  }
+  /* If ri is NULL, then result should be negative. So this check should be
+   * unreachable. */
+  if (BUG(!ri)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
+
+  ei = router_build_fresh_signed_extrainfo(ri);
+
+  /* Failing to create an ei is not an error. */
+  if (ei) {
+    router_update_routerinfo_from_extrainfo(ri, ei);
+  }
+
+  result = router_dump_and_sign_routerinfo_descriptor_body(ri);
+  if (result < 0)
+    goto err;
+
   if (ei) {
-    tor_assert(!
-          routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
-                                                 &ri->cache_info, NULL));
+     if (BUG(routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                    &ri->cache_info, NULL))) {
+       result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+       goto err;
+     }
   }
 
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  extrainfo_free(ei);
+  *r = NULL;
+  *e = NULL;
+  return result;
+
+ done:
   *r = ri;
   *e = ei;
   return 0;
@@ -2131,7 +2402,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
   /* Now we see whether we want to be retrying frequently or no.  The
    * rule here is that we'll retry frequently if we aren't listed in the
    * live consensus we have, or if the publication time of the
-   * descriptor listed for us in the consensus is very old. */
+   * descriptor listed for us in the consensus is very old, or if the
+   * consensus lists us as "stale" and we haven't regenerated since the
+   * consensus was published. */
   ns = networkstatus_get_live_consensus(now);
   if (ns) {
     rs = networkstatus_vote_find_entry(ns, server_identitykey_digest);
@@ -2139,6 +2412,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
       retry_fast_reason = "not listed in consensus";
     else if (rs->published_on < slow_cutoff)
       retry_fast_reason = "version listed in consensus is quite old";
+    else if (rs->is_staledesc && ns->valid_after > desc_clean_since)
+      retry_fast_reason = "listed as stale in consensus";
   }
 
   if (retry_fast_reason && desc_clean_since < fast_cutoff)
@@ -2384,6 +2659,10 @@ get_platform_str(char *platform, size_t len)
 /** OR only: Given a routerinfo for this router, and an identity key to sign
  * with, encode the routerinfo as a signed server descriptor and return a new
  * string encoding the result, or NULL on failure.
+ *
+ * In addition to the fields in router, this function calls
+ * onion_key_lifetime(), get_options(), and we_are_hibernating(), and uses the
+ * results to populate some fields in the descriptor.
  */
 char *
 router_dump_router_to_string(routerinfo_t *router,
@@ -2447,11 +2726,8 @@ router_dump_router_to_string(routerinfo_t *router,
       log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
       goto err;
     }
-    if (ed25519_public_to_base64(ed_fp_base64,
-                       &router->cache_info.signing_key_cert->signing_key)<0) {
-      log_err(LD_BUG,"Couldn't base64-encode identity key\n");
-      goto err;
-    }
+    ed25519_public_to_base64(ed_fp_base64,
+                            &router->cache_info.signing_key_cert->signing_key);
     tor_asprintf(&ed_cert_line, "identity-ed25519\n"
                  "-----BEGIN ED25519 CERT-----\n"
                  "%s"
@@ -2701,8 +2977,7 @@ router_dump_router_to_string(routerinfo_t *router,
     if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN,
                      signing_keypair) < 0)
       goto err;
-    if (ed25519_signature_to_base64(buf, &sig) < 0)
-      goto err;
+    ed25519_signature_to_base64(buf, &sig);
 
     smartlist_add_asprintf(chunks, "%s\n", buf);
   }
@@ -2841,34 +3116,26 @@ load_stats_file(const char *filename, const char *end_line, time_t now,
   return r;
 }
 
-/** Write the contents of <b>extrainfo</b> and aggregated statistics to
- * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on
- * success, negative on failure. */
-int
-extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
-                         crypto_pk_t *ident_key,
-                         const ed25519_keypair_t *signing_keypair)
+/** Add header strings to chunks, based on the extrainfo object extrainfo,
+ * and ed25519 keypair signing_keypair, if emit_ed_sigs is true.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_header_helper(
+                                     smartlist_t *chunks,
+                                     const extrainfo_t *extrainfo,
+                                     const ed25519_keypair_t *signing_keypair,
+                                     int emit_ed_sigs)
 {
-  const or_options_t *options = get_options();
   char identity[HEX_DIGEST_LEN+1];
   char published[ISO_TIME_LEN+1];
-  char digest[DIGEST_LEN];
-  char *bandwidth_usage;
-  int result;
-  static int write_stats_to_extrainfo = 1;
-  char sig[DIROBJ_MAX_SIG_LEN+1];
-  char *s = NULL, *pre, *contents, *cp, *s_dup = NULL;
-  time_t now = time(NULL);
-  smartlist_t *chunks = smartlist_new();
-  extrainfo_t *ei_tmp = NULL;
-  const int emit_ed_sigs = signing_keypair &&
-    extrainfo->cache_info.signing_key_cert;
   char *ed_cert_line = NULL;
+  char *pre = NULL;
+  int rv = -1;
 
   base16_encode(identity, sizeof(identity),
                 extrainfo->cache_info.identity_digest, DIGEST_LEN);
   format_iso_time(published, extrainfo->cache_info.published_on);
-  bandwidth_usage = rep_hist_get_bandwidth_lines();
   if (emit_ed_sigs) {
     if (!extrainfo->cache_info.signing_key_cert->signing_key_included ||
         !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key,
@@ -2894,21 +3161,64 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     ed_cert_line = tor_strdup("");
   }
 
-  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s",
+  /* This is the first chunk in the file. If the file is too big, other chunks
+   * are removed. So we must only add one chunk here. */
+  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n",
                extrainfo->nickname, identity,
                ed_cert_line,
-               published, bandwidth_usage);
+               published);
   smartlist_add(chunks, pre);
 
-  if (geoip_is_loaded(AF_INET))
-    smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
-                           geoip_db_digest(AF_INET));
-  if (geoip_is_loaded(AF_INET6))
-    smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
-                           geoip_db_digest(AF_INET6));
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  tor_free(ed_cert_line);
+  return rv;
+}
+
+/** Add pluggable transport and statistics strings to chunks, skipping
+ * statistics if write_stats_to_extrainfo is false.
+ * Helper for extrainfo_dump_to_string().
+ * Can not fail. */
+static void
+extrainfo_dump_to_string_stats_helper(smartlist_t *chunks,
+                                      int write_stats_to_extrainfo)
+{
+  const or_options_t *options = get_options();
+  char *contents = NULL;
+  time_t now = time(NULL);
+
+  /* If the file is too big, these chunks are removed, starting with the last
+   * chunk. So each chunk must be a complete line, and the file must be valid
+   * after each chunk. */
+
+  /* Add information about the pluggable transports we support, even if we
+   * are not publishing statistics. This information is needed by BridgeDB
+   * to distribute bridges. */
+  if (options->ServerTransportPlugin) {
+    char *pluggable_transports = pt_get_extra_info_descriptor_string();
+    if (pluggable_transports)
+      smartlist_add(chunks, pluggable_transports);
+  }
 
   if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
     log_info(LD_GENERAL, "Adding stats to extra-info descriptor.");
+    /* Bandwidth usage stats don't have their own option */
+    {
+      contents = rep_hist_get_bandwidth_lines();
+      smartlist_add(chunks, contents);
+    }
+    /* geoip hashes aren't useful unless we are publishing other stats */
+    if (geoip_is_loaded(AF_INET))
+      smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
+                             geoip_db_digest(AF_INET));
+    if (geoip_is_loaded(AF_INET6))
+      smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
+                             geoip_db_digest(AF_INET6));
     if (options->DirReqStatistics &&
         load_stats_file("stats"PATH_SEPARATOR"dirreq-stats",
                         "dirreq-stats-end", now, &contents) > 0) {
@@ -2944,50 +3254,140 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
       if (contents)
         smartlist_add(chunks, contents);
     }
+    /* bridge statistics */
+    if (should_record_bridge_info(options)) {
+      const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
+      if (bridge_stats) {
+        smartlist_add_strdup(chunks, bridge_stats);
+      }
+    }
   }
+}
 
-  /* Add information about the pluggable transports we support. */
-  if (options->ServerTransportPlugin) {
-    char *pluggable_transports = pt_get_extra_info_descriptor_string();
-    if (pluggable_transports)
-      smartlist_add(chunks, pluggable_transports);
-  }
+/** Add an ed25519 signature of chunks to chunks, using the ed25519 keypair
+ * signing_keypair.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_ed_sig_helper(
+                                     smartlist_t *chunks,
+                                     const ed25519_keypair_t *signing_keypair)
+{
+  char sha256_digest[DIGEST256_LEN];
+  ed25519_signature_t ed_sig;
+  char buf[ED25519_SIG_BASE64_LEN+1];
+  int rv = -1;
+
+  /* These are two of the three final chunks in the file. If the file is too
+   * big, other chunks are removed. So we must only add two chunks here. */
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+  crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
+                                 ED_DESC_SIGNATURE_PREFIX,
+                                 chunks, "", DIGEST_SHA256);
+  if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
+                   signing_keypair) < 0)
+    goto err;
+  ed25519_signature_to_base64(buf, &ed_sig);
 
-  if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
-    const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
-    if (bridge_stats) {
-      smartlist_add_strdup(chunks, bridge_stats);
-    }
+  smartlist_add_asprintf(chunks, "%s\n", buf);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Add an RSA signature of extrainfo_string to chunks, using the RSA key
+ * ident_key.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_rsa_sig_helper(smartlist_t *chunks,
+                                        crypto_pk_t *ident_key,
+                                        const char *extrainfo_string)
+{
+  char sig[DIROBJ_MAX_SIG_LEN+1];
+  char digest[DIGEST_LEN];
+  int rv = -1;
+
+  memset(sig, 0, sizeof(sig));
+  if (router_get_extrainfo_hash(extrainfo_string, strlen(extrainfo_string),
+                                digest) < 0 ||
+      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
+                                     ident_key) < 0) {
+    log_warn(LD_BUG, "Could not append signature to extra-info "
+                     "descriptor.");
+    goto err;
   }
+  smartlist_add_strdup(chunks, sig);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Write the contents of <b>extrainfo</b>, to * *<b>s_out</b>, signing them
+ * with <b>ident_key</b>.
+ *
+ * If ExtraInfoStatistics is 1, also write aggregated statistics and related
+ * configuration data before signing. Most statistics also have an option that
+ * enables or disables that particular statistic.
+ *
+ * Always write pluggable transport lines.
+ *
+ * Return 0 on success, negative on failure. */
+int
+extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
+                         crypto_pk_t *ident_key,
+                         const ed25519_keypair_t *signing_keypair)
+{
+  int result;
+  static int write_stats_to_extrainfo = 1;
+  char *s = NULL, *cp, *s_dup = NULL;
+  smartlist_t *chunks = smartlist_new();
+  extrainfo_t *ei_tmp = NULL;
+  const int emit_ed_sigs = signing_keypair &&
+    extrainfo->cache_info.signing_key_cert;
+  int rv = 0;
+
+  rv = extrainfo_dump_to_string_header_helper(chunks, extrainfo,
+                                              signing_keypair,
+                                              emit_ed_sigs);
+  if (rv < 0)
+    goto err;
+
+  extrainfo_dump_to_string_stats_helper(chunks, write_stats_to_extrainfo);
 
   if (emit_ed_sigs) {
-    char sha256_digest[DIGEST256_LEN];
-    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
-    crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
-                                   ED_DESC_SIGNATURE_PREFIX,
-                                   chunks, "", DIGEST_SHA256);
-    ed25519_signature_t ed_sig;
-    char buf[ED25519_SIG_BASE64_LEN+1];
-    if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
-                     signing_keypair) < 0)
-      goto err;
-    if (ed25519_signature_to_base64(buf, &ed_sig) < 0)
+    rv = extrainfo_dump_to_string_ed_sig_helper(chunks, signing_keypair);
+    if (rv < 0)
       goto err;
-
-    smartlist_add_asprintf(chunks, "%s\n", buf);
   }
 
+  /* This is one of the three final chunks in the file. If the file is too big,
+   * other chunks are removed. So we must only add one chunk here. */
   smartlist_add_strdup(chunks, "router-signature\n");
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
   while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
     /* So long as there are at least two chunks (one for the initial
      * extra-info line and one for the router-signature), we can keep removing
-     * things. */
-    if (smartlist_len(chunks) > 2) {
-      /* We remove the next-to-last element (remember, len-1 is the last
-         element), since we need to keep the router-signature element. */
-      int idx = smartlist_len(chunks) - 2;
+     * things. If emit_ed_sigs is true, we also keep 2 additional chunks at the
+     * end for the ed25519 signature. */
+    const int required_chunks = emit_ed_sigs ? 4 : 2;
+    if (smartlist_len(chunks) > required_chunks) {
+      /* We remove the next-to-last or 4th-last element (remember, len-1 is the
+       * last element), since we need to keep the router-signature elements. */
+      int idx = smartlist_len(chunks) - required_chunks;
       char *e = smartlist_get(chunks, idx);
       smartlist_del_keeporder(chunks, idx);
       log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
@@ -3004,15 +3404,10 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     }
   }
 
-  memset(sig, 0, sizeof(sig));
-  if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 ||
-      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
-                                     ident_key) < 0) {
-    log_warn(LD_BUG, "Could not append signature to extra-info "
-                     "descriptor.");
+  rv = extrainfo_dump_to_string_rsa_sig_helper(chunks, ident_key, s);
+  if (rv < 0)
     goto err;
-  }
-  smartlist_add_strdup(chunks, sig);
+
   tor_free(s);
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -3048,9 +3443,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk));
   smartlist_free(chunks);
   tor_free(s_dup);
-  tor_free(ed_cert_line);
   extrainfo_free(ei_tmp);
-  tor_free(bandwidth_usage);
 
   return result;
 }
@@ -3060,9 +3453,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
 void
 router_reset_warnings(void)
 {
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_clear(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_clear(warned_family);
   }
 }
 
@@ -3075,6 +3468,10 @@ router_free_all(void)
   crypto_pk_free(server_identitykey);
   crypto_pk_free(client_identitykey);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
   tor_mutex_free(key_lock);
   routerinfo_free(desc_routerinfo);
   extrainfo_free(desc_extrainfo);
@@ -3086,11 +3483,12 @@ router_free_all(void)
   memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key));
   memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_free(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_free(warned_family);
   }
 }
+
 /* From the given RSA key object, convert it to ASN-1 encoded format and set
  * the newly allocated object in onion_pkey_out. The length of the key is set
  * in onion_pkey_len_out. */
diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h
index bd6a8a012e..55b9ef9e68 100644
--- a/src/feature/relay/router.h
+++ b/src/feature/relay/router.h
@@ -23,11 +23,12 @@ struct ed25519_keypair_t;
 #define TOR_ROUTERINFO_ERROR_DIGEST_FAILED   (-4)
 #define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5)
 #define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6)
+#define TOR_ROUTERINFO_ERROR_INTERNAL_BUG    (-7)
 
-crypto_pk_t *get_onion_key(void);
+MOCK_DECL(crypto_pk_t *,get_onion_key,(void));
 time_t get_onion_key_set_at(void);
 void set_server_identity_key(crypto_pk_t *k);
-crypto_pk_t *get_server_identity_key(void);
+MOCK_DECL(crypto_pk_t *,get_server_identity_key,(void));
 int server_identity_key_is_set(void);
 void set_client_identity_key(crypto_pk_t *k);
 crypto_pk_t *get_tlsclient_identity_key(void);
@@ -114,9 +115,27 @@ void router_reset_reachability(void);
 void router_free_all(void);
 
 #ifdef ROUTER_PRIVATE
-/* Used only by router.c and test.c */
+/* Used only by router.c and the unit tests */
 STATIC void get_platform_str(char *platform, size_t len);
 STATIC int router_write_fingerprint(int hashed);
-#endif
+STATIC smartlist_t *get_my_declared_family(const or_options_t *options);
+
+#ifdef TOR_UNIT_TESTS
+extern time_t desc_clean_since;
+extern const char *desc_dirty_reason;
+void set_server_identity_key_digest_testing(const uint8_t *digest);
+MOCK_DECL(STATIC const struct curve25519_keypair_t *,
+                                       get_current_curve25519_keypair,(void));
+
+MOCK_DECL(STATIC int,
+              router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out));
+STATIC extrainfo_t *router_build_fresh_signed_extrainfo(
+                                                      const routerinfo_t *ri);
+STATIC void router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                                    const extrainfo_t *ei);
+STATIC int router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(ROUTER_PRIVATE) */
 
 #endif /* !defined(TOR_ROUTER_H) */
diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c
index f639fc91e7..a9190b2e13 100644
--- a/src/feature/relay/routerkeys.c
+++ b/src/feature/relay/routerkeys.c
@@ -226,7 +226,7 @@ load_ed_keys(const or_options_t *options, time_t now)
         tor_free(fname);
       }
     }
-    if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
+    if (safe_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
       sign_signing_key_with_id = NULL;
     else
       sign_signing_key_with_id = id;
@@ -631,14 +631,14 @@ get_master_identity_keypair(void)
 }
 #endif /* defined(TOR_UNIT_TESTS) */
 
-const ed25519_keypair_t *
-get_master_signing_keypair(void)
+MOCK_IMPL(const ed25519_keypair_t *,
+get_master_signing_keypair,(void))
 {
   return master_signing_key;
 }
 
-const struct tor_cert_st *
-get_master_signing_key_cert(void)
+MOCK_IMPL(const struct tor_cert_st *,
+get_master_signing_key_cert,(void))
 {
   return signing_key_cert;
 }
@@ -706,6 +706,8 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
 
   *len_out = 0;
   if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    log_info(LD_OR, "crypto_pk_get_digest failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
   }
   memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
@@ -713,8 +715,12 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
   int r = crypto_pk_private_sign(onion_key,
                                (char*)signature, sizeof(signature),
                                (const char*)signed_data, sizeof(signed_data));
-  if (r < 0)
+  if (r < 0) {
+    /* It's probably missing the private key */
+    log_info(LD_OR, "crypto_pk_private_sign failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
+  }
 
   *len_out = r;
 
diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h
index 0badd34191..cde07b52c3 100644
--- a/src/feature/relay/routerkeys.h
+++ b/src/feature/relay/routerkeys.h
@@ -7,8 +7,8 @@
 #include "lib/crypt_ops/crypto_ed25519.h"
 
 const ed25519_public_key_t *get_master_identity_key(void);
-const ed25519_keypair_t *get_master_signing_keypair(void);
-const struct tor_cert_st *get_master_signing_key_cert(void);
+MOCK_DECL(const ed25519_keypair_t *, get_master_signing_keypair,(void));
+MOCK_DECL(const struct tor_cert_st *, get_master_signing_key_cert,(void));
 
 const ed25519_keypair_t *get_current_auth_keypair(void);
 const struct tor_cert_st *get_current_link_cert_cert(void);
diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c
index 064eea6c46..f8b54ff45d 100644
--- a/src/feature/relay/selftest.c
+++ b/src/feature/relay/selftest.c
@@ -26,7 +26,7 @@
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/authority_cert_st.h"
@@ -35,6 +35,7 @@
 #include "feature/nodelist/routerlist.h" // but...
 #include "feature/nodelist/routerset.h"
 #include "feature/nodelist/torcert.h"
+#include "feature/relay/relay_periodic.h"
 #include "feature/relay/router.h"
 #include "feature/relay/selftest.h"
 
diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h
index a80ec8936e..aea77ec791 100644
--- a/src/feature/relay/selftest.h
+++ b/src/feature/relay/selftest.h
@@ -21,4 +21,4 @@ void router_orport_found_reachable(void);
 void router_dirport_found_reachable(void);
 void router_perform_bandwidth_test(int num_circs, time_t now);
 
-#endif
+#endif /* !defined(TOR_SELFTEST_H) */
diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h
index 7bd4f2fe8c..51a1798fcb 100644
--- a/src/feature/rend/rend_authorized_client_st.h
+++ b/src/feature/rend/rend_authorized_client_st.h
@@ -14,5 +14,5 @@ struct rend_authorized_client_t {
   crypto_pk_t *client_key;
 };
 
-#endif
+#endif /* !defined(REND_AUTHORIZED_CLIENT_ST_H) */
 
diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
index 05ff145d53..bd8a60f0d9 100644
--- a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
+++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
@@ -13,5 +13,5 @@ struct rend_encoded_v2_service_descriptor_t {
   char *desc_str; /**< Descriptor string. */
 };
 
-#endif
+#endif /* !defined(REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H) */
 
diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h
index de6987e569..4882b62752 100644
--- a/src/feature/rend/rend_intro_point_st.h
+++ b/src/feature/rend/rend_intro_point_st.h
@@ -73,4 +73,4 @@ struct rend_intro_point_t {
   unsigned int circuit_established:1;
 };
 
-#endif
+#endif /* !defined(REND_INTRO_POINT_ST_H) */
diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h
index aeb3178064..ff7627ce96 100644
--- a/src/feature/rend/rend_service_descriptor_st.h
+++ b/src/feature/rend/rend_service_descriptor_st.h
@@ -30,5 +30,5 @@ struct rend_service_descriptor_t {
   smartlist_t *successful_uploads;
 };
 
-#endif
+#endif /* !defined(REND_SERVICE_DESCRIPTOR_ST_H) */
 
diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c
index 1c3badaff3..c3f86d8c82 100644
--- a/src/feature/rend/rendcache.c
+++ b/src/feature/rend/rendcache.c
@@ -19,6 +19,8 @@
 #include "feature/rend/rend_intro_point_st.h"
 #include "feature/rend/rend_service_descriptor_st.h"
 
+#include "lib/ctime/di_ops.h"
+
 /** Map from service id (as generated by rend_get_service_id) to
  * rend_cache_entry_t. */
 STATIC strmap_t *rend_cache = NULL;
@@ -45,7 +47,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL;
  * looked up in this cache and if present, it is discarded from the fetched
  * descriptor. At the end, all IP(s) in the cache, for a specific service
  * ID, that were NOT present in the descriptor are removed from this cache.
- * Which means that if at least one IP was not in this cache, thus usuable,
+ * Which means that if at least one IP was not in this cache, thus usable,
  * it's considered a new descriptor so we keep it. Else, if all IPs were in
  * this cache, we discard the descriptor as it's considered unusable.
  *
@@ -593,10 +595,10 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc)
   char desc_id_digest[DIGEST_LEN];
   tor_assert(rend_cache_v2_dir);
   if (base32_decode(desc_id_digest, DIGEST_LEN,
-                    desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) {
+                    desc_id, REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Rejecting v2 rendezvous descriptor request -- descriptor ID "
-           "contains illegal characters: %s",
+           "has wrong length or illegal characters: %s",
            safe_str(desc_id));
     return -1;
   }
@@ -854,7 +856,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     *entry = NULL;
   }
   if (base32_decode(want_desc_id, sizeof(want_desc_id),
-                    desc_id_base32, strlen(desc_id_base32)) != 0) {
+                    desc_id_base32, strlen(desc_id_base32)) !=
+      sizeof(want_desc_id)) {
     log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.",
              escaped_safe_str_client(desc_id_base32));
     goto err;
@@ -888,8 +891,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   if (intro_content && intro_size > 0) {
     int n_intro_points;
     if (rend_data->auth_type != REND_NO_AUTH &&
-        !tor_mem_is_zero(rend_data->descriptor_cookie,
-                         sizeof(rend_data->descriptor_cookie))) {
+        !safe_mem_is_zero(rend_data->descriptor_cookie,
+                          sizeof(rend_data->descriptor_cookie))) {
       char *ipos_decrypted = NULL;
       size_t ipos_decrypted_size;
       if (rend_decrypt_introduction_points(&ipos_decrypted,
@@ -1005,4 +1008,3 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   tor_free(intro_content);
   return retval;
 }
-
diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c
index cde954da95..2e119d7c99 100644
--- a/src/feature/rend/rendclient.c
+++ b/src/feature/rend/rendclient.c
@@ -17,7 +17,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/relay.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
@@ -119,7 +119,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   char tmp[RELAY_PAYLOAD_SIZE];
   rend_cache_entry_t *entry = NULL;
   crypt_path_t *cpath;
-  off_t dh_offset;
+  ptrdiff_t dh_offset;
   crypto_pk_t *intro_key = NULL;
   int status = 0;
   const char *onion_address;
@@ -403,14 +403,23 @@ rend_client_introduction_acked(origin_circuit_t *circ,
     } else {
       log_info(LD_REND,"...Found no rend circ. Dropping on the floor.");
     }
+    /* Save the rend data digest to a temporary object so that we don't access
+     * it after we mark the circuit for close. */
+    const uint8_t *rend_digest_tmp = NULL;
+    size_t digest_len;
+    uint8_t *cached_rend_digest = NULL;
+    rend_digest_tmp = rend_data_get_pk_digest(circ->rend_data, &digest_len);
+    cached_rend_digest = tor_malloc_zero(digest_len);
+    memcpy(cached_rend_digest, rend_digest_tmp, digest_len);
+
     /* close the circuit: we won't need it anymore. */
     circuit_change_purpose(TO_CIRCUIT(circ),
                            CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
 
     /* close any other intros launched in parallel */
-    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
-                                                           NULL));
+    rend_client_close_other_intros(cached_rend_digest);
+    tor_free(cached_rend_digest); /* free the temporary digest */
   } else {
     /* It's a NAK; the introduction point didn't relay our request. */
     circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
@@ -469,16 +478,19 @@ directory_get_from_hs_dir(const char *desc_id,
 
   /* Automatically pick an hs dir if none given. */
   if (!rs_hsdir) {
+    bool rate_limited = false;
+
     /* Determine responsible dirs. Even if we can't get all we want, work with
      * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */
     smartlist_t *responsible_dirs = smartlist_new();
     hid_serv_get_responsible_directories(responsible_dirs, desc_id);
 
-    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32);
+    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32, &rate_limited);
     if (!hs_dir) {
       /* No suitable hs dir can be found, stop right now. */
-      control_event_hsv2_descriptor_failed(rend_query, NULL,
-                                           "QUERY_NO_HSDIR");
+      const char *query_response = (rate_limited) ? "QUERY_RATE_LIMITED" :
+                                                    "QUERY_NO_HSDIR";
+      control_event_hsv2_descriptor_failed(rend_query, NULL, query_response);
       control_event_hs_descriptor_content(rend_data_get_address(rend_query),
                                           desc_id_base32, NULL, NULL);
       return 0;
diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c
index de48af795f..0a606a9f02 100644
--- a/src/feature/rend/rendcommon.c
+++ b/src/feature/rend/rendcommon.c
@@ -15,7 +15,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/hs/hs_client.h"
@@ -171,9 +171,10 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id,
   }
   /* Convert service ID to binary. */
   if (base32_decode(service_id_binary, REND_SERVICE_ID_LEN,
-                    service_id, REND_SERVICE_ID_LEN_BASE32) < 0) {
+                    service_id, REND_SERVICE_ID_LEN_BASE32) !=
+      REND_SERVICE_ID_LEN) {
     log_warn(LD_REND, "Could not compute v2 descriptor ID: "
-                      "Illegal characters in service ID: %s",
+                      "Illegal characters or wrong length for service ID: %s",
              safe_str_client(service_id));
     return -1;
   }
@@ -785,39 +786,39 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
   switch (command) {
     case RELAY_COMMAND_ESTABLISH_INTRO:
       if (or_circ)
-        r = hs_intro_received_establish_intro(or_circ,payload,length);
+        r = hs_intro_received_establish_intro(or_circ, payload, length);
       break;
     case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
       if (or_circ)
-        r = rend_mid_establish_rendezvous(or_circ,payload,length);
+        r = rend_mid_establish_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE1:
       if (or_circ)
-        r = hs_intro_received_introduce1(or_circ,payload,length);
+        r = hs_intro_received_introduce1(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE2:
       if (origin_circ)
-        r = hs_service_receive_introduce2(origin_circ,payload,length);
+        r = hs_service_receive_introduce2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE_ACK:
       if (origin_circ)
-        r = hs_client_receive_introduce_ack(origin_circ,payload,length);
+        r = hs_client_receive_introduce_ack(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS1:
       if (or_circ)
-        r = rend_mid_rendezvous(or_circ,payload,length);
+        r = rend_mid_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS2:
       if (origin_circ)
-        r = hs_client_receive_rendezvous2(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRO_ESTABLISHED:
       if (origin_circ)
-        r = hs_service_receive_intro_established(origin_circ,payload,length);
+        r = hs_service_receive_intro_established(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       if (origin_circ)
-        r = hs_client_receive_rendezvous_acked(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous_acked(origin_circ, payload, length);
       break;
     default:
       tor_fragile_assert();
diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c
index 3ba48f8858..06471b2a7f 100644
--- a/src/feature/rend/rendmid.c
+++ b/src/feature/rend/rendmid.c
@@ -18,6 +18,7 @@
 #include "feature/rend/rendmid.h"
 #include "feature/stats/rephist.h"
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 
 #include "core/or/or_circuit_st.h"
@@ -117,6 +118,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
   /* Now, set up this circuit. */
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
   hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
+  hs_dos_setup_default_intro2_defenses(circ);
 
   log_info(LD_REND,
            "Established introduction point on circuit %u for service %s",
@@ -181,6 +183,14 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(intro_circ)) {
+    log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v2 cell due to DoS "
+                          "limitations. Sending NACK to client.");
+    goto err;
+  }
+
   log_info(LD_REND,
            "Sending introduction request for service %s "
            "from circ %u to circ %u",
@@ -237,8 +247,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
-  /* Check if we are configured to accept established rendezvous cells from
-   * client or in other words Tor2Web clients. */
+  /* Check if we are configured to defend ourselves from clients that
+   * attempt to establish rendezvous points directly to us. */
   if (channel_is_client(circ->p_chan) &&
       dos_should_refuse_single_hop_client()) {
     /* Note it down for the heartbeat log purposes. */
diff --git a/src/feature/rend/rendparse.c b/src/feature/rend/rendparse.c
index abd0feb448..a98cb3ad88 100644
--- a/src/feature/rend/rendparse.c
+++ b/src/feature/rend/rendparse.c
@@ -143,8 +143,9 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     goto err;
   }
   if (base32_decode(desc_id_out, DIGEST_LEN,
-                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
-    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
+                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Descriptor ID has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -174,8 +175,10 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
     goto err;
   }
-  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
-    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
+  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) !=
+      DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Secret ID part has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -429,8 +432,10 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed,
     /* Parse identifier. */
     tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
     if (base32_decode(info->identity_digest, DIGEST_LEN,
-                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
-      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
+                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) !=
+        DIGEST_LEN) {
+      log_warn(LD_REND,
+               "Identity digest has wrong length or illegal characters: %s",
                tok->args[0]);
       rend_intro_point_free(intro);
       goto err;
diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h
index 0cef931e90..b1ccce9b6c 100644
--- a/src/feature/rend/rendparse.h
+++ b/src/feature/rend/rendparse.h
@@ -29,4 +29,4 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
                                    size_t intro_points_encoded_size);
 int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
 
-#endif
+#endif /* !defined(TOR_REND_PARSE_H) */
diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c
index c96ecec308..e0cf06b9df 100644
--- a/src/feature/rend/rendservice.c
+++ b/src/feature/rend/rendservice.c
@@ -18,8 +18,9 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_common.h"
@@ -2126,7 +2127,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
      *
      * We only use a one-hop path on the first attempt. If the first attempt
      * fails, we use a 3-hop path for reachability / reliability.
-     * See the comment in rend_service_relauch_rendezvous() for details. */
+     * See the comment in rend_service_relaunch_rendezvous() for details. */
     if (rend_service_use_direct_connection(options, rp) && i == 0) {
       flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL;
     }
@@ -2167,7 +2168,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   cpath->rend_dh_handshake_state = dh;
   dh = NULL;
-  if (circuit_init_cpath_crypto(cpath,
+  if (cpath_init_circuit_crypto(cpath,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 1, 0)<0)
     goto err;
@@ -3016,6 +3017,10 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
 {
   origin_circuit_t *newcirc;
   cpath_build_state_t *newstate, *oldstate;
+  const char *rend_pk_digest;
+  rend_service_t *service = NULL;
+
+  int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
 
   tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
   oldstate = oldcirc->build_state;
@@ -3030,13 +3035,31 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
   log_info(LD_REND,"Reattempting rendezvous circuit to '%s'",
            safe_str(extend_info_describe(oldstate->chosen_exit)));
 
+  /* Look up the service. */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(oldcirc->rend_data, NULL);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+
+  if (!service) {
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                  rend_pk_digest, REND_SERVICE_ID_LEN);
+
+    log_warn(LD_BUG, "Internal error: Trying to relaunch a rendezvous circ "
+                     "for an unrecognized service %s.",
+                     safe_str_client(serviceid));
+    return;
+  }
+
+  if (hs_service_requires_uptime_circ(service->ports)) {
+    flags |= CIRCLAUNCH_NEED_UPTIME;
+  }
+
   /* You'd think Single Onion Services would want to retry the rendezvous
    * using a direct connection. But if it's blocked by a firewall, or the
    * service is IPv6-only, or the rend point avoiding becoming a one-hop
    * proxy, we need a 3-hop connection. */
   newcirc = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
-                            oldstate->chosen_exit,
-                            CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+                            oldstate->chosen_exit, flags);
 
   if (!newcirc) {
     log_warn(LD_REND,"Couldn't relaunch rendezvous circuit to '%s'.",
@@ -3536,7 +3559,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   hop->package_window = circuit_initial_package_window();
   hop->deliver_window = CIRCWINDOW_START;
 
-  onion_append_to_cpath(&circuit->cpath, hop);
+  cpath_extend_linked_list(&circuit->cpath, hop);
   circuit->build_state->pending_final_cpath = NULL; /* prevent double-free */
 
   /* Change the circuit purpose. */
@@ -3976,7 +3999,7 @@ remove_invalid_intro_points(rend_service_t *service,
        * accounted for when considiring uploading a descriptor. */
       intro->circuit_established = 0;
 
-      /* Node is gone or we've reached our maximum circuit creationg retry
+      /* Node is gone or we've reached our maximum circuit creation retry
        * count, clean up everything, we'll find a new one. */
       if (node == NULL ||
           intro->circuit_retries >= MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -4216,6 +4239,7 @@ rend_consider_services_intro_points(time_t now)
        * directly ourselves. */
       intro->extend_info = extend_info_from_node(node, 0);
       if (BUG(intro->extend_info == NULL)) {
+        tor_free(intro);
         break;
       }
       intro->intro_key = crypto_pk_new();
diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c
index a54b589eb6..6fb21f4f79 100644
--- a/src/feature/stats/geoip_stats.c
+++ b/src/feature/stats/geoip_stats.c
@@ -30,9 +30,9 @@
 #include "core/or/or.h"
 
 #include "ht.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/dnsserv.h"
 #include "core/or/dos.h"
 #include "lib/geoip/geoip.h"
diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h
index 272344da2f..45b206c23a 100644
--- a/src/feature/stats/predict_ports.h
+++ b/src/feature/stats/predict_ports.h
@@ -27,4 +27,4 @@ int rep_hist_circbuilding_dormant(time_t now);
 int predicted_ports_prediction_time_remaining(time_t now);
 void predicted_ports_free_all(void);
 
-#endif
+#endif /* !defined(TOR_PREDICT_PORTS_H) */
diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h
index 3accc8c610..0d72946382 100644
--- a/src/feature/stats/rephist.h
+++ b/src/feature/stats/rephist.h
@@ -103,7 +103,7 @@ typedef struct bw_array_t bw_array_t;
 STATIC uint64_t find_largest_max(bw_array_t *b);
 STATIC void commit_max(bw_array_t *b);
 STATIC void advance_obs(bw_array_t *b);
-#endif
+#endif /* defined(REPHIST_PRIVATE) */
 
 /**
  * Represents the type of a cell for padding accounting
diff --git a/src/include.am b/src/include.am
index d2f83da814..065bdc31cb 100644
--- a/src/include.am
+++ b/src/include.am
@@ -1,12 +1,16 @@
 include src/ext/include.am
 include src/lib/arch/include.am
+include src/lib/buf/include.am
 include src/lib/err/include.am
 include src/lib/cc/include.am
 include src/lib/ctime/include.am
 include src/lib/compress/include.am
+include src/lib/conf/include.am
+include src/lib/confmgt/include.am
 include src/lib/container/include.am
 include src/lib/crypt_ops/include.am
 include src/lib/defs/include.am
+include src/lib/dispatch/include.am
 include src/lib/encoding/include.am
 include src/lib/evloop/include.am
 include src/lib/fdio/include.am
@@ -23,8 +27,10 @@ include src/lib/malloc/include.am
 include src/lib/net/include.am
 include src/lib/osinfo/include.am
 include src/lib/process/include.am
+include src/lib/pubsub/include.am
 include src/lib/sandbox/include.am
 include src/lib/string/include.am
+include src/lib/subsys/include.am
 include src/lib/smartlist_core/include.am
 include src/lib/term/include.am
 include src/lib/testsupport/include.am
@@ -32,6 +38,7 @@ include src/lib/thread/include.am
 include src/lib/time/include.am
 include src/lib/tls/include.am
 include src/lib/trace/include.am
+include src/lib/version/include.am
 include src/lib/wallclock/include.am
 include src/trunnel/include.am
 
diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h
index fa82241b28..4756ca2beb 100644
--- a/src/lib/arch/bytes.h
+++ b/src/lib/arch/bytes.h
@@ -129,7 +129,7 @@ tor_ntohll(uint64_t a)
 {
   return a;
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
 static inline uint16_t
 tor_htons(uint16_t a)
 {
@@ -177,6 +177,6 @@ tor_ntohll(uint64_t a)
 {
   return tor_htonll(a);
 }
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
-#endif
+#endif /* !defined(TOR_BYTES_H) */
diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am
index f92ee9222f..c5926c6330 100644
--- a/src/lib/arch/include.am
+++ b/src/lib/arch/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/arch/bytes.h
diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include
new file mode 100644
index 0000000000..c4be73bce2
--- /dev/null
+++ b/src/lib/buf/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/buf/*.h
+lib/cc/*.h
+lib/ctime/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
+lib/log/*.h
+lib/string/*.h
+lib/time/*.h
diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c
index fe4cf7c385..ace5bdc4a4 100644
--- a/src/lib/container/buffers.c
+++ b/src/lib/buf/buffers.c
@@ -25,7 +25,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -158,7 +158,7 @@ chunk_new_with_alloc_size(size_t alloc)
 static inline chunk_t *
 chunk_grow(chunk_t *chunk, size_t sz)
 {
-  off_t offset;
+  ptrdiff_t offset;
   const size_t memlen_orig = chunk->memlen;
   const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
   const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
@@ -440,7 +440,7 @@ chunk_copy(const chunk_t *in_chunk)
 #endif
   newch->next = NULL;
   if (in_chunk->data) {
-    off_t offset = in_chunk->data - in_chunk->mem;
+    ptrdiff_t offset = in_chunk->data - in_chunk->mem;
     newch->data = newch->mem + offset;
   }
   return newch;
@@ -712,7 +712,8 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in)
 /** Internal structure: represents a position in a buffer. */
 typedef struct buf_pos_t {
   const chunk_t *chunk; /**< Which chunk are we pointing to? */
-  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  ptrdiff_t pos;/**< Which character inside the chunk's data are we pointing
+                 * to? */
   size_t chunk_pos; /**< Total length of all previous chunks. */
 } buf_pos_t;
 
@@ -728,15 +729,15 @@ buf_pos_init(const buf_t *buf, buf_pos_t *out)
 /** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
  * position of <b>out</b>, or later.  Return -1 if no instances are found;
  * otherwise returns the absolute position of the character. */
-static off_t
+static ptrdiff_t
 buf_find_pos_of_char(char ch, buf_pos_t *out)
 {
   const chunk_t *chunk;
-  int pos;
+  ptrdiff_t pos;
   tor_assert(out);
   if (out->chunk) {
     if (out->chunk->datalen) {
-      tor_assert(out->pos < (off_t)out->chunk->datalen);
+      tor_assert(out->pos < (ptrdiff_t)out->chunk->datalen);
     } else {
       tor_assert(out->pos == 0);
     }
@@ -764,7 +765,7 @@ buf_pos_inc(buf_pos_t *pos)
 {
   tor_assert(pos->pos < INT_MAX - 1);
   ++pos->pos;
-  if (pos->pos == (off_t)pos->chunk->datalen) {
+  if (pos->pos == (ptrdiff_t)pos->chunk->datalen) {
     if (!pos->chunk->next)
       return -1;
     pos->chunk_pos += pos->chunk->datalen;
@@ -838,11 +839,11 @@ buf_peek_startswith(const buf_t *buf, const char *cmd)
 
 /** Return the index within <b>buf</b> at which <b>ch</b> first appears,
  * or -1 if <b>ch</b> does not appear on buf. */
-static off_t
+static ptrdiff_t
 buf_find_offset_of_char(buf_t *buf, char ch)
 {
   chunk_t *chunk;
-  off_t offset = 0;
+  ptrdiff_t offset = 0;
   tor_assert(buf->datalen < INT_MAX);
   for (chunk = buf->head; chunk; chunk = chunk->next) {
     char *cp = memchr(chunk->data, ch, chunk->datalen);
@@ -865,7 +866,7 @@ int
 buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
 {
   size_t sz;
-  off_t offset;
+  ptrdiff_t offset;
 
   if (!buf->head)
     return 0;
diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h
index c103b93a82..c103b93a82 100644
--- a/src/lib/container/buffers.h
+++ b/src/lib/buf/buffers.h
diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am
new file mode 100644
index 0000000000..27430d1d38
--- /dev/null
+++ b/src/lib/buf/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-buf.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-buf-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_buf_a_SOURCES =			\
+	src/lib/buf/buffers.c
+
+src_lib_libtor_buf_testing_a_SOURCES = \
+	$(src_lib_libtor_buf_a_SOURCES)
+src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/buf/buffers.h
diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include
index 2b06e8519c..fa1478ce46 100644
--- a/src/lib/cc/.may_include
+++ b/src/lib/cc/.may_include
@@ -1 +1,2 @@
 orconfig.h
+lib/cc/*.h
\ No newline at end of file
diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h
index fbe6a38f1f..d9ccabb2b3 100644
--- a/src/lib/cc/compat_compiler.h
+++ b/src/lib/cc/compat_compiler.h
@@ -88,7 +88,7 @@
 #    define ENABLE_GCC_WARNING(warningopt) \
          PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
 #endif /* defined(__clang__) || GCC_VERSION >= 406 */
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 /* not gcc at all */
 # define DISABLE_GCC_WARNING(warning)
 # define ENABLE_GCC_WARNING(warning)
@@ -201,7 +201,7 @@
  * structure <b>st</b>.  Example:
  * <pre>
  *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = offsetof(struct a, bar);
+ *   ptrdiff_t bar_offset = offsetof(struct a, bar);
  *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
  *   *bar_p = 3;
  * </pre>
@@ -223,4 +223,16 @@
 /** Macro: Yields the number of elements in array x. */
 #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
 
-#endif /* !defined(TOR_COMPAT_H) */
+/**
+ * "Eat" a semicolon that somebody puts at the end of a top-level macro.
+ *
+ * Frequently, we want to declare a macro that people will use at file scope,
+ * and we want to allow people to put a semicolon after the macro.
+ *
+ * This declaration of a struct can be repeated any number of times, and takes
+ * a trailing semicolon afterwards.
+ **/
+#define EAT_SEMICOLON                                   \
+  struct dummy_semicolon_eater__
+
+#endif /* !defined(TOR_COMPAT_COMPILER_H) */
diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h
new file mode 100644
index 0000000000..bedf0b83a6
--- /dev/null
+++ b/src/lib/cc/ctassert.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ctassert.h
+ *
+ * \brief Compile-time assertions: CTASSERT(expression).
+ */
+
+#ifndef TOR_CTASSERT_H
+#define TOR_CTASSERT_H
+
+#include "lib/cc/compat_compiler.h"
+
+/**
+ * CTASSERT(expression)
+ *
+ *       Trigger a compiler error if expression is false.
+ */
+#if __STDC_VERSION__ >= 201112L
+
+/* If C11 is available, just use _Static_assert.  */
+#define CTASSERT(x) _Static_assert((x), #x)
+
+#else /* !(__STDC_VERSION__ >= 201112L) */
+
+/*
+ * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__
+ * and __LINE__, or just __LINE__, with an intermediate preprocessor
+ * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the
+ * expansions together into a unique name.
+ *
+ * We use this name as a typedef of an array type with a positive
+ * length if the assertion is true, and a negative length of the
+ * assertion is false, which is invalid and hence triggers a compiler
+ * error.
+ */
+#if defined(__COUNTER__)
+#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__)
+#elif defined(__INCLUDE_LEVEL__)
+#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__)
+#else
+/* hope it's unique enough */
+#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__)
+#endif /* defined(__COUNTER__) || ... */
+
+#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b)
+#define CTASSERT_DECL(x, a, b) \
+  typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED
+
+#endif /* __STDC_VERSION__ >= 201112L */
+
+#endif /* !defined(TOR_CTASSERT_H) */
diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am
index 2ae90f97dd..1aa722dd82 100644
--- a/src/lib/cc/include.am
+++ b/src/lib/cc/include.am
@@ -1,4 +1,6 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/cc/compat_compiler.h \
+	src/lib/cc/ctassert.h \
 	src/lib/cc/torint.h
diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h
index c9b2d329f2..94b79d30a1 100644
--- a/src/lib/cc/torint.h
+++ b/src/lib/cc/torint.h
@@ -96,9 +96,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIuSZ PRIu32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIuSZ "zu"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef _WIN32
 #  ifdef _WIN64
@@ -106,9 +106,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIdSZ PRId32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIdSZ "zd"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef SSIZE_MAX
 #if (SIZEOF_SIZE_T == 4)
@@ -125,4 +125,13 @@ typedef int32_t ssize_t;
 /** Any size_t larger than this amount is likely to be an underflow. */
 #define SIZE_T_CEILING  ((size_t)(SSIZE_MAX-16))
 
+#if SIZEOF_INT > SIZEOF_VOID_P
+#error "sizeof(int) > sizeof(void *) - Tor cannot be built on this platform!"
+#endif
+
+#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P
+#error "sizeof(unsigned int) > sizeof(void *) - Tor cannot be built on this \
+platform!"
+#endif
+
 #endif /* !defined(TOR_TORINT_H) */
diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include
index 68fe9f1c54..6cd80086e6 100644
--- a/src/lib/compress/.may_include
+++ b/src/lib/compress/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/compress/*.h
 lib/container/*.h
@@ -8,5 +9,6 @@ lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c
index 95fd73bb32..51591410a2 100644
--- a/src/lib/compress/compress.c
+++ b/src/lib/compress/compress.c
@@ -29,10 +29,12 @@
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_none.h"
+#include "lib/compress/compress_sys.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
 #include "lib/intmath/cmp.h"
 #include "lib/malloc/malloc.h"
+#include "lib/subsys/subsys.h"
 #include "lib/thread/threads.h"
 
 /** Total number of bytes allocated for compression state overhead. */
@@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state)
 }
 
 /** Initialize all compression modules. */
-void
+int
 tor_compress_init(void)
 {
   atomic_counter_init(&total_compress_allocation);
@@ -668,6 +670,8 @@ tor_compress_init(void)
   tor_zlib_init();
   tor_lzma_init();
   tor_zstd_init();
+
+  return 0;
 }
 
 /** Warn if we had any problems while setting up our compression libraries.
@@ -677,5 +681,20 @@ tor_compress_init(void)
 void
 tor_compress_log_init_warnings(void)
 {
+  // XXXX can we move this into tor_compress_init() after all?  log.c queues
+  // XXXX log messages at startup.
   tor_zstd_warn_if_version_mismatched();
 }
+
+static int
+subsys_compress_initialize(void)
+{
+  return tor_compress_init();
+}
+
+const subsys_fns_t sys_compress = {
+  .name = "compress",
+  .supported = true,
+  .level = -70,
+  .initialize = subsys_compress_initialize,
+};
diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h
index 5f16a2ab27..8cea4ead60 100644
--- a/src/lib/compress/compress.h
+++ b/src/lib/compress/compress.h
@@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state);
 
 size_t tor_compress_state_size(const tor_compress_state_t *state);
 
-void tor_compress_init(void);
+int tor_compress_init(void);
 void tor_compress_log_init_warnings(void);
 
 struct buf_t;
diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c
index 198128b261..2e704466f2 100644
--- a/src/lib/compress/compress_buf.c
+++ b/src/lib/compress/compress_buf.c
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/cc/compat_compiler.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/compress/compress.h"
 #include "lib/log/util_bug.h"
 
diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c
index 2dab37e433..915f4949ae 100644
--- a/src/lib/compress/compress_lzma.c
+++ b/src/lib/compress/compress_lzma.c
@@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress,
   tor_free(result);
   return NULL;
  /* LCOV_EXCL_STOP */
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)compress;
   (void)method;
   (void)level;
@@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state,
                lzma_error_str(retval));
       return TOR_COMPRESS_ERROR;
   }
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)state;
   (void)out;
   (void)out_len;
diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h
new file mode 100644
index 0000000000..6181072315
--- /dev/null
+++ b/src/lib/compress/compress_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_sys.h
+ * \brief Declare subsystem object for the compress module
+ **/
+
+#ifndef TOR_COMPRESS_SYS_H
+#define TOR_COMPRESS_SYS_H
+
+extern const struct subsys_fns_t sys_compress;
+
+#endif /* !defined(TOR_COMPRESS_SYS_H) */
diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c
index 45d0d4d602..9076665295 100644
--- a/src/lib/compress/compress_zstd.c
+++ b/src/lib/compress/compress_zstd.c
@@ -25,7 +25,7 @@
  * all invocations of zstd's static-only functions in a check to make sure
  * that the compile-time version matches the run-time version. */
 #define ZSTD_STATIC_LINKING_ONLY
-#endif
+#endif /* defined(ENABLE_ZSTD_ADVANCED_APIS) */
 
 #ifdef HAVE_ZSTD
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
@@ -35,7 +35,7 @@ DISABLE_GCC_WARNING(unused-const-variable)
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
 ENABLE_GCC_WARNING(unused-const-variable)
 #endif
-#endif
+#endif /* defined(HAVE_ZSTD) */
 
 /** Total number of bytes allocated for Zstandard state. */
 static atomic_counter_t total_zstd_allocation;
@@ -77,7 +77,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
                version_number / 100 % 100,
                version_number % 100);
 }
-#endif
+#endif /* defined(HAVE_ZSTD) */
 
 #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
 
@@ -93,7 +93,7 @@ tor_zstd_get_version_str(void)
                           ZSTD_versionNumber());
 
   return version_str;
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   return NULL;
 #endif /* defined(HAVE_ZSTD) */
 }
@@ -125,9 +125,9 @@ tor_zstd_can_use_static_apis(void)
   }
 #endif
   return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
-#else
+#else /* !(defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)) */
   return 0;
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) */
 }
 
 /** Internal Zstandard state for incremental compression/decompression.
@@ -237,7 +237,7 @@ tor_zstd_state_size_precalc(int compress, int preset)
 #endif
     }
   }
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) */
   return tor_zstd_state_size_precalc_fake(compress, preset);
 }
 #endif /* defined(HAVE_ZSTD) */
@@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress,
   tor_free(result);
   return NULL;
   // LCOV_EXCL_STOP
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)compress;
   (void)method;
   (void)level;
@@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state,
       return TOR_COMPRESS_OK;
   }
 
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)state;
   (void)out;
   (void)out_len;
@@ -527,7 +527,7 @@ tor_zstd_warn_if_version_mismatched(void)
              "For safety, we'll avoid using advanced zstd functionality.",
              header_version, runtime_version);
   }
-#endif
+#endif /* defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) */
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -538,4 +538,4 @@ tor_zstd_set_static_apis_disabled_for_testing(int disabled)
 {
   static_apis_disable_for_testing = disabled;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am
index 75c9032bd2..60dd447d4e 100644
--- a/src/lib/compress/include.am
+++ b/src/lib/compress/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-compress-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_compress_a_SOURCES =		\
 	src/lib/compress/compress.c		\
 	src/lib/compress/compress_buf.c		\
@@ -18,9 +19,11 @@ src_lib_libtor_compress_testing_a_SOURCES = \
 src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS	+=				\
 	src/lib/compress/compress.h		\
 	src/lib/compress/compress_lzma.h	\
 	src/lib/compress/compress_none.h	\
+	src/lib/compress/compress_sys.h		\
 	src/lib/compress/compress_zlib.h	\
 	src/lib/compress/compress_zstd.h
diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include
new file mode 100644
index 0000000000..629e2f897d
--- /dev/null
+++ b/src/lib/conf/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h
new file mode 100644
index 0000000000..68121891f1
--- /dev/null
+++ b/src/lib/conf/confmacros.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file confmacros.h
+ * @brief Macro definitions for declaring configuration variables
+ **/
+
+#ifndef TOR_LIB_CONF_CONFMACROS_H
+#define TOR_LIB_CONF_CONFMACROS_H
+
+#include "orconfig.h"
+#include "lib/conf/conftesting.h"
+
+/**
+ * Used to indicate the end of an array of configuration variables.
+ **/
+#define END_OF_CONFIG_VARS                                      \
+  { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS }
+
+/**
+ * Declare a config_var_t as a member named <b>membername</b> of the structure
+ * <b>structtype</b>, whose user-visible name is <b>varname</b>, whose
+ * type corresponds to the config_type_t member CONFIG_TYPE_<b>vartype</b>,
+ * and whose initial value is <b>intval</b>.
+ *
+ * Most modules that use this macro should wrap it in a local macro that
+ * sets structtype to the local configuration type.
+ **/
+#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername,      \
+                         varflags, initval)                             \
+  { .member =                                                           \
+    { .name = varname,                                                  \
+      .type = CONFIG_TYPE_ ## vartype,                                  \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+/**
+ * As CONFIG_VAR_XTYPE, but declares a value using an extension type whose
+ * type definition is <b>vartype</b>_type_defn.
+ **/
+#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername,       \
+                        varflags, initval)                              \
+  { .member =                                                           \
+    { .name = varname,                                                \
+      .type = CONFIG_TYPE_EXTENDED,                                     \
+      .type_def = &vartype ## _type_defn,                               \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+#define CONFIG_VAR_OBSOLETE(varname)            \
+  { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE },        \
+    .flags = CFLG_GROUP_OBSOLETE                                        \
+  }
+
+#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */
diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h
new file mode 100644
index 0000000000..f01f52d59e
--- /dev/null
+++ b/src/lib/conf/conftesting.h
@@ -0,0 +1,86 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftesting.h
+ * @brief Macro and type declarations for testing
+ **/
+
+#ifndef TOR_LIB_CONF_CONFTESTING_H
+#define TOR_LIB_CONF_CONFTESTING_H
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Union used when building in test mode typechecking the members of a type
+ * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
+ * it is used. */
+typedef union {
+  char **STRING;
+  char **FILENAME;
+  int *POSINT; /* yes, this is really an int, and not an unsigned int.  For
+                * historical reasons, many configuration values are restricted
+                * to the range [0,INT_MAX], and stored in signed ints.
+                */
+  uint64_t *UINT64;
+  int *INT;
+  int *INTERVAL;
+  int *MSEC_INTERVAL;
+  uint64_t *MEMUNIT;
+  double *DOUBLE;
+  int *BOOL;
+  int *AUTOBOOL;
+  time_t *ISOTIME;
+  struct smartlist_t **CSV;
+  int *CSV_INTERVAL;
+  struct config_line_t **LINELIST;
+  struct config_line_t **LINELIST_S;
+  struct config_line_t **LINELIST_V;
+  // XXXX this doesn't belong at this level of abstraction.
+  struct routerset_t **ROUTERSET;
+} confparse_dummy_values_t;
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/* Macros to define extra members inside config_var_t fields, and at the
+ * end of a list of them.
+ */
+#ifdef TOR_UNIT_TESTS
+/* This is a somewhat magic type-checking macro for users of confparse.c.
+ * It initializes a union member "confparse_dummy_values_t.conftype" with
+ * the address of a static member "tp_dummy.member".   This
+ * will give a compiler warning unless the member field is of the correct
+ * type.
+ *
+ * (This warning is mandatory, because a type mismatch here violates the type
+ * compatibility constraint for simple assignment, and requires a diagnostic,
+ * according to the C spec.)
+ *
+ * For example, suppose you say:
+ *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
+ * Then this macro will evaluate to:
+ *     { .STRING = &or_options_t_dummy.Address }
+ * And since confparse_dummy_values_t.STRING has type "char **", that
+ * expression will create a warning unless or_options_t.Address also
+ * has type "char *".
+ */
+#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
+  { . conftype = &tp ## _dummy . member }
+#define CONF_TEST_MEMBERS(tp, conftype, member) \
+  , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member)
+#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL }
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  static tp tp ## _dummy
+
+#else /* !defined(TOR_UNIT_TESTS) */
+
+#define CONF_TEST_MEMBERS(tp, conftype, member)
+/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  struct tor_semicolon_eater
+#define DUMMY_CONF_TEST_MEMBERS
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */
diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h
new file mode 100644
index 0000000000..274065cff2
--- /dev/null
+++ b/src/lib/conf/conftypes.h
@@ -0,0 +1,202 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftypes.h
+ * @brief Types used to specify configurable options.
+ *
+ * This header defines the types that different modules will use in order to
+ * declare their configuration and state variables, and tell the configuration
+ * management code about those variables.  From the individual module's point
+ * of view, its configuration and state are simply data structures.
+ *
+ * For defining new variable types, see var_type_def_st.h.
+ *
+ * For the code that manipulates variables defined via this module, see
+ * lib/confmgt/, especially typedvar.h and (later) structvar.h.  The
+ * configuration manager is responsible for encoding, decoding, and
+ * maintaining the configuration structures used by the various modules.
+ *
+ * STATUS NOTE: This is a work in process refactoring.  It is not yet possible
+ *   for modules to define their own variables, and much of the configuration
+ *   management code is still in src/app/config/.
+ **/
+
+#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H
+#define TOR_SRC_LIB_CONF_CONFTYPES_H
+
+#include "lib/cc/torint.h"
+#ifdef TOR_UNIT_TESTS
+#include "lib/conf/conftesting.h"
+#endif
+
+#include <stddef.h>
+
+/** Enumeration of types which option values can take */
+typedef enum config_type_t {
+  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
+  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
+  CONFIG_TYPE_POSINT,       /**< A non-negative integer less than MAX_INT */
+  CONFIG_TYPE_INT,          /**< Any integer. */
+  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
+  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
+  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
+                              * units */
+  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
+  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
+  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
+  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
+                             * 1 for true, and -1 for auto  */
+  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
+  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
+                              * optional whitespace. */
+  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
+                              * optional whitespace, representing intervals in
+                              * seconds, with optional units.  We allow
+                              * multiple values here for legacy reasons, but
+                              * ignore every value after the first. */
+  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
+  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
+                             * mixed with other keywords. */
+  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
+                             * context-sensitive config lines when fetching.
+                             */
+  CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
+  /**
+   * Extended type: definition appears in the <b>type_def</b> pointer
+   * of the corresponding struct_member_t.
+   *
+   * For some types, we cannot define them as particular values of this
+   * enumeration, since those types are abstractions defined at a higher level
+   * than this module.  (For example, parsing a routerset_t is higher-level
+   * than this module.)  To handle this, we use CONFIG_TYPE_EXTENDED for those
+   * types, and give a definition for them in the struct_member_t.type_def.
+   **/
+  CONFIG_TYPE_EXTENDED,
+} config_type_t;
+
+/* Forward delcaration for var_type_def_t, for extended types. */
+struct var_type_def_t;
+
+/** Structure to specify a named, typed member within a structure. */
+typedef struct struct_member_t {
+  /** Name of the field. */
+  const char *name;
+  /**
+   * Type of the field, according to the config_type_t enumeration.
+   *
+   * For any type not otherwise listed in config_type_t, this field's value
+   * should be CONFIG_TYPE_EXTENDED.  When it is, the <b>type_def</b> pointer
+   * must be set.
+   **/
+  /*
+   * NOTE: In future refactoring, we might remove this field entirely, along
+   * with its corresponding enumeration.  In that case, we will require that
+   * type_def be set in all cases. If we do, we will also need a new mechanism
+   * to enforce consistency between configuration variable types and their
+   * corresponding structures, since our current design in
+   * lib/conf/conftesting.h won't work any more.
+   */
+  config_type_t type;
+  /**
+   * Pointer to a type definition for the type of this field. Overrides
+   * <b>type</b> if it is not NULL.  Must be set when <b>type</b> is
+   * CONFIG_TYPE_EXTENDED.
+   **/
+  const struct var_type_def_t *type_def;
+  /**
+   * Offset of this field within the structure.  Compute this with
+   * offsetof(structure, fieldname).
+   **/
+  ptrdiff_t offset;
+} struct_member_t;
+
+/**
+ * Structure to describe the location and preferred value of a "magic number"
+ * field within a structure.
+ *
+ * These 'magic numbers' are 32-bit values used to tag objects to make sure
+ * that they have the correct type.
+ */
+typedef struct struct_magic_decl_t {
+  /** The name of the structure */
+  const char *typename;
+  /** A value used to recognize instances of this structure. */
+  uint32_t magic_val;
+  /** The location within the structure at which we expect to find
+   * <b>magic_val</b>. */
+  ptrdiff_t magic_offset;
+} struct_magic_decl_t;
+
+/**
+ * Flag to indicate that an option or type is "undumpable". An
+ * undumpable option is never saved to disk.
+ *
+ * For historical reasons its name is usually is prefixed with __.
+ **/
+#define CFLG_NODUMP    (1u<<0)
+/**
+ * Flag to indicate that an option or type is "unlisted".
+ *
+ * We don't tell the controller about unlisted options when it asks for a
+ * list of them.
+ **/
+#define CFLG_NOLIST (1u<<1)
+/**
+ * Flag to indicate that an option or type is "unsettable".
+ *
+ * An unsettable option can never be set directly by name.
+ **/
+#define CFLG_NOSET (1u<<2)
+/**
+ * Flag to indicate that an option or type does not need to be copied when
+ * copying the structure that contains it.
+ *
+ * (Usually, if an option does not need to be copied, then either it contains
+ * no data, or the data that it does contain is completely contained within
+ * another option.)
+ **/
+#define CFLG_NOCOPY (1u<<3)
+/**
+ * Flag to indicate that an option or type does not need to be compared
+ * when telling the controller about the differences between two
+ * configurations.
+ *
+ * (Usually, if an option does not need to be compared, then either it
+ * contains no data, or the data that it does contain is completely contained
+ * within another option.)
+ **/
+#define CFLG_NOCMP (1u<<4)
+/**
+ * Flag to indicate that an option or type should not be replaced when setting
+ * it.
+ *
+ * For most options, setting them replaces their old value.  For some options,
+ * however, setting them appends to their old value.
+ */
+#define CFLG_NOREPLACE    (1u<<5)
+
+/**
+ * A group of flags that should be set on all obsolete options and types.
+ **/
+#define CFLG_GROUP_OBSOLETE \
+  (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST)
+
+/** A variable allowed in the configuration file or on the command line. */
+typedef struct config_var_t {
+  struct_member_t member; /** A struct member corresponding to this
+                           * variable. */
+  const char *initvalue; /**< String (or null) describing initial value. */
+  uint32_t flags; /**< One or more flags describing special handling for this
+                   * variable */
+#ifdef TOR_UNIT_TESTS
+  /** Used for compiler-magic to typecheck the corresponding field in the
+   * corresponding struct. Only used in unit test mode, at compile-time. */
+  confparse_dummy_values_t var_ptr_dummy;
+#endif
+} config_var_t;
+
+#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */
diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am
new file mode 100644
index 0000000000..cb7126184d
--- /dev/null
+++ b/src/lib/conf/include.am
@@ -0,0 +1,6 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/conf/conftesting.h              \
+	src/lib/conf/conftypes.h                \
+	src/lib/conf/confmacros.h
diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include
new file mode 100644
index 0000000000..2564133917
--- /dev/null
+++ b/src/lib/confmgt/.may_include
@@ -0,0 +1,11 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
+lib/confmgt/*.h
+lib/container/*.h
+lib/encoding/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+ext/*.h
diff --git a/src/lib/confmgt/confparse.c b/src/lib/confmgt/confparse.c
new file mode 100644
index 0000000000..08e562f654
--- /dev/null
+++ b/src/lib/confmgt/confparse.c
@@ -0,0 +1,1239 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confparse.c
+ *
+ * \brief Back-end for parsing and generating key-value files, used to
+ *   implement the torrc file format and the state file.
+ *
+ * This module is used by config.c to parse and encode torrc
+ * configuration files, and by statefile.c to parse and encode the
+ * $DATADIR/state file.
+ *
+ * To use this module, its callers provide an instance of
+ * config_format_t to describe the mappings from a set of configuration
+ * options to a number of fields in a C structure.  With this mapping,
+ * the functions here can convert back and forth between the C structure
+ * specified, and a linked list of key-value pairs.
+ */
+
+#define CONFPARSE_PRIVATE
+#include "orconfig.h"
+#include "lib/confmgt/confparse.h"
+
+#include "lib/confmgt/structvar.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include "ext/siphash.h"
+
+/**
+ * A managed_var_t is an internal wrapper around a config_var_t in
+ * a config_format_t structure.  It is used by config_mgr_t to
+ * keep track of which option goes with which structure. */
+typedef struct managed_var_t {
+  /**
+   * A pointer to the config_var_t for this option.
+   */
+  const config_var_t *cvar;
+  /**
+   * The index of the object in which this option is stored.  It is
+   * IDX_TOPLEVEL to indicate that the object is the top-level object.
+   **/
+  int object_idx;
+} managed_var_t;
+
+static void config_reset(const config_mgr_t *fmt, void *options,
+                         const managed_var_t *var, int use_defaults);
+static void config_mgr_register_fmt(config_mgr_t *mgr,
+                                    const config_format_t *fmt,
+                                    int object_idx);
+
+/** Release all storage held in a managed_var_t. */
+static void
+managed_var_free_(managed_var_t *mv)
+{
+  if (!mv)
+    return;
+  tor_free(mv);
+}
+#define managed_var_free(mv) \
+  FREE_AND_NULL(managed_var_t, managed_var_free_, (mv))
+
+struct config_suite_t {
+  /** A list of configuration objects managed by a given configuration
+   * manager. They are stored in the same order as the config_format_t
+   * objects in the manager's list of subformats. */
+  smartlist_t *configs;
+};
+
+/**
+ * Allocate a new empty config_suite_t.
+ **/
+static config_suite_t *
+config_suite_new(void)
+{
+  config_suite_t *suite = tor_malloc_zero(sizeof(config_suite_t));
+  suite->configs = smartlist_new();
+  return suite;
+}
+
+/** Release all storage held by a config_suite_t.  (Does not free
+ * any configuration objects it holds; the caller must do that first.) */
+static void
+config_suite_free_(config_suite_t *suite)
+{
+  if (!suite)
+    return;
+  smartlist_free(suite->configs);
+  tor_free(suite);
+}
+
+#define config_suite_free(suite) \
+  FREE_AND_NULL(config_suite_t, config_suite_free_, (suite))
+
+struct config_mgr_t {
+  /** The 'top-level' configuration format.  This one is used for legacy
+   * options that have not yet been assigned to different sub-modules.
+   *
+   * (NOTE: for now, this is the only config_format_t that a config_mgr_t
+   * contains.  A subsequent commit will add more. XXXX)
+   */
+  const config_format_t *toplevel;
+  /**
+   * List of second-level configuration format objects that this manager
+   * also knows about.
+   */
+  smartlist_t *subconfigs;
+  /** A smartlist of managed_var_t objects for all configuration formats. */
+  smartlist_t *all_vars;
+  /** A smartlist of config_abbrev_t objects for all configuration
+   * formats. These objects are used to track synonyms and abbreviations for
+   * different configuration options. */
+  smartlist_t *all_abbrevs;
+  /** A smartlist of config_deprecation_t for all configuration formats. */
+  smartlist_t *all_deprecations;
+  /** True if this manager has been frozen and cannot have any more formats
+   * added to it. A manager must be frozen before it can be used to construct
+   * or manipulate objects. */
+  bool frozen;
+  /** A replacement for the magic number of the toplevel object. We override
+   * that number to make it unique for this particular config_mgr_t, so that
+   * an object constructed with one mgr can't be used with another, even if
+   * those managers' contents are equal.
+   */
+  struct_magic_decl_t toplevel_magic;
+};
+
+#define IDX_TOPLEVEL (-1)
+
+/** Create a new config_mgr_t to manage a set of configuration objects to be
+ * wrapped under <b>toplevel_fmt</b>. */
+config_mgr_t *
+config_mgr_new(const config_format_t *toplevel_fmt)
+{
+  config_mgr_t *mgr = tor_malloc_zero(sizeof(config_mgr_t));
+  mgr->subconfigs = smartlist_new();
+  mgr->all_vars = smartlist_new();
+  mgr->all_abbrevs = smartlist_new();
+  mgr->all_deprecations = smartlist_new();
+
+  config_mgr_register_fmt(mgr, toplevel_fmt, IDX_TOPLEVEL);
+  mgr->toplevel = toplevel_fmt;
+
+  return mgr;
+}
+
+/** Add a config_format_t to a manager, with a specified (unique) index. */
+static void
+config_mgr_register_fmt(config_mgr_t *mgr,
+                        const config_format_t *fmt,
+                        int object_idx)
+{
+  int i;
+
+  tor_assertf(!mgr->frozen,
+              "Tried to add a format to a configuration manager after "
+              "it had been frozen.");
+
+  if (object_idx != IDX_TOPLEVEL) {
+    tor_assertf(fmt->config_suite_offset < 0,
+          "Tried to register a toplevel format in a non-toplevel position");
+  }
+  tor_assertf(fmt != mgr->toplevel &&
+              ! smartlist_contains(mgr->subconfigs, fmt),
+              "Tried to register an already-registered format.");
+
+  /* register variables */
+  for (i = 0; fmt->vars[i].member.name; ++i) {
+    managed_var_t *mv = tor_malloc_zero(sizeof(managed_var_t));
+    mv->cvar = &fmt->vars[i];
+    mv->object_idx = object_idx;
+    smartlist_add(mgr->all_vars, mv);
+  }
+
+  /* register abbrevs */
+  if (fmt->abbrevs) {
+    for (i = 0; fmt->abbrevs[i].abbreviated; ++i) {
+      smartlist_add(mgr->all_abbrevs, (void*)&fmt->abbrevs[i]);
+    }
+  }
+
+  /* register deprecations. */
+  if (fmt->deprecations) {
+    const config_deprecation_t *d;
+    for (d = fmt->deprecations; d->name; ++d) {
+      smartlist_add(mgr->all_deprecations, (void*)d);
+    }
+  }
+}
+
+/**
+ * Add a new format to this configuration object.  Asserts on failure.
+ *
+ * Returns an internal "index" value used to identify this format within
+ * all of those formats contained in <b>mgr</b>.  This index value
+ * should not generally be used outside of this module.
+ **/
+int
+config_mgr_add_format(config_mgr_t *mgr,
+                      const config_format_t *fmt)
+{
+  tor_assert(mgr);
+  int idx = smartlist_len(mgr->subconfigs);
+  config_mgr_register_fmt(mgr, fmt, idx);
+  smartlist_add(mgr->subconfigs, (void *)fmt);
+  return idx;
+}
+
+/** Return a pointer to the config_suite_t * pointer inside a
+ * configuration object; returns NULL if there is no such member. */
+static inline config_suite_t **
+config_mgr_get_suite_ptr(const config_mgr_t *mgr, void *toplevel)
+{
+  if (mgr->toplevel->config_suite_offset < 0)
+    return NULL;
+  return STRUCT_VAR_P(toplevel, mgr->toplevel->config_suite_offset);
+}
+
+/**
+ * Return a pointer to the configuration object within <b>toplevel</b> whose
+ * index is <b>idx</b>.
+ *
+ * NOTE: XXXX Eventually, there will be multiple objects supported within the
+ * toplevel object.  For example, the or_options_t will contain pointers
+ * to configuration objects for other modules.  This function gets
+ * the sub-object for a particular module.
+ */
+STATIC void *
+config_mgr_get_obj_mutable(const config_mgr_t *mgr, void *toplevel, int idx)
+{
+  tor_assert(mgr);
+  tor_assert(toplevel);
+  if (idx == IDX_TOPLEVEL)
+    return toplevel;
+
+  tor_assertf(idx >= 0 && idx < smartlist_len(mgr->subconfigs),
+              "Index %d is out of range.", idx);
+  config_suite_t **suite = config_mgr_get_suite_ptr(mgr, toplevel);
+  tor_assert(suite);
+  tor_assert(smartlist_len(mgr->subconfigs) ==
+             smartlist_len((*suite)->configs));
+
+  return smartlist_get((*suite)->configs, idx);
+}
+
+/** As config_mgr_get_obj_mutable(), but return a const pointer. */
+STATIC const void *
+config_mgr_get_obj(const config_mgr_t *mgr, const void *toplevel, int idx)
+{
+  return config_mgr_get_obj_mutable(mgr, (void*)toplevel, idx);
+}
+
+/** Sorting helper for smartlist of managed_var_t */
+static int
+managed_var_cmp(const void **a, const void **b)
+{
+  const managed_var_t *mv1 = *(const managed_var_t**)a;
+  const managed_var_t *mv2 = *(const managed_var_t**)b;
+
+  return strcasecmp(mv1->cvar->member.name, mv2->cvar->member.name);
+}
+
+/**
+ * Mark a configuration manager as "frozen", so that no more formats can be
+ * added, and so that it can be used for manipulating configuration objects.
+ **/
+void
+config_mgr_freeze(config_mgr_t *mgr)
+{
+  static uint64_t mgr_count = 0;
+
+  smartlist_sort(mgr->all_vars, managed_var_cmp);
+  memcpy(&mgr->toplevel_magic, &mgr->toplevel->magic,
+         sizeof(struct_magic_decl_t));
+  uint64_t magic_input[3] = { mgr->toplevel_magic.magic_val,
+                              (uint64_t) (uintptr_t) mgr,
+                              ++mgr_count };
+  mgr->toplevel_magic.magic_val =
+    (uint32_t)siphash24g(magic_input, sizeof(magic_input));
+  mgr->frozen = true;
+}
+
+/** Release all storage held in <b>mgr</b> */
+void
+config_mgr_free_(config_mgr_t *mgr)
+{
+  if (!mgr)
+    return;
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, managed_var_free(mv));
+  smartlist_free(mgr->all_vars);
+  smartlist_free(mgr->all_abbrevs);
+  smartlist_free(mgr->all_deprecations);
+  smartlist_free(mgr->subconfigs);
+  memset(mgr, 0, sizeof(*mgr));
+  tor_free(mgr);
+}
+
+/** Return a new smartlist_t containing a config_var_t for every variable that
+ * <b>mgr</b> knows about.  The elements of this smartlist do not need
+ * to be freed; they have the same lifespan as <b>mgr</b>. */
+smartlist_t *
+config_mgr_list_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv,
+                    smartlist_add(result, (void*) mv->cvar));
+  return result;
+}
+
+/** Return a new smartlist_t containing the names of all deprecated variables.
+ * The elements of this smartlist do not need to be freed; they have the same
+ * lifespan as <b>mgr</b>.
+ */
+smartlist_t *
+config_mgr_list_deprecated_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_deprecations, config_deprecation_t *, d,
+                    smartlist_add(result, (char*)d->name));
+  return result;
+}
+
+/** Assert that the magic fields in <b>options</b> and its subsidiary
+ * objects are all okay. */
+static void
+config_mgr_assert_magic_ok(const config_mgr_t *mgr,
+                           const void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+  tor_assert(mgr->frozen);
+  struct_check_magic(options, &mgr->toplevel_magic);
+
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, (void*)options);
+  if (suitep == NULL) {
+    tor_assert(smartlist_len(mgr->subconfigs) == 0);
+    return;
+  }
+
+  tor_assert(smartlist_len((*suitep)->configs) ==
+             smartlist_len(mgr->subconfigs));
+  SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+    void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+    tor_assert(obj);
+    struct_check_magic(obj, &fmt->magic);
+  } SMARTLIST_FOREACH_END(fmt);
+}
+
+/** Macro: assert that <b>cfg</b> has the right magic field for
+ * <b>mgr</b>. */
+#define CONFIG_CHECK(mgr, cfg) STMT_BEGIN                               \
+    config_mgr_assert_magic_ok((mgr), (cfg));                           \
+  STMT_END
+
+/** Allocate an empty configuration object of a given format type. */
+void *
+config_new(const config_mgr_t *mgr)
+{
+  tor_assert(mgr->frozen);
+  void *opts = tor_malloc_zero(mgr->toplevel->size);
+  struct_set_magic(opts, &mgr->toplevel_magic);
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, opts);
+  if (suitep) {
+    *suitep = config_suite_new();
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = tor_malloc_zero(fmt->size);
+      struct_set_magic(obj, &fmt->magic);
+      smartlist_add((*suitep)->configs, obj);
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+  CONFIG_CHECK(mgr, opts);
+  return opts;
+}
+
+/*
+ * Functions to parse config options
+ */
+
+/** If <b>option</b> is an official abbreviation for a longer option,
+ * return the longer option.  Otherwise return <b>option</b>.
+ * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
+ * apply abbreviations that work for the config file and the command line.
+ * If <b>warn_obsolete</b> is set, warn about deprecated names. */
+const char *
+config_expand_abbrev(const config_mgr_t *mgr, const char *option,
+                     int command_line, int warn_obsolete)
+{
+  SMARTLIST_FOREACH_BEGIN(mgr->all_abbrevs, const config_abbrev_t *, abbrev) {
+    /* Abbreviations are case insensitive. */
+    if (!strcasecmp(option, abbrev->abbreviated) &&
+        (command_line || !abbrev->commandline_only)) {
+      if (warn_obsolete && abbrev->warn) {
+        log_warn(LD_CONFIG,
+                 "The configuration option '%s' is deprecated; "
+                 "use '%s' instead.",
+                 abbrev->abbreviated,
+                 abbrev->full);
+      }
+      /* Keep going through the list in case we want to rewrite it more.
+       * (We could imagine recursing here, but I don't want to get the
+       * user into an infinite loop if we craft our list wrong.) */
+      option = abbrev->full;
+    }
+  } SMARTLIST_FOREACH_END(abbrev);
+  return option;
+}
+
+/** If <b>key</b> is a deprecated configuration option, return the message
+ * explaining why it is deprecated (which may be an empty string). Return NULL
+ * if it is not deprecated. The <b>key</b> field must be fully expanded. */
+const char *
+config_find_deprecation(const config_mgr_t *mgr, const char *key)
+{
+  if (BUG(mgr == NULL) || BUG(key == NULL))
+    return NULL; // LCOV_EXCL_LINE
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_deprecations, const config_deprecation_t *,
+                          d) {
+    if (!strcasecmp(d->name, key)) {
+      return d->why_deprecated ? d->why_deprecated : "";
+    }
+  } SMARTLIST_FOREACH_END(d);
+  return NULL;
+}
+
+/**
+ * Find the managed_var_t object for a variable whose name is <b>name</b>
+ * according to <b>mgr</b>.   Return that object, or NULL if none exists.
+ *
+ * If <b>allow_truncated</b> is true, then accept any variable whose
+ * name begins with <b>name</b>.
+ *
+ * If <b>idx_out</b> is not NULL, set *<b>idx_out</b> to the position of
+ * that variable within mgr-&gt;all_vars, or to -1 if the variable is
+ * not found.
+ */
+static const managed_var_t *
+config_mgr_find_var(const config_mgr_t *mgr,
+                    const char *key,
+                    bool allow_truncated, int *idx_out)
+{
+  const size_t keylen = strlen(key);
+  if (idx_out)
+    *idx_out = -1;
+
+  if (!keylen)
+    return NULL; /* if they say "--" on the command line, it's not an option */
+
+  /* First, check for an exact (case-insensitive) match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strcasecmp(mv->cvar->member.name, key)) {
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (!allow_truncated)
+    return NULL;
+
+  /* If none, check for an abbreviated match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strncasecmp(key, mv->cvar->member.name, keylen)) {
+      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
+               "Please use '%s' instead",
+               key, mv->cvar->member.name);
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  /* Okay, unrecognized option */
+  return NULL;
+}
+
+/**
+ * If <b>key</b> is a name or an abbreviation configuration option, return
+ * the corresponding canonical name for it.  Warn if the abbreviation is
+ * non-standard.  Return NULL if the option does not exist.
+ */
+const char *
+config_find_option_name(const config_mgr_t *mgr, const char *key)
+{
+  key = config_expand_abbrev(mgr, key, 0, 0);
+  const managed_var_t *mv = config_mgr_find_var(mgr, key, true, NULL);
+  if (mv)
+    return mv->cvar->member.name;
+  else
+    return NULL;
+}
+
+/** Return the number of option entries in <b>fmt</b>. */
+static int
+config_count_options(const config_mgr_t *mgr)
+{
+  return smartlist_len(mgr->all_vars);
+}
+
+/**
+ * Return true iff at least one bit from <b>flag</b> is set on <b>var</b>,
+ * either in <b>var</b>'s flags, or on the flags of its type.
+ **/
+static bool
+config_var_has_flag(const config_var_t *var, uint32_t flag)
+{
+  uint32_t have_flags = var->flags | struct_var_get_flags(&var->member);
+
+  return (have_flags & flag) != 0;
+}
+
+/**
+ * Return true if assigning a value to <b>var</b> replaces the previous
+ * value.  Return false if assigning a value to <b>var</b> appends
+ * to the previous value.
+ **/
+static bool
+config_var_is_replaced_on_set(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOREPLACE);
+}
+
+/**
+ * Return true iff <b>var</b> may be assigned by name (e.g., via the
+ * CLI, the configuration files, or the controller API).
+ **/
+bool
+config_var_is_settable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOSET);
+}
+
+/**
+ * Return true iff the controller is allowed to fetch the value of
+ * <b>var</b>.
+ **/
+static bool
+config_var_is_gettable(const config_var_t *var)
+{
+  /* Arguably, invisible or obsolete options should not be gettable.  However,
+   * they have been gettable for a long time, and making them ungettable could
+   * have compatibility effects.  For now, let's leave them alone.
+   */
+
+  // return ! config_var_has_flag(var, CVFLAG_OBSOLETE|CFGLAGS_INVISIBLE);
+  (void)var;
+  return true;
+}
+
+/**
+ * Return true iff we need to check <b>var</b> for changes when we are
+ * comparing config options for changes.
+ *
+ * A false result might mean that the variable is a derived variable, and that
+ * comparing the variable it derives from compares this one too-- or it might
+ * mean that there is no data to compare.
+ **/
+static bool
+config_var_should_list_changes(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCMP);
+}
+
+/**
+ * Return true iff we need to copy the data for <b>var</b> when we are
+ * copying a config option.
+ *
+ * A false option might mean that the variable is a derived variable, and that
+ * copying the variable it derives from copies it-- or it might mean that
+ * there is no data to copy.
+ **/
+static bool
+config_var_needs_copy(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCOPY);
+}
+
+/**
+ * Return true iff variable <b>var</b> should appear on list of variable
+ * names given to the controller or the CLI.
+ *
+ * (Note that this option is imperfectly obeyed. The
+ * --list-torrc-options command looks at the "settable" flag, whereas
+ * "GETINFO config/defaults" and "list_deprecated_*()" do not filter
+ * their results. It would be good for consistency to try to converge
+ * these behaviors in the future.)
+ **/
+bool
+config_var_is_listable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOLIST);
+}
+
+/**
+ * Return true iff variable <b>var</b> should be written out when we
+ * are writing our configuration to disk, to a controller, or via the
+ * --dump-config command.
+ *
+ * This option may be set because a variable is hidden, or because it is
+ * derived from another variable which will already be written out.
+ **/
+static bool
+config_var_is_dumpable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NODUMP);
+}
+
+/*
+ * Functions to assign config options.
+ */
+
+/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
+ * with <b>c</b>-\>value and return 0, or return -1 if bad value.
+ *
+ * Called from config_assign_line() and option_reset().
+ */
+static int
+config_assign_value(const config_mgr_t *mgr, void *options,
+                    config_line_t *c, char **msg)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, c->key, true, NULL);
+  tor_assert(var);
+  tor_assert(!strcmp(c->key, var->cvar->member.name));
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+
+  return struct_var_kvassign(object, c, msg, &var->cvar->member);
+}
+
+/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
+ * to it will replace old ones. */
+static void
+config_mark_lists_fragile(const config_mgr_t *mgr, void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    void *object = config_mgr_get_obj_mutable(mgr, options, mv->object_idx);
+    struct_var_mark_fragile(object, &mv->cvar->member);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/**
+ * Log a warning that declaring that the option called <b>what</b>
+ * is deprecated because of the reason in <b>why</b>.
+ *
+ * (Both arguments must be non-NULL.)
+ **/
+void
+warn_deprecated_option(const char *what, const char *why)
+{
+  const char *space = (why && strlen(why)) ? " " : "";
+  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
+           "be removed in a future version of Tor.%s%s (If you think this is "
+           "a mistake, please let us know!)",
+           what, space, why);
+}
+
+/** If <b>c</b> is a syntactically valid configuration line, update
+ * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
+ * key, -2 for bad value.
+ *
+ * If <b>clear_first</b> is set, clear the value first. Then if
+ * <b>use_defaults</b> is set, set the value to the default.
+ *
+ * Called from config_assign().
+ */
+static int
+config_assign_line(const config_mgr_t *mgr, void *options,
+                   config_line_t *c, unsigned flags,
+                   bitarray_t *options_seen, char **msg)
+{
+  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
+  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
+  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
+  const managed_var_t *mvar;
+
+  CONFIG_CHECK(mgr, options);
+
+  int var_index = -1;
+  mvar = config_mgr_find_var(mgr, c->key, true, &var_index);
+  if (!mvar) {
+    const config_format_t *fmt = mgr->toplevel;
+    if (fmt->extra) {
+      void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset);
+      log_info(LD_CONFIG,
+               "Found unrecognized option '%s'; saving it.", c->key);
+      config_line_append((config_line_t**)lvalue, c->key, c->value);
+      return 0;
+    } else {
+      tor_asprintf(msg,
+                "Unknown option '%s'.  Failing.", c->key);
+      return -1;
+    }
+  }
+
+  const config_var_t *cvar = mvar->cvar;
+  tor_assert(cvar);
+
+  /* Put keyword into canonical case. */
+  if (strcmp(cvar->member.name, c->key)) {
+    tor_free(c->key);
+    c->key = tor_strdup(cvar->member.name);
+  }
+
+  const char *deprecation_msg;
+  if (warn_deprecations &&
+      (deprecation_msg = config_find_deprecation(mgr, cvar->member.name))) {
+    warn_deprecated_option(cvar->member.name, deprecation_msg);
+  }
+
+  if (!strlen(c->value)) {
+    /* reset or clear it, then return */
+    if (!clear_first) {
+      if (! config_var_is_replaced_on_set(cvar) &&
+          c->command != CONFIG_LINE_CLEAR) {
+        /* We got an empty linelist from the torrc or command line.
+           As a special case, call this an error. Warn and ignore. */
+        log_warn(LD_CONFIG,
+                 "Linelist option '%s' has no value. Skipping.", c->key);
+      } else { /* not already cleared */
+        config_reset(mgr, options, mvar, use_defaults);
+      }
+    }
+    return 0;
+  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
+    // This block is unreachable, since a CLEAR line always has an
+    // empty value, and so will trigger be handled by the previous
+    // "if (!strlen(c->value))" block.
+
+    // LCOV_EXCL_START
+    tor_assert_nonfatal_unreached();
+    config_reset(mgr, options, mvar, use_defaults);
+    // LCOV_EXCL_STOP
+  }
+
+  if (options_seen && config_var_is_replaced_on_set(cvar)) {
+    /* We're tracking which options we've seen, and this option is not
+     * supposed to occur more than once. */
+    tor_assert(var_index >= 0);
+    if (bitarray_is_set(options_seen, var_index)) {
+      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
+               "value will be ignored.", cvar->member.name);
+    }
+    bitarray_set(options_seen, var_index);
+  }
+
+  if (config_assign_value(mgr, options, c, msg) < 0)
+    return -2;
+  return 0;
+}
+
+/** Restore the option named <b>key</b> in options to its default value.
+ * Called from config_assign(). */
+STATIC void
+config_reset_line(const config_mgr_t *mgr, void *options,
+                  const char *key, int use_defaults)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var)
+    return; /* give error on next pass. */
+
+  config_reset(mgr, options, var, use_defaults);
+}
+
+/** Return true iff value needs to be quoted and escaped to be used in
+ * a configuration file. */
+static int
+config_value_needs_escape(const char *value)
+{
+  if (*value == '\"')
+    return 1;
+  while (*value) {
+    switch (*value)
+    {
+    case '\r':
+    case '\n':
+    case '#':
+      /* Note: quotes and backspaces need special handling when we are using
+       * quotes, not otherwise, so they don't trigger escaping on their
+       * own. */
+      return 1;
+    default:
+      if (!TOR_ISPRINT(*value))
+        return 1;
+    }
+    ++value;
+  }
+  return 0;
+}
+
+/** Return newly allocated line or lines corresponding to <b>key</b> in the
+ * configuration <b>options</b>.  If <b>escape_val</b> is true and a
+ * value needs to be quoted before it's put in a config file, quote and
+ * escape that value. Return NULL if no such key exists. */
+config_line_t *
+config_get_assigned_option(const config_mgr_t *mgr, const void *options,
+                           const char *key, int escape_val)
+{
+  const managed_var_t *var;
+  config_line_t *result;
+
+  tor_assert(options && key);
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var) {
+    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
+    return NULL;
+  }
+  if (! config_var_is_gettable(var->cvar)) {
+    log_warn(LD_CONFIG, "Option '%s' is obsolete or unfetchable. Failing.",
+             key);
+    return NULL;
+  }
+  const void *object = config_mgr_get_obj(mgr, options, var->object_idx);
+
+  result = struct_var_kvencode(object, &var->cvar->member);
+
+  if (escape_val) {
+    config_line_t *line;
+    for (line = result; line; line = line->next) {
+      if (line->value && config_value_needs_escape(line->value)) {
+        char *newval = esc_for_log(line->value);
+        tor_free(line->value);
+        line->value = newval;
+      }
+    }
+  }
+
+  return result;
+}
+/** Iterate through the linked list of requested options <b>list</b>.
+ * For each item, convert as appropriate and assign to <b>options</b>.
+ * If an item is unrecognized, set *msg and return -1 immediately,
+ * else return 0 for success.
+ *
+ * If <b>clear_first</b>, interpret config options as replacing (not
+ * extending) their previous values. If <b>clear_first</b> is set,
+ * then <b>use_defaults</b> to decide if you set to defaults after
+ * clearing, or make the value 0 or NULL.
+ *
+ * Here are the use cases:
+ * 1. A non-empty AllowInvalid line in your torrc. Appends to current
+ *    if linelist, replaces current if csv.
+ * 2. An empty AllowInvalid line in your torrc. Should clear it.
+ * 3. "RESETCONF AllowInvalid" sets it to default.
+ * 4. "SETCONF AllowInvalid" makes it NULL.
+ * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
+ *
+ * Use_defaults   Clear_first
+ *    0                0       "append"
+ *    1                0       undefined, don't use
+ *    0                1       "set to null first"
+ *    1                1       "set to defaults first"
+ * Return 0 on success, -1 on bad key, -2 on bad value.
+ *
+ * As an additional special case, if a LINELIST config option has
+ * no value and clear_first is 0, then warn and ignore it.
+ */
+
+/*
+There are three call cases for config_assign() currently.
+
+Case one: Torrc entry
+options_init_from_torrc() calls config_assign(0, 0)
+  calls config_assign_line(0, 0).
+    if value is empty, calls config_reset(0) and returns.
+    calls config_assign_value(), appends.
+
+Case two: setconf
+options_trial_assign() calls config_assign(0, 1)
+  calls config_reset_line(0)
+    calls config_reset(0)
+      calls option_clear().
+  calls config_assign_line(0, 1).
+    if value is empty, returns.
+    calls config_assign_value(), appends.
+
+Case three: resetconf
+options_trial_assign() calls config_assign(1, 1)
+  calls config_reset_line(1)
+    calls config_reset(1)
+      calls option_clear().
+      calls config_assign_value(default)
+  calls config_assign_line(1, 1).
+    returns.
+*/
+int
+config_assign(const config_mgr_t *mgr, void *options, config_line_t *list,
+              unsigned config_assign_flags, char **msg)
+{
+  config_line_t *p;
+  bitarray_t *options_seen;
+  const int n_options = config_count_options(mgr);
+  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
+  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
+
+  CONFIG_CHECK(mgr, options);
+
+  /* pass 1: normalize keys */
+  for (p = list; p; p = p->next) {
+    const char *full = config_expand_abbrev(mgr, p->key, 0, 1);
+    if (strcmp(full,p->key)) {
+      tor_free(p->key);
+      p->key = tor_strdup(full);
+    }
+  }
+
+  /* pass 2: if we're reading from a resetting source, clear all
+   * mentioned config options, and maybe set to their defaults. */
+  if (clear_first) {
+    for (p = list; p; p = p->next)
+      config_reset_line(mgr, options, p->key, use_defaults);
+  }
+
+  options_seen = bitarray_init_zero(n_options);
+  /* pass 3: assign. */
+  while (list) {
+    int r;
+    if ((r=config_assign_line(mgr, options, list, config_assign_flags,
+                              options_seen, msg))) {
+      bitarray_free(options_seen);
+      return r;
+    }
+    list = list->next;
+  }
+  bitarray_free(options_seen);
+
+  /** Now we're done assigning a group of options to the configuration.
+   * Subsequent group assignments should _replace_ linelists, not extend
+   * them. */
+  config_mark_lists_fragile(mgr, options);
+
+  return 0;
+}
+
+/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
+ * Called from config_reset() and config_free(). */
+static void
+config_clear(const config_mgr_t *mgr, void *options, const managed_var_t *var)
+{
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+  struct_var_free(object, &var->cvar->member);
+}
+
+/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
+ * <b>use_defaults</b>, set it to its default value.
+ * Called by config_init() and option_reset_line() and option_assign_line(). */
+static void
+config_reset(const config_mgr_t *mgr, void *options,
+             const managed_var_t *var, int use_defaults)
+{
+  config_line_t *c;
+  char *msg = NULL;
+  CONFIG_CHECK(mgr, options);
+  config_clear(mgr, options, var); /* clear it first */
+
+  if (!use_defaults)
+    return; /* all done */
+
+  if (var->cvar->initvalue) {
+    c = tor_malloc_zero(sizeof(config_line_t));
+    c->key = tor_strdup(var->cvar->member.name);
+    c->value = tor_strdup(var->cvar->initvalue);
+    if (config_assign_value(mgr, options, c, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Failed to assign default: %s", msg);
+      tor_free(msg); /* if this happens it's a bug */
+      // LCOV_EXCL_STOP
+    }
+    config_free_lines(c);
+  }
+}
+
+/** Release storage held by <b>options</b>. */
+void
+config_free_(const config_mgr_t *mgr, void *options)
+{
+  if (!options)
+    return;
+
+  tor_assert(mgr);
+
+  if (mgr->toplevel->clear_fn) {
+    mgr->toplevel->clear_fn(mgr, options);
+  }
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, options);
+  if (suitep) {
+    tor_assert(smartlist_len((*suitep)->configs) ==
+               smartlist_len(mgr->subconfigs));
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+      if (fmt->clear_fn) {
+        fmt->clear_fn(mgr, obj);
+      }
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    config_clear(mgr, options, mv);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (mgr->toplevel->extra) {
+    config_line_t **linep = STRUCT_VAR_P(options,
+                                         mgr->toplevel->extra->offset);
+    config_free_lines(*linep);
+    *linep = NULL;
+  }
+
+  if (suitep) {
+    SMARTLIST_FOREACH((*suitep)->configs, void *, obj, tor_free(obj));
+    config_suite_free(*suitep);
+  }
+
+  tor_free(options);
+}
+
+/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
+ * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
+ */
+int
+config_is_same(const config_mgr_t *mgr,
+               const void *o1, const void *o2,
+               const char *name)
+{
+  CONFIG_CHECK(mgr, o1);
+  CONFIG_CHECK(mgr, o2);
+
+  const managed_var_t *var = config_mgr_find_var(mgr, name, true, NULL);
+  if (!var) {
+    return true;
+  }
+  const void *obj1 = config_mgr_get_obj(mgr, o1, var->object_idx);
+  const void *obj2 = config_mgr_get_obj(mgr, o2, var->object_idx);
+
+  return struct_var_eq(obj1, obj2, &var->cvar->member);
+}
+
+/**
+ * Return a list of the options which have changed between <b>options1</b> and
+ * <b>options2</b>. If an option has reverted to its default value, it has a
+ * value entry of NULL.
+ *
+ * <b>options1</b> and <b>options2</b> must be top-level configuration objects
+ * of the type managed by <b>mgr</b>.
+ **/
+config_line_t *
+config_get_changes(const config_mgr_t *mgr,
+                   const void *options1, const void *options2)
+{
+  config_line_t *result = NULL;
+  config_line_t **next = &result;
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_should_list_changes(mv->cvar)) {
+      /* something else will check this var, or it doesn't need checking */
+      continue;
+    }
+    const void *obj1 = config_mgr_get_obj(mgr, options1, mv->object_idx);
+    const void *obj2 = config_mgr_get_obj(mgr, options2, mv->object_idx);
+
+    if (struct_var_eq(obj1, obj2, &mv->cvar->member)) {
+      continue;
+    }
+
+    const char *varname = mv->cvar->member.name;
+    config_line_t *line =
+      config_get_assigned_option(mgr, options2, varname, 1);
+
+    if (line) {
+      *next = line;
+    } else {
+      *next = tor_malloc_zero(sizeof(config_line_t));
+      (*next)->key = tor_strdup(varname);
+    }
+    while (*next)
+      next = &(*next)->next;
+  } SMARTLIST_FOREACH_END(mv);
+
+  return result;
+}
+
+/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
+void *
+config_dup(const config_mgr_t *mgr, const void *old)
+{
+  void *newopts;
+
+  newopts = config_new(mgr);
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_needs_copy(mv->cvar)) {
+      // Something else will copy this option, or it doesn't need copying.
+      continue;
+    }
+    const void *oldobj = config_mgr_get_obj(mgr, old, mv->object_idx);
+    void *newobj = config_mgr_get_obj_mutable(mgr, newopts, mv->object_idx);
+    if (struct_var_copy(newobj, oldobj, &mv->cvar->member) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Unable to copy value for %s.",
+              mv->cvar->member.name);
+      tor_assert_unreached();
+      // LCOV_EXCL_STOP
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return newopts;
+}
+/** Set all vars in the configuration object <b>options</b> to their default
+ * values. */
+void
+config_init(const config_mgr_t *mgr, void *options)
+{
+  CONFIG_CHECK(mgr, options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!mv->cvar->initvalue)
+      continue; /* defaults to NULL or 0 */
+    config_reset(mgr, options, mv, 1);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/** Allocate and return a new string holding the written-out values of the vars
+ * in 'options'.  If 'minimal', do not write out any default-valued vars.
+ * Else, if comment_defaults, write default values as comments.
+ */
+char *
+config_dump(const config_mgr_t *mgr, const void *default_options,
+            const void *options, int minimal,
+            int comment_defaults)
+{
+  const config_format_t *fmt = mgr->toplevel;
+  smartlist_t *elements;
+  const void *defaults = default_options;
+  void *defaults_tmp = NULL;
+  config_line_t *line, *assigned;
+  char *result;
+  char *msg = NULL;
+
+  if (defaults == NULL) {
+    defaults = defaults_tmp = config_new(mgr);
+    config_init(mgr, defaults_tmp);
+  }
+
+  /* XXX use a 1 here so we don't add a new log line while dumping */
+  if (default_options == NULL) {
+    if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Failed to validate default config: %s", msg);
+      tor_free(msg);
+      tor_assert(0);
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  elements = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    int comment_option = 0;
+    /* Don't save 'hidden' control variables. */
+    if (! config_var_is_dumpable(mv->cvar))
+      continue;
+    const char *name = mv->cvar->member.name;
+    if (minimal && config_is_same(mgr, options, defaults, name))
+      continue;
+    else if (comment_defaults &&
+             config_is_same(mgr, options, defaults, name))
+      comment_option = 1;
+
+    line = assigned =
+      config_get_assigned_option(mgr, options, name, 1);
+
+    for (; line; line = line->next) {
+      if (!strcmpstart(line->key, "__")) {
+        /* This check detects "hidden" variables inside LINELIST_V structures.
+         */
+        continue;
+      }
+      smartlist_add_asprintf(elements, "%s%s %s\n",
+                   comment_option ? "# " : "",
+                   line->key, line->value);
+    }
+    config_free_lines(assigned);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (fmt->extra) {
+    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset);
+    for (; line; line = line->next) {
+      smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value);
+    }
+  }
+
+  result = smartlist_join_strings(elements, "", 0, NULL);
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+  config_free(mgr, defaults_tmp);
+  return result;
+}
+
+/**
+ * Return true if every member of <b>options</b> is in-range and well-formed.
+ * Return false otherwise.  Log errors at level <b>severity</b>.
+ */
+bool
+config_check_ok(const config_mgr_t *mgr, const void *options, int severity)
+{
+  bool all_ok = true;
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!struct_var_ok(options, &mv->cvar->member)) {
+      log_fn(severity, LD_BUG, "Invalid value for %s",
+             mv->cvar->member.name);
+      all_ok = false;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return all_ok;
+}
diff --git a/src/lib/confmgt/confparse.h b/src/lib/confmgt/confparse.h
new file mode 100644
index 0000000000..2332f69790
--- /dev/null
+++ b/src/lib/confmgt/confparse.h
@@ -0,0 +1,212 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confparse.h
+ *
+ * \brief Header for confparse.c.
+ */
+
+#ifndef TOR_CONFPARSE_H
+#define TOR_CONFPARSE_H
+
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confmacros.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * An abbreviation or alias for a configuration option.
+ **/
+typedef struct config_abbrev_t {
+  /** The option name as abbreviated.  Not case-sensitive. */
+  const char *abbreviated;
+  /** The full name of the option. Not case-sensitive. */
+  const char *full;
+  /** True if this abbreviation should only be allowed on the command line. */
+  int commandline_only;
+  /** True if we should warn whenever this abbreviation is used. */
+  int warn;
+} config_abbrev_t;
+
+/**
+ * A note that a configuration option is deprecated, with an explanation why.
+ */
+typedef struct config_deprecation_t {
+  /** The option that is deprecated. */
+  const char *name;
+  /** A user-facing string explaining why the option is deprecated. */
+  const char *why_deprecated;
+} config_deprecation_t;
+
+/**
+ * Handy macro for declaring "In the config file or on the command line, you
+ * can abbreviate <b>tok</b>s as <b>tok</b>". Used inside an array of
+ * config_abbrev_t.
+ *
+ * For example, to declare "NumCpu" as an abbreviation for "NumCPUs",
+ * you can say PLURAL(NumCpu).
+ **/
+#define PLURAL(tok) { #tok, #tok "s", 0, 0 }
+
+/**
+ * Type of a callback to validate whether a given configuration is
+ * well-formed and consistent.
+ *
+ * The configuration to validate is passed as <b>newval</b>. The previous
+ * configuration, if any, is provided in <b>oldval</b>.  The
+ * <b>default_val</b> argument receives a configuration object initialized
+ * with default values for all its fields.  The <b>from_setconf</b> argument
+ * is true iff the input comes from a SETCONF controller command.
+ *
+ * On success, return 0.  On failure, set *<b>msg_out</b> to a newly allocated
+ * error message, and return -1.
+ *
+ * REFACTORING NOTE: Currently, this callback type is only used from inside
+ * config_dump(); later in our refactoring, it will be cleaned up and used
+ * more generally.
+ */
+typedef int (*validate_fn_t)(void *oldval,
+                             void *newval,
+                             void *default_val,
+                             int from_setconf,
+                             char **msg_out);
+
+struct config_mgr_t;
+
+/**
+ * Callback to clear all non-managed fields of a configuration object.
+ *
+ * <b>obj</b> is the configuration object whose non-managed fields should be
+ * cleared.
+ *
+ * (Regular fields get cleared by config_reset(), but you might have fields
+ * in the object that do not correspond to configuration variables.  If those
+ * fields need to be cleared or freed, this is where to do it.)
+ */
+typedef void (*clear_cfg_fn_t)(const struct config_mgr_t *mgr, void *obj);
+
+/** Information on the keys, value types, key-to-struct-member mappings,
+ * variable descriptions, validation functions, and abbreviations for a
+ * configuration or storage format. */
+typedef struct config_format_t {
+  size_t size; /**< Size of the struct that everything gets parsed into. */
+  struct_magic_decl_t magic; /**< Magic number info for this struct. */
+  const config_abbrev_t *abbrevs; /**< List of abbreviations that we expand
+                             * when parsing this format. */
+  const config_deprecation_t *deprecations; /** List of deprecated options */
+  const config_var_t *vars; /**< List of variables we recognize, their default
+                             * values, and where we stick them in the
+                             * structure. */
+  validate_fn_t validate_fn; /**< Function to validate config. */
+  clear_cfg_fn_t clear_fn; /**< Function to clear the configuration. */
+  /** If present, extra denotes a LINELIST variable for unrecognized
+   * lines.  Otherwise, unrecognized lines are an error. */
+  const struct_member_t *extra;
+  /** The position of a config_suite_t pointer within the toplevel object,
+   * or -1 if there is no such pointer. */
+  ptrdiff_t config_suite_offset;
+} config_format_t;
+
+/**
+ * A collection of config_format_t objects to describe several objects
+ * that are all configured with the same configuration file.
+ *
+ * (NOTE: for now, this only handles a single config_format_t.)
+ **/
+typedef struct config_mgr_t config_mgr_t;
+
+config_mgr_t *config_mgr_new(const config_format_t *toplevel_fmt);
+void config_mgr_free_(config_mgr_t *mgr);
+int config_mgr_add_format(config_mgr_t *mgr,
+                          const config_format_t *fmt);
+void config_mgr_freeze(config_mgr_t *mgr);
+#define config_mgr_free(mgr) \
+  FREE_AND_NULL(config_mgr_t, config_mgr_free_, (mgr))
+struct smartlist_t *config_mgr_list_vars(const config_mgr_t *mgr);
+struct smartlist_t *config_mgr_list_deprecated_vars(const config_mgr_t *mgr);
+
+/** A collection of managed configuration objects. */
+typedef struct config_suite_t config_suite_t;
+
+/**
+ * Flag for config_assign: if set, then "resetting" an option changes it to
+ * its default value, as specified in the config_var_t.  Otherwise,
+ * "resetting" an option changes it to a type-dependent null value --
+ * typically 0 or NULL.
+ *
+ * (An option is "reset" when it is set to an empty value, or as described in
+ * CAL_CLEAR_FIRST).
+ **/
+#define CAL_USE_DEFAULTS      (1u<<0)
+/**
+ * Flag for config_assign: if set, then we reset every provided config
+ * option before we set it.
+ *
+ * For example, if this flag is not set, then passing a multi-line option to
+ * config_assign will cause any previous value to be extended. But if this
+ * flag is set, then a multi-line option will replace any previous value.
+ **/
+#define CAL_CLEAR_FIRST       (1u<<1)
+/**
+ * Flag for config_assign: if set, we warn about deprecated options.
+ **/
+#define CAL_WARN_DEPRECATIONS (1u<<2)
+
+void *config_new(const config_mgr_t *fmt);
+void config_free_(const config_mgr_t *fmt, void *options);
+#define config_free(mgr, options) do {                \
+    config_free_((mgr), (options));                   \
+    (options) = NULL;                                 \
+  } while (0)
+
+struct config_line_t *config_get_assigned_option(const config_mgr_t *mgr,
+                                          const void *options, const char *key,
+                                          int escape_val);
+int config_is_same(const config_mgr_t *fmt,
+                   const void *o1, const void *o2,
+                   const char *name);
+struct config_line_t *config_get_changes(const config_mgr_t *mgr,
+                                  const void *options1, const void *options2);
+void config_init(const config_mgr_t *mgr, void *options);
+void *config_dup(const config_mgr_t *mgr, const void *old);
+char *config_dump(const config_mgr_t *mgr, const void *default_options,
+                  const void *options, int minimal,
+                  int comment_defaults);
+bool config_check_ok(const config_mgr_t *mgr, const void *options,
+                     int severity);
+int config_assign(const config_mgr_t *mgr, void *options,
+                  struct config_line_t *list,
+                  unsigned flags, char **msg);
+const char *config_find_deprecation(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_find_option_name(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_expand_abbrev(const config_mgr_t *mgr,
+                                 const char *option,
+                                 int command_line, int warn_obsolete);
+void warn_deprecated_option(const char *what, const char *why);
+
+bool config_var_is_settable(const config_var_t *var);
+bool config_var_is_listable(const config_var_t *var);
+
+/* Helper macros to compare an option across two configuration objects */
+#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
+#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
+#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
+#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
+
+#ifdef CONFPARSE_PRIVATE
+STATIC void config_reset_line(const config_mgr_t *mgr, void *options,
+                              const char *key, int use_defaults);
+STATIC void *config_mgr_get_obj_mutable(const config_mgr_t *mgr,
+                                        void *toplevel, int idx);
+STATIC const void *config_mgr_get_obj(const config_mgr_t *mgr,
+                                       const void *toplevel, int idx);
+#endif /* defined(CONFPARSE_PRIVATE) */
+
+#endif /* !defined(TOR_CONFPARSE_H) */
diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am
new file mode 100644
index 0000000000..81cd868e5e
--- /dev/null
+++ b/src/lib/confmgt/include.am
@@ -0,0 +1,27 @@
+noinst_LIBRARIES += src/lib/libtor-confmgt.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_confmgt_a_SOURCES =			\
+	src/lib/confmgt/confparse.c			\
+	src/lib/confmgt/structvar.c                      \
+	src/lib/confmgt/type_defs.c                      \
+	src/lib/confmgt/typedvar.c                      \
+	src/lib/confmgt/unitparse.c
+
+src_lib_libtor_confmgt_testing_a_SOURCES = \
+	$(src_lib_libtor_confmgt_a_SOURCES)
+src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/confmgt/confparse.h			\
+	src/lib/confmgt/structvar.h			\
+	src/lib/confmgt/type_defs.h			\
+	src/lib/confmgt/typedvar.h			\
+	src/lib/confmgt/unitparse.h			\
+	src/lib/confmgt/var_type_def_st.h
diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c
new file mode 100644
index 0000000000..7a3b8c7df2
--- /dev/null
+++ b/src/lib/confmgt/structvar.c
@@ -0,0 +1,221 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.c
+ * @brief Functions to manipulate named and typed elements of
+ *    a structure.
+ *
+ * These functions represent a low-level API for accessing a member of a
+ * structure.  They use typedvar.c to work, and they are used in turn by the
+ * configuration system to examine and set fields in configuration objects
+ * used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/structvar.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/log/util_bug.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+
+/**
+ * Set the 'magic number' on <b>object</b> to correspond to decl.
+ **/
+void
+struct_set_magic(void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(object);
+  tor_assert(decl);
+  uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  *ptr = decl->magic_val;
+}
+
+/**
+ * Assert that the 'magic number' on <b>object</b> to corresponds to decl.
+ **/
+void
+struct_check_magic(const void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(object);
+  tor_assert(decl);
+
+  const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  tor_assertf(*ptr == decl->magic_val,
+              "Bad magic number on purported %s object. "
+              "Expected %"PRIu32"x but got %"PRIu32"x.",
+              decl->typename, decl->magic_val, *ptr);
+}
+
+/**
+ * Return a mutable pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+void *
+struct_get_mptr(void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Return a const pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+const void *
+struct_get_ptr(const void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Helper: given a struct_member_t, look up the type definition for its
+ * variable.
+ */
+static const var_type_def_t *
+get_type_def(const struct_member_t *member)
+{
+  if (member->type_def)
+    return member->type_def;
+
+  return lookup_type_def(member->type);
+}
+
+/**
+ * (As typed_var_free, but free and clear the member of <b>object</b> defined
+ * by <b>member</b>.)
+ **/
+void
+struct_var_free(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  typed_var_free(p, def);
+}
+
+/**
+ * (As typed_var_copy, but copy from <b>src</b> to <b>dest</b> the member
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_copy(void *dest, const void *src, const struct_member_t *member)
+{
+  void *p_dest = struct_get_mptr(dest, member);
+  const void *p_src = struct_get_ptr(src, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_copy(p_dest, p_src, def);
+}
+
+/**
+ * (As typed_var_eq, but compare the members of <b>a</b> and <b>b</b>
+ * defined by <b>member</b>.)
+ **/
+bool
+struct_var_eq(const void *a, const void *b, const struct_member_t *member)
+{
+  const void *p_a = struct_get_ptr(a, member);
+  const void *p_b = struct_get_ptr(b, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_eq(p_a, p_b, def);
+}
+
+/**
+ * (As typed_var_ok, but validate the member of <b>object</b> defined by
+ * <b>member</b>.)
+ **/
+bool
+struct_var_ok(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_ok(p, def);
+}
+
+/**
+ * (As typed_var_kvassign, but assign a value to the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_kvassign(void *object, const struct config_line_t *line,
+                    char **errmsg,
+                    const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvassign(p, line, errmsg, def);
+}
+
+/**
+ * (As typed_var_kvencode, but encode the value of the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+struct config_line_t *
+struct_var_kvencode(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvencode(member->name, p, def);
+}
+
+/**
+ * Mark the field in <b>object</b> determined by <b>member</b> -- a variable
+ * that ordinarily would be extended by assignment -- as "fragile", so that it
+ * will get replaced by the next assignment instead.
+ */
+void
+struct_var_mark_fragile(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+  return typed_var_mark_fragile(p, def);
+}
+
+/**
+ * Return the official name of this struct member.
+ **/
+const char *
+struct_var_get_name(const struct_member_t *member)
+{
+  return member->name;
+}
+
+/**
+ * Return the type name for this struct member.
+ *
+ * Do not use the output of this function to inspect a type within Tor.  It is
+ * suitable for debugging, informing the controller or user of a variable's
+ * type, etc.
+ **/
+const char *
+struct_var_get_typename(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->name : NULL;
+}
+
+/** Return all of the flags set for this struct member. */
+uint32_t
+struct_var_get_flags(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->flags : 0;
+}
diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h
new file mode 100644
index 0000000000..bcb4b58c3f
--- /dev/null
+++ b/src/lib/confmgt/structvar.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.h
+ * @brief Header for lib/confmgt/structvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H
+#define TOR_LIB_CONFMGT_STRUCTVAR_H
+
+struct struct_magic_decl_t;
+struct struct_member_t;
+struct config_line_t;
+
+#include <stdbool.h>
+#include "lib/cc/torint.h"
+
+void struct_set_magic(void *object,
+                      const struct struct_magic_decl_t *decl);
+void struct_check_magic(const void *object,
+                        const struct struct_magic_decl_t *decl);
+
+void *struct_get_mptr(void *object,
+                      const struct struct_member_t *member);
+const void *struct_get_ptr(const void *object,
+                           const struct struct_member_t *member);
+
+void struct_var_free(void *object,
+                     const struct struct_member_t *member);
+int struct_var_copy(void *dest, const void *src,
+                    const struct struct_member_t *member);
+bool struct_var_eq(const void *a, const void *b,
+                   const struct struct_member_t *member);
+bool struct_var_ok(const void *object,
+                   const struct struct_member_t *member);
+void struct_var_mark_fragile(void *object,
+                             const struct struct_member_t *member);
+
+const char *struct_var_get_name(const struct struct_member_t *member);
+const char *struct_var_get_typename(const struct struct_member_t *member);
+uint32_t struct_var_get_flags(const struct struct_member_t *member);
+
+int struct_var_kvassign(void *object, const struct config_line_t *line,
+                        char **errmsg,
+                        const struct struct_member_t *member);
+struct config_line_t *struct_var_kvencode(
+                        const void *object,
+                        const struct struct_member_t *member);
+
+#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */
diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c
new file mode 100644
index 0000000000..5066e12265
--- /dev/null
+++ b/src/lib/confmgt/type_defs.c
@@ -0,0 +1,791 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.c
+ * @brief Definitions for various low-level configuration types.
+ *
+ * This module creates a number of var_type_def_t objects, to be used by
+ * typedvar.c in manipulating variables.
+ *
+ * The types here are common types that can be implemented with Tor's
+ * low-level functionality.  To define new types, see var_type_def_st.h.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/unitparse.h"
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/conf/conftypes.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+#include <errno.h>
+
+//////
+// CONFIG_TYPE_STRING
+// CONFIG_TYPE_FILENAME
+//
+// These two types are the same for now, but they have different names.
+//
+// Warning: For this type, the default value (NULL) and "" are considered
+// different values.  That is generally risky, and best avoided for other
+// types in the future.
+//////
+
+static int
+string_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  char **p = (char**)target;
+  *p = tor_strdup(value);
+  return 0;
+}
+
+static char *
+string_encode(const void *value, const void *params)
+{
+  (void)params;
+  const char **p = (const char**)value;
+  return *p ? tor_strdup(*p) : NULL;
+}
+
+static void
+string_clear(void *value, const void *params)
+{
+  (void)params;
+  char **p = (char**)value;
+  tor_free(*p); // sets *p to NULL.
+}
+
+static const var_type_fns_t string_fns = {
+  .parse = string_parse,
+  .encode = string_encode,
+  .clear = string_clear,
+};
+
+/////
+// CONFIG_TYPE_INT
+// CONFIG_TYPE_POSINT
+//
+// These types are implemented as int, possibly with a restricted range.
+/////
+
+typedef struct int_type_params_t {
+  int minval;
+  int maxval;
+} int_parse_params_t;
+
+static const int_parse_params_t INT_PARSE_UNRESTRICTED = {
+  .minval = INT_MIN,
+  .maxval = INT_MAX,
+};
+
+static const int_parse_params_t INT_PARSE_POSINT = {
+  .minval = 0,
+  .maxval = INT_MAX,
+};
+
+static int
+int_parse(void *target, const char *value, char **errmsg, const void *params,
+          const char *key)
+{
+  (void)key;
+  const int_parse_params_t *pp;
+  if (params) {
+    pp = params;
+  } else {
+    pp = &INT_PARSE_UNRESTRICTED;
+  }
+  int *p = target;
+  int ok=0;
+  *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.",
+                 value);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+int_encode(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  char *result;
+  tor_asprintf(&result, "%d", v);
+  return result;
+}
+
+static void
+int_clear(void *value, const void *params)
+{
+  (void)params;
+  *(int*)value = 0;
+}
+
+static bool
+int_ok(const void *value, const void *params)
+{
+  const int_parse_params_t *pp = params;
+  if (pp) {
+    int v = *(int*)value;
+    return pp->minval <= v && v <= pp->maxval;
+  } else {
+    return true;
+  }
+}
+
+static const var_type_fns_t int_fns = {
+  .parse = int_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+  .ok = int_ok,
+};
+
+/////
+// CONFIG_TYPE_UINT64
+//
+// This type is an unrestricted u64.
+/////
+
+static int
+uint64_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  uint64_t *p = target;
+  int ok=0;
+  *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.",
+                 value);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+uint64_encode(const void *value, const void *params)
+{
+  (void)params;
+  uint64_t v = *(uint64_t*)value;
+  char *result;
+  tor_asprintf(&result, "%"PRIu64, v);
+  return result;
+}
+
+static void
+uint64_clear(void *value, const void *params)
+{
+  (void)params;
+  *(uint64_t*)value = 0;
+}
+
+static const var_type_fns_t uint64_fns = {
+  .parse = uint64_parse,
+  .encode = uint64_encode,
+  .clear = uint64_clear,
+};
+
+/////
+// CONFIG_TYPE_INTERVAL
+// CONFIG_TYPE_MSEC_INTERVAL
+// CONFIG_TYPE_MEMUNIT
+//
+// These types are implemented using the config_parse_units() function.
+// The intervals are stored as ints, whereas memory units are stored as
+// uint64_ts.
+/////
+
+static int
+units_parse_u64(void *target, const char *value, char **errmsg,
+                const void *params, const char *key)
+{
+  (void)key;
+  const unit_table_t *table = params;
+  tor_assert(table);
+  uint64_t *v = (uint64_t*)target;
+  int ok=1;
+  *v = config_parse_units(value, table, &ok);
+  if (!ok) {
+    *errmsg = tor_strdup("Provided value is malformed or out of bounds.");
+    return -1;
+  }
+  return 0;
+}
+
+static int
+units_parse_int(void *target, const char *value, char **errmsg,
+                const void *params, const char *key)
+{
+  (void)key;
+  const unit_table_t *table = params;
+  tor_assert(table);
+  int *v = (int*)target;
+  int ok=1;
+  uint64_t u64 = config_parse_units(value, table, &ok);
+  if (!ok) {
+    *errmsg = tor_strdup("Provided value is malformed or out of bounds.");
+    return -1;
+  }
+  if (u64 > INT_MAX) {
+    tor_asprintf(errmsg, "Provided value %s is too large", value);
+    return -1;
+  }
+  *v = (int) u64;
+  return 0;
+}
+
+static bool
+units_ok_int(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  return v >= 0;
+}
+
+static const var_type_fns_t memunit_fns = {
+  .parse = units_parse_u64,
+  .encode = uint64_encode, // doesn't use params
+  .clear = uint64_clear, // doesn't use params
+};
+
+static const var_type_fns_t interval_fns = {
+  .parse = units_parse_int,
+  .encode = int_encode, // doesn't use params
+  .clear = int_clear, // doesn't use params,
+  .ok = units_ok_int // can't use int_ok, since that expects int params.
+};
+
+/////
+// CONFIG_TYPE_DOUBLE
+//
+// This is a nice simple double.
+/////
+
+static int
+double_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  double *v = (double*)target;
+  char *endptr=NULL;
+  errno = 0;
+  *v = strtod(value, &endptr);
+  if (endptr == value || *endptr != '\0') {
+    // Either there are no converted characters, or there were some characters
+    // that didn't get converted.
+    tor_asprintf(errmsg, "Could not convert %s to a number.", escaped(value));
+    return -1;
+  }
+  if (errno == ERANGE) {
+    // strtod will set errno to ERANGE on underflow or overflow.
+    bool underflow = -.00001 < *v && *v < .00001;
+    tor_asprintf(errmsg,
+                 "%s is too %s to express as a floating-point number.",
+                 escaped(value), underflow ? "small" : "large");
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+double_encode(const void *value, const void *params)
+{
+  (void)params;
+  double v = *(double*)value;
+  char *result;
+  tor_asprintf(&result, "%f", v);
+  return result;
+}
+
+static void
+double_clear(void *value, const void *params)
+{
+  (void)params;
+  double *v = (double *)value;
+  *v = 0.0;
+}
+
+static const var_type_fns_t double_fns = {
+  .parse = double_parse,
+  .encode = double_encode,
+  .clear = double_clear,
+};
+
+/////
+// CONFIG_TYPE_BOOL
+// CONFIG_TYPE_AUTOBOOL
+//
+// These types are implemented as a case-insensitive string-to-integer
+// mapping.
+/////
+
+typedef struct enumeration_table_t {
+  const char *name;
+  int value;
+} enumeration_table_t;
+
+static int
+enum_parse(void *target, const char *value, char **errmsg,
+           const void *params, const char *key)
+{
+  (void)key;
+  const enumeration_table_t *table = params;
+  int *p = (int *)target;
+  for (; table->name; ++table) {
+    if (!strcasecmp(value, table->name)) {
+      *p = table->value;
+      return 0;
+    }
+  }
+  tor_asprintf(errmsg, "Unrecognized value %s.", value);
+  return -1;
+}
+
+static char *
+enum_encode(const void *value, const void *params)
+{
+  int v = *(const int*)value;
+  const enumeration_table_t *table = params;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return tor_strdup(table->name);
+  }
+  return NULL; // error.
+}
+
+static void
+enum_clear(void *value, const void *params)
+{
+  int *p = (int*)value;
+  const enumeration_table_t *table = params;
+  tor_assert(table->name);
+  *p = table->value;
+}
+
+static bool
+enum_ok(const void *value, const void *params)
+{
+  int v = *(const int*)value;
+  const enumeration_table_t *table = params;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return true;
+  }
+  return false;
+}
+
+static const enumeration_table_t enum_table_bool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { NULL, 0 },
+};
+
+static const enumeration_table_t enum_table_autobool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { "auto", -1 },
+  { NULL, 0 },
+};
+
+static const var_type_fns_t enum_fns = {
+  .parse = enum_parse,
+  .encode = enum_encode,
+  .clear = enum_clear,
+  .ok = enum_ok,
+};
+
+/////
+// CONFIG_TYPE_ISOTIME
+//
+// This is a time_t, encoded in ISO8601 format.
+/////
+
+static int
+time_parse(void *target, const char *value, char **errmsg,
+           const void *params, const char *key)
+{
+  (void) params;
+  (void) key;
+  time_t *p = target;
+  if (parse_iso_time(value, p) < 0) {
+    tor_asprintf(errmsg, "Invalid time %s", escaped(value));
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+time_encode(const void *value, const void *params)
+{
+  (void)params;
+  time_t v = *(const time_t *)value;
+  char *result = tor_malloc(ISO_TIME_LEN+1);
+  format_iso_time(result, v);
+  return result;
+}
+
+static void
+time_clear(void *value, const void *params)
+{
+  (void)params;
+  time_t *t = value;
+  *t = 0;
+}
+
+static const var_type_fns_t time_fns = {
+  .parse = time_parse,
+  .encode = time_encode,
+  .clear = time_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV
+//
+// This type is a comma-separated list of strings, stored in a smartlist_t.
+// An empty list may be encoded either as an empty smartlist, or as NULL.
+/////
+
+static int
+csv_parse(void *target, const char *value, char **errmsg,
+          const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  smartlist_t **sl = (smartlist_t**)target;
+  *sl = smartlist_new();
+  smartlist_split_string(*sl, value, ",",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  return 0;
+}
+
+static char *
+csv_encode(const void *value, const void *params)
+{
+  (void)params;
+  const smartlist_t *sl = *(const smartlist_t **)value;
+  if (! sl)
+    return tor_strdup("");
+
+  return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
+}
+
+static void
+csv_clear(void *value, const void *params)
+{
+  (void)params;
+  smartlist_t **sl = (smartlist_t**)value;
+  if (!*sl)
+    return;
+  SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp));
+  smartlist_free(*sl); // clears pointer.
+}
+
+static const var_type_fns_t csv_fns = {
+  .parse = csv_parse,
+  .encode = csv_encode,
+  .clear = csv_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV_INTERVAL
+//
+// This type used to be a list of time intervals, used to determine a download
+// schedule.  Now, only the first interval counts: everything after the first
+// comma is discarded.
+/////
+
+static int
+legacy_csv_interval_parse(void *target, const char *value, char **errmsg,
+                          const void *params, const char *key)
+{
+  (void)params;
+  /* We used to have entire smartlists here.  But now that all of our
+   * download schedules use exponential backoff, only the first part
+   * matters. */
+  const char *comma = strchr(value, ',');
+  const char *val = value;
+  char *tmp = NULL;
+  if (comma) {
+    tmp = tor_strndup(val, comma - val);
+    val = tmp;
+  }
+
+  int rv = units_parse_int(target, val, errmsg, &time_units, key);
+  tor_free(tmp);
+  return rv;
+}
+
+static const var_type_fns_t legacy_csv_interval_fns = {
+  .parse = legacy_csv_interval_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+};
+
+/////
+// CONFIG_TYPE_LINELIST
+// CONFIG_TYPE_LINELIST_S
+// CONFIG_TYPE_LINELIST_V
+//
+// A linelist is a raw config_line_t list.  Order is preserved.
+//
+// The LINELIST type is used for homogeneous lists, where all the lines
+// have the same key.
+//
+// The LINELIST_S and LINELIST_V types are used for the case where multiple
+// lines of different keys are kept in a single list, to preserve their
+// relative order.  The unified list is stored as a "virtual" variable whose
+// type is LINELIST_V; the individual sublists are treated as variables of
+// type LINELIST_S.
+//
+// A linelist may be fragile or non-fragile. Assigning a line to a fragile
+// linelist replaces the list with the line.  If the line has the "APPEND"
+// command set on it, or if the list is non-fragile, the line is appended.
+// Either way, the new list is non-fragile.
+/////
+
+static int
+linelist_kv_parse(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  config_line_t **lines = target;
+
+  if (*lines && (*lines)->fragile) {
+    if (line->command == CONFIG_LINE_APPEND) {
+      (*lines)->fragile = 0;
+    } else {
+      config_free_lines(*lines); // sets it to NULL
+    }
+  }
+
+  config_line_append(lines, line->key, line->value);
+  return 0;
+}
+
+static int
+linelist_kv_virt_noparse(void *target, const struct config_line_t *line,
+                         char **errmsg, const void *params)
+{
+  (void)target;
+  (void)line;
+  (void)params;
+  *errmsg = tor_strdup("Cannot assign directly to virtual option.");
+  return -1;
+}
+
+static struct config_line_t *
+linelist_kv_encode(const char *key, const void *value,
+                   const void *params)
+{
+  (void)key;
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup(lines);
+}
+
+static struct config_line_t *
+linelist_s_kv_encode(const char *key, const void *value,
+                     const void *params)
+{
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup_and_filter(lines, key);
+}
+
+static void
+linelist_clear(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **lines = target;
+  config_free_lines(*lines); // sets it to NULL
+}
+
+static bool
+linelist_eq(const void *a, const void *b, const void *params)
+{
+  (void)params;
+  const config_line_t *lines_a = *(const config_line_t **)a;
+  const config_line_t *lines_b = *(const config_line_t **)b;
+  return config_lines_eq(lines_a, lines_b);
+}
+
+static int
+linelist_copy(void *target, const void *value, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  const config_line_t *val = *(const config_line_t **)value;
+  config_free_lines(*ptr);
+  *ptr = config_lines_dup(val);
+  return 0;
+}
+
+static void
+linelist_mark_fragile(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  if (*ptr)
+    (*ptr)->fragile = 1;
+}
+
+static const var_type_fns_t linelist_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_v_fns = {
+  .kv_parse = linelist_kv_virt_noparse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_s_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_s_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+};
+
+/////
+// CONFIG_TYPE_ROUTERSET
+//
+// XXXX This type is not implemented here, since routerset_t is not available
+// XXXX to this module.
+/////
+
+/////
+// CONFIG_TYPE_OBSOLETE
+//
+// Used to indicate an obsolete option.
+//
+// XXXX This is not a type, and should be handled at a higher level of
+// XXXX abstraction.
+/////
+
+static int
+ignore_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)target;
+  (void)value;
+  (void)errmsg;
+  (void)params;
+  // XXXX move this to a higher level, once such a level exists.
+  log_warn(LD_GENERAL, "Skipping obsolete configuration option%s%s%s",
+           key && *key ? " \"" : "",
+           key && *key ? key : "",
+           key && *key ? "\"." : ".");
+  return 0;
+}
+
+static char *
+ignore_encode(const void *value, const void *params)
+{
+  (void)value;
+  (void)params;
+  return NULL;
+}
+
+static const var_type_fns_t ignore_fns = {
+  .parse = ignore_parse,
+  .encode = ignore_encode,
+};
+
+/**
+ * Table mapping conf_type_t values to var_type_def_t objects.
+ **/
+static const var_type_def_t type_definitions_table[] = {
+  [CONFIG_TYPE_STRING] = { .name="String", .fns=&string_fns },
+  [CONFIG_TYPE_FILENAME] = { .name="Filename", .fns=&string_fns },
+  [CONFIG_TYPE_INT] = { .name="SignedInteger", .fns=&int_fns,
+                        .params=&INT_PARSE_UNRESTRICTED },
+  [CONFIG_TYPE_POSINT] = { .name="Integer", .fns=&int_fns,
+                           .params=&INT_PARSE_POSINT },
+  [CONFIG_TYPE_UINT64] = { .name="Integer", .fns=&uint64_fns, },
+  [CONFIG_TYPE_MEMUNIT] = { .name="DataSize", .fns=&memunit_fns,
+                            .params=&memory_units },
+  [CONFIG_TYPE_INTERVAL] = { .name="TimeInterval", .fns=&interval_fns,
+                             .params=&time_units },
+  [CONFIG_TYPE_MSEC_INTERVAL] = { .name="TimeMsecInterval",
+                                  .fns=&interval_fns,
+                                  .params=&time_msec_units },
+  [CONFIG_TYPE_DOUBLE] = { .name="Float", .fns=&double_fns, },
+  [CONFIG_TYPE_BOOL] = { .name="Boolean", .fns=&enum_fns,
+                         .params=&enum_table_bool },
+  [CONFIG_TYPE_AUTOBOOL] = { .name="Boolean+Auto", .fns=&enum_fns,
+                             .params=&enum_table_autobool },
+  [CONFIG_TYPE_ISOTIME] = { .name="Time", .fns=&time_fns, },
+  [CONFIG_TYPE_CSV] = { .name="CommaList", .fns=&csv_fns, },
+  [CONFIG_TYPE_CSV_INTERVAL] = { .name="TimeInterval",
+                                 .fns=&legacy_csv_interval_fns, },
+  [CONFIG_TYPE_LINELIST] = { .name="LineList", .fns=&linelist_fns,
+                             .flags=CFLG_NOREPLACE },
+  /*
+   * A "linelist_s" is a derived view of a linelist_v: inspecting
+   * it gets part of a linelist_v, and setting it adds to the linelist_v.
+   */
+  [CONFIG_TYPE_LINELIST_S] = { .name="Dependent", .fns=&linelist_s_fns,
+                               .flags=CFLG_NOREPLACE|
+                               /* The operations we disable here are
+                                * handled by the linelist_v. */
+                               CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP },
+  [CONFIG_TYPE_LINELIST_V] = { .name="Virtual", .fns=&linelist_v_fns,
+                               .flags=CFLG_NOREPLACE|CFLG_NOSET },
+  [CONFIG_TYPE_OBSOLETE] = {
+         .name="Obsolete", .fns=&ignore_fns,
+         .flags=CFLG_GROUP_OBSOLETE,
+  }
+};
+
+/**
+ * Return a pointer to the var_type_def_t object for the given
+ * config_type_t value, or NULL if no such type definition exists.
+ **/
+const var_type_def_t *
+lookup_type_def(config_type_t type)
+{
+  int t = type;
+  tor_assert(t >= 0);
+  if (t >= (int)ARRAY_LENGTH(type_definitions_table))
+    return NULL;
+  return &type_definitions_table[t];
+}
diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h
new file mode 100644
index 0000000000..ecf040529e
--- /dev/null
+++ b/src/lib/confmgt/type_defs.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.h
+ * @brief Header for lib/confmgt/type_defs.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H
+#define TOR_LIB_CONFMGT_TYPE_DEFS_H
+
+const struct var_type_def_t *lookup_type_def(config_type_t type);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */
diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c
new file mode 100644
index 0000000000..ce11a69379
--- /dev/null
+++ b/src/lib/confmgt/typedvar.c
@@ -0,0 +1,228 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.c
+ * @brief Functions for accessing a pointer as an object of a given type.
+ *
+ * These functions represent a low-level API for accessing a typed variable.
+ * They are used in the configuration system to examine and set fields in
+ * configuration objects used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/util_string.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+
+/**
+ * Try to parse a string in <b>value</b> that encodes an object of the type
+ * defined by <b>def</b>. If not NULL, <b>key</b> is the name of the option,
+ * which may be used for logging.
+ *
+ * On success, adjust the lvalue pointed to by <b>target</b> to hold that
+ * value, and return 0.  On failure, set *<b>errmsg</b> to a newly allocated
+ * string holding an error message, and return -1.
+ **/
+int
+typed_var_assign(void *target, const char *value, char **errmsg,
+                 const var_type_def_t *def, const char *key)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  // clear old value if needed.
+  typed_var_free(target, def);
+
+  tor_assert(def->fns->parse);
+  return def->fns->parse(target, value, errmsg, def->params, key);
+}
+
+/**
+ * Try to parse a single line from the head of<b>line</b> that encodes an
+ * object of the type defined in <b>def</b>. On success and failure, behave as
+ * typed_var_assign().
+ *
+ * All types for which keys are significant should use this function.
+ *
+ * Note that although multiple lines may be provided in <b>line</b>,
+ * only the first one is handled by this function.
+ **/
+int
+typed_var_kvassign(void *target, const config_line_t *line,
+                      char **errmsg, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+
+  if (def->fns->kv_parse) {
+    // We do _not_ free the old value here, since linelist options
+    // sometimes have append semantics.
+    return def->fns->kv_parse(target, line, errmsg, def->params);
+  }
+
+  return typed_var_assign(target, line->value, errmsg, def, line->key);
+}
+
+/**
+ * Release storage held by a variable in <b>target</b> of type defined by
+ * <b>def</b>, and set <b>target</b> to a reasonable default.
+ **/
+void
+typed_var_free(void *target, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return; // LCOV_EXCL_LINE
+  if (def->fns->clear) {
+    def->fns->clear(target, def->params);
+  }
+}
+
+/**
+ * Encode a value of type <b>def</b> pointed to by <b>value</b>, and return
+ * its result in a newly allocated string.  The string may need to be escaped.
+ *
+ * Returns NULL if this option has a NULL value, or on internal error.
+ **/
+char *
+typed_var_encode(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  tor_assert(def->fns->encode);
+  return def->fns->encode(value, def->params);
+}
+
+/**
+ * As typed_var_encode(), but returns a newly allocated config_line_t
+ * object.  The provided <b>key</b> is used as the key of the lines, unless
+ * the type is one (line a linelist) that encodes its own keys.
+ *
+ * This function may return a list of multiple lines.
+ *
+ * Returns NULL if there are no lines to encode, or on internal error.
+ */
+config_line_t *
+typed_var_kvencode(const char *key, const void *value,
+                      const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  if (def->fns->kv_encode) {
+    return def->fns->kv_encode(key, value, def->params);
+  }
+  char *encoded_value = typed_var_encode(value, def);
+  if (!encoded_value)
+    return NULL;
+
+  config_line_t *result = tor_malloc_zero(sizeof(config_line_t));
+  result->key = tor_strdup(key);
+  result->value = encoded_value;
+  return result;
+}
+
+/**
+ * Set <b>dest</b> to contain the same value as <b>src</b>. Both types
+ * must be as defined by <b>def</b>.
+ *
+ * Return 0 on success, and -1 on failure.
+ **/
+int
+typed_var_copy(void *dest, const void *src, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  if (def->fns->copy) {
+    // If we have been provided a copy fuction, use it.
+    return def->fns->copy(dest, src, def);
+  }
+
+  // Otherwise, encode 'src' and parse the result into 'def'.
+  char *enc = typed_var_encode(src, def);
+  if (!enc) {
+    typed_var_free(dest, def);
+    return 0;
+  }
+  char *err = NULL;
+  int rv = typed_var_assign(dest, enc, &err, def, NULL);
+  if (BUG(rv < 0)) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s",
+             escaped(enc), def->name, err?err:"");
+    // LCOV_EXCL_STOP
+  }
+  tor_free(err);
+  tor_free(enc);
+  return rv;
+}
+
+/**
+ * Return true if <b>a</b> and <b>b</b> are semantically equivalent.
+ * Both types must be as defined by <b>def</b>.
+ **/
+bool
+typed_var_eq(const void *a, const void *b, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->eq) {
+    // Use a provided eq function if we got one.
+    return def->fns->eq(a, b, def->params);
+  }
+
+  // Otherwise, encode the values and compare them.
+  char *enc_a = typed_var_encode(a, def);
+  char *enc_b = typed_var_encode(b, def);
+  bool eq = !strcmp_opt(enc_a,enc_b);
+  tor_free(enc_a);
+  tor_free(enc_b);
+  return eq;
+}
+
+/**
+ * Check whether <b>value</b> encodes a valid value according to the
+ * type definition in <b>def</b>.
+ */
+bool
+typed_var_ok(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->ok)
+    return def->fns->ok(value, def->params);
+
+  return true;
+}
+
+/**
+ * Mark <b>value</b> -- a variable that ordinarily would be extended by
+ * assignment -- as "fragile", so that it will get replaced by the next
+ * assignment instead.
+ **/
+void
+typed_var_mark_fragile(void *value, const var_type_def_t *def)
+{
+  if (BUG(!def)) {
+    return; // LCOV_EXCL_LINE
+  }
+
+  if (def->fns->mark_fragile)
+    def->fns->mark_fragile(value, def->params);
+}
diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h
new file mode 100644
index 0000000000..4382613833
--- /dev/null
+++ b/src/lib/confmgt/typedvar.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.h
+ * @brief Header for lib/confmgt/typedvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H
+#define TOR_LIB_CONFMGT_TYPEDVAR_H
+
+#include <stdbool.h>
+
+enum config_type_t;
+struct config_line_t;
+
+typedef struct var_type_fns_t var_type_fns_t;
+typedef struct var_type_def_t var_type_def_t;
+
+int typed_var_assign(void *target, const char *value, char **errmsg,
+                     const var_type_def_t *def, const char *key);
+void typed_var_free(void *target, const var_type_def_t *def);
+char *typed_var_encode(const void *value, const var_type_def_t *def);
+int typed_var_copy(void *dest, const void *src, const var_type_def_t *def);
+bool typed_var_eq(const void *a, const void *b, const var_type_def_t *def);
+bool typed_var_ok(const void *value, const var_type_def_t *def);
+
+int typed_var_kvassign(void *target, const struct config_line_t *line,
+                           char **errmsg, const var_type_def_t *def);
+struct config_line_t *typed_var_kvencode(const char *key, const void *value,
+                                            const var_type_def_t *def);
+
+void typed_var_mark_fragile(void *value, const var_type_def_t *def);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */
diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c
new file mode 100644
index 0000000000..c3ed8285a4
--- /dev/null
+++ b/src/lib/confmgt/unitparse.c
@@ -0,0 +1,206 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.c
+ * @brief Functions for parsing values with units from a configuration file.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/util_string.h"
+
+#include <string.h>
+
+/** Table to map the names of memory units to the number of bytes they
+ * contain. */
+const struct unit_table_t memory_units[] = {
+  { "",          1 },
+  { "b",         1<< 0 },
+  { "byte",      1<< 0 },
+  { "bytes",     1<< 0 },
+  { "kb",        1<<10 },
+  { "kbyte",     1<<10 },
+  { "kbytes",    1<<10 },
+  { "kilobyte",  1<<10 },
+  { "kilobytes", 1<<10 },
+  { "kilobits",  1<<7  },
+  { "kilobit",   1<<7  },
+  { "kbits",     1<<7  },
+  { "kbit",      1<<7  },
+  { "m",         1<<20 },
+  { "mb",        1<<20 },
+  { "mbyte",     1<<20 },
+  { "mbytes",    1<<20 },
+  { "megabyte",  1<<20 },
+  { "megabytes", 1<<20 },
+  { "megabits",  1<<17 },
+  { "megabit",   1<<17 },
+  { "mbits",     1<<17 },
+  { "mbit",      1<<17 },
+  { "gb",        1<<30 },
+  { "gbyte",     1<<30 },
+  { "gbytes",    1<<30 },
+  { "gigabyte",  1<<30 },
+  { "gigabytes", 1<<30 },
+  { "gigabits",  1<<27 },
+  { "gigabit",   1<<27 },
+  { "gbits",     1<<27 },
+  { "gbit",      1<<27 },
+  { "tb",        UINT64_C(1)<<40 },
+  { "tbyte",     UINT64_C(1)<<40 },
+  { "tbytes",    UINT64_C(1)<<40 },
+  { "terabyte",  UINT64_C(1)<<40 },
+  { "terabytes", UINT64_C(1)<<40 },
+  { "terabits",  UINT64_C(1)<<37 },
+  { "terabit",   UINT64_C(1)<<37 },
+  { "tbits",     UINT64_C(1)<<37 },
+  { "tbit",      UINT64_C(1)<<37 },
+  { NULL, 0 },
+};
+
+/** Table to map the names of time units to the number of seconds they
+ * contain. */
+const struct unit_table_t time_units[] = {
+  { "",         1 },
+  { "second",   1 },
+  { "seconds",  1 },
+  { "minute",   60 },
+  { "minutes",  60 },
+  { "hour",     60*60 },
+  { "hours",    60*60 },
+  { "day",      24*60*60 },
+  { "days",     24*60*60 },
+  { "week",     7*24*60*60 },
+  { "weeks",    7*24*60*60 },
+  { "month",    2629728, }, /* about 30.437 days */
+  { "months",   2629728, },
+  { NULL, 0 },
+};
+
+/** Table to map the names of time units to the number of milliseconds
+ * they contain. */
+const struct unit_table_t time_msec_units[] = {
+  { "",         1 },
+  { "msec",     1 },
+  { "millisecond", 1 },
+  { "milliseconds", 1 },
+  { "second",   1000 },
+  { "seconds",  1000 },
+  { "minute",   60*1000 },
+  { "minutes",  60*1000 },
+  { "hour",     60*60*1000 },
+  { "hours",    60*60*1000 },
+  { "day",      24*60*60*1000 },
+  { "days",     24*60*60*1000 },
+  { "week",     7*24*60*60*1000 },
+  { "weeks",    7*24*60*60*1000 },
+  { NULL, 0 },
+};
+
+/** Parse a string <b>val</b> containing a number, zero or more
+ * spaces, and an optional unit string.  If the unit appears in the
+ * table <b>u</b>, then multiply the number by the unit multiplier.
+ * On success, set *<b>ok</b> to 1 and return this product.
+ * Otherwise, set *<b>ok</b> to 0.
+ */
+uint64_t
+config_parse_units(const char *val, const unit_table_t *u, int *ok)
+{
+  uint64_t v = 0;
+  double d = 0;
+  int use_float = 0;
+  char *cp;
+
+  tor_assert(ok);
+
+  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
+  if (!*ok || (cp && *cp == '.')) {
+    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
+    if (!*ok)
+      goto done;
+    use_float = 1;
+  }
+
+  if (BUG(!cp)) {
+    // cp should always be non-NULL if the parse operation succeeds.
+
+    // LCOV_EXCL_START
+    *ok = 1;
+    v = use_float ? ((uint64_t)d) :  v;
+    goto done;
+    // LCOV_EXCL_STOP
+  }
+
+  cp = (char*) eat_whitespace(cp);
+
+  for ( ;u->unit;++u) {
+    if (!strcasecmp(u->unit, cp)) {
+      if (use_float)
+        v = (uint64_t)(u->multiplier * d);
+      else
+        v *= u->multiplier;
+      *ok = 1;
+      goto done;
+    }
+  }
+  log_warn(LD_CONFIG, "Unknown unit '%s'.", cp);
+  *ok = 0;
+ done:
+
+  if (*ok)
+    return v;
+  else
+    return 0;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
+ * and return the number of bytes specified.  Otherwise, set
+ * *<b>ok</b> to false and return 0. */
+uint64_t
+config_parse_memunit(const char *s, int *ok)
+{
+  uint64_t u = config_parse_units(s, memory_units, ok);
+  return u;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * time in milliseconds.  On success, set *<b>ok</b> to true and return
+ * the number of milliseconds in the provided interval.  Otherwise, set
+ * *<b>ok</b> to 0 and return -1. */
+int
+config_parse_msec_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_msec_units, ok);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of time.
+ * On success, set *<b>ok</b> to true and return the number of seconds in
+ * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
+ */
+int
+config_parse_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_units, ok);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h
new file mode 100644
index 0000000000..216361a7d4
--- /dev/null
+++ b/src/lib/confmgt/unitparse.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.h
+ * @brief Header for lib/confmgt/unitparse.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_UNITPARSE_H
+#define TOR_LIB_CONFMGT_UNITPARSE_H
+
+#include <lib/cc/torint.h>
+
+/** Mapping from a unit name to a multiplier for converting that unit into a
+ * base unit.  Used by config_parse_unit. */
+typedef struct unit_table_t {
+  const char *unit; /**< The name of the unit */
+  uint64_t multiplier; /**< How many of the base unit appear in this unit */
+} unit_table_t;
+
+extern const unit_table_t memory_units[];
+extern const unit_table_t time_units[];
+extern const struct unit_table_t time_msec_units[];
+
+uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok);
+
+uint64_t config_parse_memunit(const char *s, int *ok);
+int config_parse_msec_interval(const char *s, int *ok);
+int config_parse_interval(const char *s, int *ok);
+
+#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */
diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h
new file mode 100644
index 0000000000..aa9ded39e9
--- /dev/null
+++ b/src/lib/confmgt/var_type_def_st.h
@@ -0,0 +1,170 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file var_type_def_st.h
+ * @brief Structure declarations for typedvar type definitions.
+ *
+ * This structure is used for defining new variable types.  If you are not
+ * defining a new variable type for use by the configuration management
+ * system, you don't need this structure.
+ *
+ * For defining new variables, see the types in conftypes.h.
+ *
+ * For data-driven access to configuration variables, see the other members of
+ * lib/confmgt/.
+ *
+ * STATUS NOTE: It is not yet possible to actually define new variables
+ *   outside of config.c, and many of the types that will eventually be used
+ *   to do so are not yet moved. This will change as more of #29211 is
+ *   completed.
+ **/
+
+#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+
+#include <stdbool.h>
+
+struct config_line_t;
+
+/**
+ * A structure full of functions pointers to implement a variable type.
+ *
+ * Every type MUST implement parse or kv_parse and encode or kv_encode;
+ * the other functions pointers MAY be NULL.
+ *
+ * All functions here take a <b>params</b> argument, whose value
+ * is determined by the type definition.  Two types may have the
+ * same functions, but differ only in parameters.
+ *
+ * Implementation considerations: If "" encodes a valid value for a type, try
+ * to make sure that it encodes the same thing as the default value for the
+ * type (that is, the value that is set by config_clear() or memset(0)). If
+ * this is not the case, you need to make extra certain that your parse/encode
+ * implementations preserve the NULL/"" distinction.
+ **/
+struct var_type_fns_t {
+  /**
+   * Try to parse a string in <b>value</b> that encodes an object of this
+   * type.  On success, adjust the lvalue pointed to by <b>target</b> to hold
+   * that value, and return 0.  On failure, set *<b>errmsg</b> to a newly
+   * allocated string holding an error message, and return -1.
+   *
+   * If not NULL, <b>key</b> is the name of the option, which may be used for
+   * logging.
+   **/
+  int (*parse)(void *target, const char *value, char **errmsg,
+               const void *params, const char *key);
+  /**
+   * Try to parse a single line from the head of<b>line</b> that encodes
+   * an object of this type.  On success and failure, behave as in the parse()
+   * function.
+   *
+   * If this function is absent, it is implemented in terms of parse().
+   *
+   * All types for which keys are significant should use this method. For
+   * example, a "linelist" type records the actual keys that are given
+   * for each line, and so should use this method.
+   *
+   * Note that although multiple lines may be provided in <b>line</b>,
+   * only the first one should be handled by this function.
+   **/
+  int (*kv_parse)(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params);
+  /**
+   * Encode a value pointed to by <b>value</b> and return its result
+   * in a newly allocated string.  The string may need to be escaped.
+   *
+   * If this function is absent, it is implemented in terms of kv_encode().
+   *
+   * Returns NULL if this option has a NULL value, or on internal error.
+   *
+   * Requirement: all strings generated by encode() should produce a
+   * semantically equivalent value when given to parse().
+   **/
+  char *(*encode)(const void *value, const void *params);
+  /**
+   * As encode(), but returns a newly allocated config_line_t object.  The
+   * provided <b>key</b> is used as the key of the lines, unless the type is
+   * one that encodes its own keys.
+   *
+   * Unlike kv_parse(), this function will return a list of multiple lines,
+   * if <b>value</b> is such that it must be encoded by multiple lines.
+   *
+   * Returns NULL if there are no lines to encode, or on internal error.
+   *
+   * If this function is absent, it is implemented in terms of encode().
+   **/
+  struct config_line_t *(*kv_encode)(const char *key, const void *value,
+                                     const void *params);
+  /**
+   * Free all storage held in <b>arg</b>, and set <b>arg</b> to a default
+   * value -- usually zero or NULL.
+   *
+   * If this function is absent, the default implementation does nothing.
+   **/
+  void (*clear)(void *arg, const void *params);
+  /**
+   * Return true if <b>a</b> and <b>b</b> hold the same value, and false
+   * otherwise.
+   *
+   * If this function is absent, it is implemented by encoding both a and
+   * b and comparing their encoded strings for equality.
+   **/
+  bool (*eq)(const void *a, const void *b, const void *params);
+  /**
+   * Try to copy the value from <b>value</b> into <b>target</b>.
+   * On success return 0; on failure return -1.
+   *
+   * If this function is absent, it is implemented by encoding the value
+   * into a string, and then parsing it into the target.
+   **/
+  int (*copy)(void *target, const void *value, const void *params);
+  /**
+   * Check whether <b>value</b> holds a valid value according to the
+   * rules of this type; return true if it does and false if it doesn't.
+   *
+   * The default implementation for this function assumes that all
+   * values are valid.
+   **/
+  bool (*ok)(const void *value, const void *params);
+  /**
+   * Mark a value of this variable as "fragile", so that future attempts to
+   * assign to this variable will replace rather than extending it.
+   *
+   * The default implementation for this function does nothing.
+   *
+   * Only meaningful for types with is_cumulative set.
+   **/
+  void (*mark_fragile)(void *value, const void *params);
+};
+
+/**
+ * A structure describing a type that can be manipulated with the typedvar_*
+ * functions.
+ **/
+struct var_type_def_t {
+  /**
+   * The name of this type. Should not include spaces. Used for
+   * debugging, log messages, and the controller API. */
+  const char *name;
+  /**
+   * A function table for this type.
+   */
+  const struct var_type_fns_t *fns;
+  /**
+   * A pointer to a value that should be passed as the 'params' argument when
+   * calling the functions in this type's function table.
+   */
+  const void *params;
+  /**
+   * A bitwise OR of one or more VTFLAG_* values, describing properties
+   * for all values of this type.
+   **/
+  uint32_t flags;
+};
+
+#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */
diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include
index 90de5eda40..81507527d3 100644
--- a/src/lib/container/.may_include
+++ b/src/lib/container/.may_include
@@ -7,12 +7,9 @@ lib/malloc/*.h
 lib/err/*.h
 lib/smartlist_core/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
 lib/intmath/*.h
 lib/log/*.h
 
-# XXXX I am unsure about this one. It's only here for buffers.c
-lib/time/*.h
-
-ht.h
-siphash.h
+ext/ht.h
+ext/siphash.h
diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h
index 910d5fea65..45992796af 100644
--- a/src/lib/container/bitarray.h
+++ b/src/lib/container/bitarray.h
@@ -83,4 +83,4 @@ bitarray_is_set(bitarray_t *b, int bit)
   return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_BITARRAY_H) */
diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c
index 9aa9b1ee56..8c61db81d6 100644
--- a/src/lib/container/bloomfilt.c
+++ b/src/lib/container/bloomfilt.c
@@ -14,7 +14,7 @@
 #include "lib/container/bloomfilt.h"
 #include "lib/intmath/bits.h"
 #include "lib/log/util_bug.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** How many bloom-filter bits we set per address. This is twice the
  * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit
diff --git a/src/lib/container/include.am b/src/lib/container/include.am
index e6492098b5..00d7b8e587 100644
--- a/src/lib/container/include.am
+++ b/src/lib/container/include.am
@@ -5,10 +5,11 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-container-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_container_a_SOURCES =			\
 	src/lib/container/bloomfilt.c			\
-	src/lib/container/buffers.c			\
 	src/lib/container/map.c				\
+	src/lib/container/namemap.c			\
 	src/lib/container/order.c			\
 	src/lib/container/smartlist.c
 
@@ -17,11 +18,13 @@ src_lib_libtor_container_testing_a_SOURCES = \
 src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/container/bitarray.h			\
 	src/lib/container/bloomfilt.h			\
-	src/lib/container/buffers.h			\
 	src/lib/container/handles.h			\
 	src/lib/container/map.h				\
+	src/lib/container/namemap.h			\
+	src/lib/container/namemap_st.h			\
 	src/lib/container/order.h			\
 	src/lib/container/smartlist.h
diff --git a/src/lib/container/map.c b/src/lib/container/map.c
index d213ad50bf..fde33d6ace 100644
--- a/src/lib/container/map.c
+++ b/src/lib/container/map.c
@@ -21,7 +21,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "ht.h"
+#include "ext/ht.h"
 
 /** Helper: Declare an entry type and a map type to implement a mapping using
  * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
diff --git a/src/lib/container/map.h b/src/lib/container/map.h
index a2d1b01d12..9da1d3072c 100644
--- a/src/lib/container/map.h
+++ b/src/lib/container/map.h
@@ -15,7 +15,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "lib/cc/torint.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
   typedef struct maptype maptype;                                       \
@@ -258,4 +258,4 @@ void* strmap_remove_lc(strmap_t *map, const char *key);
     return digestmap_iter_done((digestmap_iter_t*)iter);                \
   }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_MAP_H) */
diff --git a/src/lib/container/namemap.c b/src/lib/container/namemap.c
new file mode 100644
index 0000000000..a90057b32c
--- /dev/null
+++ b/src/lib/container/namemap.c
@@ -0,0 +1,184 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/container/smartlist.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "ext/siphash.h"
+
+#include <string.h>
+
+/** Helper for namemap hashtable implementation: compare two entries. */
+static inline int
+mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b)
+{
+  return !strcmp(a->name, b->name);
+}
+
+/** Helper for namemap hashtable implementation: hash an entry. */
+static inline unsigned
+mapped_name_hash(const mapped_name_t *a)
+{
+  return (unsigned) siphash24g(a->name, strlen(a->name));
+}
+
+HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq)
+HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** Set up an uninitialized <b>map</b>. */
+void
+namemap_init(namemap_t *map)
+{
+  memset(map, 0, sizeof(*map));
+  HT_INIT(namemap_ht, &map->ht);
+  map->names = smartlist_new();
+}
+
+/** Return the name that <b>map</b> associates with a given <b>id</b>, or
+ * NULL if there is no such name. */
+const char *
+namemap_get_name(const namemap_t *map, unsigned id)
+{
+  if (map->names && id < (unsigned)smartlist_len(map->names)) {
+    mapped_name_t *name = smartlist_get(map->names, (int)id);
+    return name->name;
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * Return the name that <b>map</b> associates with a given <b>id</b>, or a
+ * pointer to a statically allocated string describing the value of <b>id</b>
+ * if no such name exists.
+ **/
+const char *
+namemap_fmt_name(const namemap_t *map, unsigned id)
+{
+  static char buf[32];
+
+  const char *name = namemap_get_name(map, id);
+  if (name)
+    return name;
+
+  tor_snprintf(buf, sizeof(buf), "{%u}", id);
+
+  return buf;
+}
+
+/**
+ * Helper: As namemap_get_id(), but requires that <b>name</b> is
+ * <b>namelen</b> charaters long, and that <b>namelen</b> is no more than
+ * MAX_NAMEMAP_NAME_LEN.
+ */
+static unsigned
+namemap_get_id_unchecked(const namemap_t *map,
+                         const char *name,
+                         size_t namelen)
+{
+  union {
+    mapped_name_t n;
+    char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1];
+  } u;
+  memcpy(u.n.name, name, namelen);
+  u.n.name[namelen] = 0;
+  const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n);
+  if (found) {
+    tor_assert(map->names);
+    tor_assert(smartlist_get(map->names, found->intval) == found);
+    return found->intval;
+  }
+
+  return NAMEMAP_ERR;
+}
+
+/**
+ * Return the identifier currently associated by <b>map</b> with the name
+ * <b>name</b>, or NAMEMAP_ERR if no such identifier exists.
+ **/
+unsigned
+namemap_get_id(const namemap_t *map,
+               const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  return namemap_get_id_unchecked(map, name, namelen);
+}
+
+/**
+ * Return the identifier associated by <b>map</b> with the name
+ * <b>name</b>, allocating a new identifier in <b>map</b> if none exists.
+ *
+ * Return NAMEMAP_ERR if <b>name</b> is too long, or if there are no more
+ * identifiers we can allocate.
+ **/
+unsigned
+namemap_get_or_create_id(namemap_t *map,
+                         const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    map->names = smartlist_new();
+
+  unsigned found = namemap_get_id_unchecked(map, name, namelen);
+  if (found != NAMEMAP_ERR)
+    return found;
+
+  unsigned new_id = (unsigned)smartlist_len(map->names);
+  if (new_id == NAMEMAP_ERR)
+    return NAMEMAP_ERR; /* Can't allocate any more. */
+
+  mapped_name_t *insert = tor_malloc_zero(
+                       offsetof(mapped_name_t, name) + namelen + 1);
+  memcpy(insert->name, name, namelen+1);
+  insert->intval = new_id;
+
+  HT_INSERT(namemap_ht, &map->ht, insert);
+  smartlist_add(map->names, insert);
+
+  return new_id;
+}
+
+/** Return the number of entries in 'names' */
+size_t
+namemap_get_size(const namemap_t *map)
+{
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    return 0;
+
+  return smartlist_len(map->names);
+}
+
+/**
+ * Release all storage held in <b>map</b>.
+ */
+void
+namemap_clear(namemap_t *map)
+{
+  if (!map)
+    return;
+
+  HT_CLEAR(namemap_ht, &map->ht);
+  if (map->names) {
+    SMARTLIST_FOREACH(map->names, mapped_name_t *, n,
+                      tor_free(n));
+    smartlist_free(map->names);
+  }
+  memset(map, 0, sizeof(*map));
+}
diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h
new file mode 100644
index 0000000000..b96bc13f3a
--- /dev/null
+++ b/src/lib/container/namemap.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NAMEMAP_H
+#define TOR_NAMEMAP_H
+
+/**
+ * \file namemap.h
+ *
+ * \brief Header for namemap.c
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+#include <stddef.h>
+
+typedef struct namemap_t namemap_t;
+
+/** Returned in place of an identifier when an error occurs. */
+#define NAMEMAP_ERR UINT_MAX
+
+void namemap_init(namemap_t *map);
+const char *namemap_get_name(const namemap_t *map, unsigned id);
+const char *namemap_fmt_name(const namemap_t *map, unsigned id);
+unsigned namemap_get_id(const namemap_t *map,
+                        const char *name);
+unsigned namemap_get_or_create_id(namemap_t *map,
+                                  const char *name);
+size_t namemap_get_size(const namemap_t *map);
+void namemap_clear(namemap_t *map);
+
+#endif /* !defined(TOR_NAMEMAP_H) */
diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h
new file mode 100644
index 0000000000..5008fd5855
--- /dev/null
+++ b/src/lib/container/namemap_st.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NAMEMAP_ST_H
+#define NAMEMAP_ST_H
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+struct smartlist_t;
+
+/** Longest allowed name that's allowed in a namemap_t. */
+#define MAX_NAMEMAP_NAME_LEN 128
+
+/** An entry inside a namemap_t. Maps a string to a numeric identifier. */
+typedef struct mapped_name_t {
+  HT_ENTRY(mapped_name_t) node;
+  unsigned intval;
+  char name[FLEXIBLE_ARRAY_MEMBER];
+} mapped_name_t;
+
+/** A structure that allocates small numeric identifiers for names and maps
+ * back and forth between them.  */
+struct namemap_t {
+  HT_HEAD(namemap_ht, mapped_name_t) ht;
+  struct smartlist_t *names;
+};
+
+/** Macro to initialize a namemap. */
+#define NAMEMAP_INIT() { HT_INITIALIZER(), NULL }
+
+#endif /* !defined(NAMEMAP_ST_H) */
diff --git a/src/lib/container/order.h b/src/lib/container/order.h
index a176d6d8a6..3f2fd054a0 100644
--- a/src/lib/container/order.h
+++ b/src/lib/container/order.h
@@ -57,4 +57,4 @@ third_quartile_uint32(uint32_t *array, int n_elements)
   return find_nth_uint32(array, n_elements, (n_elements*3)/4);
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_ORDER_H) */
diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c
index 3ab2797d68..2b71c11287 100644
--- a/src/lib/container/smartlist.c
+++ b/src/lib/container/smartlist.c
@@ -678,7 +678,7 @@ smartlist_sort_pointers(smartlist_t *sl)
 static inline void
 smartlist_heapify(smartlist_t *sl,
                   int (*compare)(const void *a, const void *b),
-                  int idx_field_offset,
+                  ptrdiff_t idx_field_offset,
                   int idx)
 {
   while (1) {
@@ -725,7 +725,7 @@ smartlist_heapify(smartlist_t *sl,
 void
 smartlist_pqueue_add(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset,
+                     ptrdiff_t idx_field_offset,
                      void *item)
 {
   int idx;
@@ -754,7 +754,7 @@ smartlist_pqueue_add(smartlist_t *sl,
 void *
 smartlist_pqueue_pop(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset)
+                     ptrdiff_t idx_field_offset)
 {
   void *top;
   tor_assert(sl->num_used);
@@ -778,7 +778,7 @@ smartlist_pqueue_pop(smartlist_t *sl,
 void
 smartlist_pqueue_remove(smartlist_t *sl,
                         int (*compare)(const void *a, const void *b),
-                        int idx_field_offset,
+                        ptrdiff_t idx_field_offset,
                         void *item)
 {
   int idx = IDX_OF_ITEM(item);
@@ -802,7 +802,7 @@ smartlist_pqueue_remove(smartlist_t *sl,
 void
 smartlist_pqueue_assert_ok(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset)
+                           ptrdiff_t idx_field_offset)
 {
   int i;
   for (i = sl->num_used - 1; i >= 0; --i) {
diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h
index 77682db03e..25638e4b22 100644
--- a/src/lib/container/smartlist.h
+++ b/src/lib/container/smartlist.h
@@ -13,6 +13,7 @@
  **/
 
 #include <stdarg.h>
+#include <stddef.h>
 
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -72,18 +73,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
 
 void smartlist_pqueue_add(smartlist_t *sl,
                           int (*compare)(const void *a, const void *b),
-                          int idx_field_offset,
+                          ptrdiff_t idx_field_offset,
                           void *item);
 void *smartlist_pqueue_pop(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset);
+                           ptrdiff_t idx_field_offset);
 void smartlist_pqueue_remove(smartlist_t *sl,
                              int (*compare)(const void *a, const void *b),
-                             int idx_field_offset,
+                             ptrdiff_t idx_field_offset,
                              void *item);
 void smartlist_pqueue_assert_ok(smartlist_t *sl,
                                 int (*compare)(const void *a, const void *b),
-                                int idx_field_offset);
+                                ptrdiff_t idx_field_offset);
 
 char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
                              size_t *len_out) ATTR_MALLOC;
@@ -165,4 +166,4 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
   }                                             \
   STMT_END
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_H) */
diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include
index a0fa4ec05c..0739699686 100644
--- a/src/lib/crypt_ops/.may_include
+++ b/src/lib/crypt_ops/.may_include
@@ -12,7 +12,8 @@ lib/malloc/*.h
 lib/intmath/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/thread/*.h
 lib/log/*.h
 
@@ -21,4 +22,4 @@ trunnel/pwbox.h
 keccak-tiny/*.h
 ed25519/*.h
 
-siphash.h
+ext/siphash.h
diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c
index 2f985d4512..64564892ad 100644
--- a/src/lib/crypt_ops/aes_openssl.c
+++ b/src/lib/crypt_ops/aes_openssl.c
@@ -148,7 +148,7 @@ evaluate_ctr_for_aes(void)
 {
   return 0;
 }
-#else /* !(defined(USE_EVP_AES_CTR)) */
+#else /* !defined(USE_EVP_AES_CTR) */
 
 /*======================================================================*/
 /* Interface to AES code, and counter implementation */
diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h
index 9c10386c34..61ca51315f 100644
--- a/src/lib/crypt_ops/compat_openssl.h
+++ b/src/lib/crypt_ops/compat_openssl.h
@@ -45,7 +45,7 @@
    ((st) == SSL3_ST_SW_SRVR_HELLO_B))
 #define OSSL_HANDSHAKE_STATE int
 #define CONST_IF_OPENSSL_1_1_API
-#else /* !(!defined(OPENSSL_1_1_API)) */
+#else /* defined(OPENSSL_1_1_API) */
 #define STATE_IS_SW_SERVER_HELLO(st) \
   ((st) == TLS_ST_SW_SRVR_HELLO)
 #define CONST_IF_OPENSSL_1_1_API const
diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h
index cc4fbf7a41..88d63c1df2 100644
--- a/src/lib/crypt_ops/crypto_cipher.h
+++ b/src/lib/crypt_ops/crypto_cipher.h
@@ -54,4 +54,4 @@ int crypto_cipher_decrypt_with_iv(const char *key,
                                   char *to, size_t tolen,
                                   const char *from, size_t fromlen);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_CIPHER_H) */
diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h
index 061a7a3505..cd23169cd5 100644
--- a/src/lib/crypt_ops/crypto_curve25519.h
+++ b/src/lib/crypt_ops/crypto_curve25519.h
@@ -76,8 +76,8 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret);
 
 int curve25519_public_from_base64(curve25519_public_key_t *pkey,
                                   const char *input);
-int curve25519_public_to_base64(char *output,
-                                const curve25519_public_key_t *pkey);
+void curve25519_public_to_base64(char *output,
+                                 const curve25519_public_key_t *pkey);
 
 void curve25519_set_impl_params(int use_ed);
 void curve25519_init(void);
diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c
index 8c6388fd5d..8ae97373e8 100644
--- a/src/lib/crypt_ops/crypto_dh_openssl.c
+++ b/src/lib/crypt_ops/crypto_dh_openssl.c
@@ -34,7 +34,7 @@ static int tor_check_dh_key(int severity, const BIGNUM *bn);
 struct crypto_dh_t {
   DH *dh; /**< The openssl DH object */
 };
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g);
 
@@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     goto out;
   if (!DH_set0_pqg(dh, dh_p, NULL, dh_g))
     goto out;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (!(dh->p = BN_dup(p)))
     goto out;
   if (!(dh->g = BN_dup(g)))
@@ -100,7 +100,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     DH_free(dh);
   return ret;
 }
-#endif
+#endif /* 0 */
 
 /**
  * Helper: convert <b>hex<b> to a bignum, and return it.  Assert that the
@@ -202,7 +202,7 @@ crypto_dh_new(int dh_type)
     tor_free(res); // sets res to NULL.
   return res;
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 /** Create and return a new openssl DH from a given prime and generator. */
 static DH *
@@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g)
 
   if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS))
     goto err;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   res_dh->p = dh_p;
   res_dh->g = dh_g;
   res_dh->length = DH_PRIVATE_KEY_BITS;
@@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh)
              "the-universe chances really do happen.  Treating as a failure.");
     return -1;
   }
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) {
     /* LCOV_EXCL_START
      * If this happens, then openssl's DH implementation is busted. */
@@ -461,7 +461,7 @@ crypto_dh_free_(crypto_dh_t *dh)
   DH_free(dh->dh);
   tor_free(dh);
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 void
 crypto_dh_free_all_openssl(void)
diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c
index de81b87b7e..ba226f8756 100644
--- a/src/lib/crypt_ops/crypto_digest.c
+++ b/src/lib/crypt_ops/crypto_digest.c
@@ -23,171 +23,6 @@
 
 #include "lib/arch/bytes.h"
 
-#ifdef ENABLE_NSS
-DISABLE_GCC_WARNING(strict-prototypes)
-#include <pk11pub.h>
-ENABLE_GCC_WARNING(strict-prototypes)
-#else
-
-#include "lib/crypt_ops/crypto_openssl_mgt.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-
-#include <openssl/hmac.h>
-#include <openssl/sha.h>
-
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
-
-#ifdef ENABLE_NSS
-/**
- * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
- * On failure, return SEC_OID_UNKNOWN. */
-static SECOidTag
-digest_alg_to_nss_oid(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: return SEC_OID_SHA1;
-    case DIGEST_SHA256: return SEC_OID_SHA256;
-    case DIGEST_SHA512: return SEC_OID_SHA512;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return SEC_OID_UNKNOWN;
-  }
-}
-
-/* Helper: get an unkeyed digest via pk11wrap */
-static int
-digest_nss_internal(SECOidTag alg,
-                    char *digest, unsigned len_out,
-                    const char *msg, size_t msg_len)
-{
-  if (alg == SEC_OID_UNKNOWN)
-    return -1;
-  tor_assert(msg_len <= UINT_MAX);
-
-  int rv = -1;
-  SECStatus s;
-  PK11Context *ctx = PK11_CreateDigestContext(alg);
-  if (!ctx)
-    return -1;
-
-  s = PK11_DigestBegin(ctx);
-  if (s != SECSuccess)
-    goto done;
-
-  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-
-  unsigned int len = 0;
-  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
-  if (s != SECSuccess)
-    goto done;
-
-  rv = 0;
- done:
-  PK11_DestroyContext(ctx, PR_TRUE);
-  return rv;
-}
-
-/** True iff alg is implemented in our crypto library, and we want to use that
- * implementation */
-static bool
-library_supports_digest(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return true;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return false;
-  }
-}
-#endif
-
-/* Crypto digest functions */
-
-/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
- * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, -1 on failure.
- */
-MOCK_IMPL(int,
-crypto_digest,(char *digest, const char *m, size_t len))
-{
-  tor_assert(m);
-  tor_assert(digest);
-#ifdef ENABLE_NSS
-  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
-#else
-  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
-    return -1;
-  }
-#endif
-  return 0;
-}
-
-/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest256(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA256) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
-#else
-    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
-#endif
-  } else {
-    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
-/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest512(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA512) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
-#else
-    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-           != NULL);
-#endif
-  } else {
-    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
 /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
  * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
  * success, -1 on failure. */
@@ -267,485 +102,6 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg)
   }
 }
 
-/** Intermediate information about the digest of a stream of data. */
-struct crypto_digest_t {
-  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
-   /** State for the digest we're using.  Only one member of the
-    * union is usable, depending on the value of <b>algorithm</b>. Note also
-    * that space for other members might not even be allocated!
-    */
-  union {
-#ifdef ENABLE_NSS
-    PK11Context *ctx;
-#else
-    SHA_CTX sha1; /**< state for SHA1 */
-    SHA256_CTX sha2; /**< state for SHA256 */
-    SHA512_CTX sha512; /**< state for SHA512 */
-#endif
-    keccak_state sha3; /**< state for SHA3-[256,512] */
-  } d;
-};
-
-#ifdef TOR_UNIT_TESTS
-
-digest_algorithm_t
-crypto_digest_get_algorithm(crypto_digest_t *digest)
-{
-  tor_assert(digest);
-
-  return digest->algorithm;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/**
- * Return the number of bytes we need to malloc in order to get a
- * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
- * when we free one.
- */
-static size_t
-crypto_digest_alloc_bytes(digest_algorithm_t alg)
-{
-  /* Helper: returns the number of bytes in the 'f' field of 'st' */
-#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
-  /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
-                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
-  switch (alg) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.ctx);
-#else
-    case DIGEST_SHA1:
-      return END_OF_FIELD(d.sha1);
-    case DIGEST_SHA256:
-      return END_OF_FIELD(d.sha2);
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.sha512);
-#endif
-    case DIGEST_SHA3_256:
-    case DIGEST_SHA3_512:
-      return END_OF_FIELD(d.sha3);
-    default:
-      tor_assert(0); // LCOV_EXCL_LINE
-      return 0;      // LCOV_EXCL_LINE
-  }
-#undef END_OF_FIELD
-#undef STRUCT_FIELD_SIZE
-}
-
-/**
- * Internal function: create and return a new digest object for 'algorithm'.
- * Does not typecheck the algorithm.
- */
-static crypto_digest_t *
-crypto_digest_new_internal(digest_algorithm_t algorithm)
-{
-  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
-  r->algorithm = algorithm;
-
-  switch (algorithm)
-    {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
-      if (BUG(!r->d.ctx)) {
-        tor_free(r);
-        return NULL;
-      }
-      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
-        crypto_digest_free(r);
-        return NULL;
-      }
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Init(&r->d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Init(&r->d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Init(&r->d.sha512);
-      break;
-#endif
-    case DIGEST_SHA3_256:
-      keccak_digest_init(&r->d.sha3, 256);
-      break;
-    case DIGEST_SHA3_512:
-      keccak_digest_init(&r->d.sha3, 512);
-      break;
-    default:
-      tor_assert_unreached();
-    }
-
-  return r;
-}
-
-/** Allocate and return a new digest object to compute SHA1 digests.
- */
-crypto_digest_t *
-crypto_digest_new(void)
-{
-  return crypto_digest_new_internal(DIGEST_SHA1);
-}
-
-/** Allocate and return a new digest object to compute 256-bit digests
- * using <b>algorithm</b>.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
- * C_RUST_COUPLED: `crypto::digest::Sha256::default`
- */
-crypto_digest_t *
-crypto_digest256_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Allocate and return a new digest object to compute 512-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest512_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Deallocate a digest object.
- */
-void
-crypto_digest_free_(crypto_digest_t *digest)
-{
-  if (!digest)
-    return;
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
-  }
-#endif
-  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memwipe(digest, 0, bytes);
-  tor_free(digest);
-}
-
-/** Add <b>len</b> bytes from <b>data</b> to the digest object.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
- * C_RUST_COUPLED: `crypto::digest::Sha256::process`
- */
-void
-crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                        size_t len)
-{
-  tor_assert(digest);
-  tor_assert(data);
-  /* Using the SHA*_*() calls directly means we don't support doing
-   * SHA in hardware. But so far the delay of getting the question
-   * to the hardware, and hearing the answer, is likely higher than
-   * just doing it ourselves. Hashes are fast.
-   */
-  switch (digest->algorithm) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      tor_assert(len <= UINT_MAX);
-      SECStatus s = PK11_DigestOp(digest->d.ctx,
-                                  (const unsigned char *)data,
-                                  (unsigned int)len);
-      tor_assert(s == SECSuccess);
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Update(&digest->d.sha1, (void*)data, len);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Update(&digest->d.sha2, (void*)data, len);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Update(&digest->d.sha512, (void*)data, len);
-      break;
-#endif
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
-      break;
-    default:
-      /* LCOV_EXCL_START */
-      tor_fragile_assert();
-      break;
-      /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Compute the hash of the data that has been passed to the digest
- * object; write the first out_len bytes of the result to <b>out</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
- * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
- */
-void
-crypto_digest_get_digest(crypto_digest_t *digest,
-                         char *out, size_t out_len)
-{
-  unsigned char r[DIGEST512_LEN];
-  tor_assert(digest);
-  tor_assert(out);
-  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
-
-  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
-   * short output buffers by truncating appropriately. */
-  if (digest->algorithm == DIGEST_SHA3_256 ||
-      digest->algorithm == DIGEST_SHA3_512) {
-    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
-    return;
-  }
-
-#ifdef ENABLE_NSS
-  /* Copy into a temporary buffer since DigestFinal (alters) the context */
-  unsigned char buf[1024];
-  unsigned int saved_len = 0;
-  unsigned rlen;
-  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
-                                               buf, sizeof(buf),
-                                               &saved_len);
-  tor_assert(saved);
-  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
-  tor_assert(s == SECSuccess);
-  tor_assert(rlen >= out_len);
-  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
-  tor_assert(s == SECSuccess);
-  if (saved != buf) {
-    PORT_ZFree(saved, saved_len);
-  }
-#else
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t tmpenv;
-  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
-  memcpy(&tmpenv, digest, alloc_bytes);
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Final(r, &tmpenv.d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Final(r, &tmpenv.d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Final(r, &tmpenv.d.sha512);
-      break;
-//LCOV_EXCL_START
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-    default:
-      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
-      /* This is fatal, because it should never happen. */
-      tor_assert_unreached();
-      break;
-//LCOV_EXCL_STOP
-  }
-#endif
-  memcpy(out, r, out_len);
-  memwipe(r, 0, sizeof(r));
-}
-
-/** Allocate and return a new digest object with the same state as
- * <b>digest</b>
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
- * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
- */
-crypto_digest_t *
-crypto_digest_dup(const crypto_digest_t *digest)
-{
-  tor_assert(digest);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    result->d.ctx = PK11_CloneContext(digest->d.ctx);
-  }
-#endif
-  return result;
-}
-
-/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object.
- */
-void
-crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
-                         const crypto_digest_t *digest)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  tor_assert(bytes <= sizeof(checkpoint->mem));
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    unsigned char *allocated;
-    allocated = PK11_SaveContextAlloc(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      sizeof(checkpoint->mem),
-                                      &checkpoint->bytes_used);
-    /* No allocation is allowed here. */
-    tor_assert(allocated == checkpoint->mem);
-    return;
-  }
-#endif
-  memcpy(checkpoint->mem, digest, bytes);
-}
-
-/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
- * state was previously stored with crypto_digest_checkpoint() */
-void
-crypto_digest_restore(crypto_digest_t *digest,
-                      const crypto_digest_checkpoint_t *checkpoint)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    SECStatus s = PK11_RestoreContext(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      checkpoint->bytes_used);
-    tor_assert(s == SECSuccess);
-    return;
-  }
-#endif
-  memcpy(digest, checkpoint->mem, bytes);
-}
-
-/** Replace the state of the digest object <b>into</b> with the state
- * of the digest object <b>from</b>.  Requires that 'into' and 'from'
- * have the same digest type.
- */
-void
-crypto_digest_assign(crypto_digest_t *into,
-                     const crypto_digest_t *from)
-{
-  tor_assert(into);
-  tor_assert(from);
-  tor_assert(into->algorithm == from->algorithm);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(from->algorithm)) {
-    PK11_DestroyContext(into->d.ctx, PR_TRUE);
-    into->d.ctx = PK11_CloneContext(from->d.ctx);
-    return;
-  }
-#endif
-  memcpy(into,from,alloc_bytes);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of those strings,
- * plus the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist(char *digest_out, size_t len_out,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of: the
- * optional string <b>prepend</b>, those strings,
- * and the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>len_out</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                        const char *prepend,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_t *d = crypto_digest_new_internal(alg);
-  if (prepend)
-    crypto_digest_add_bytes(d, prepend, strlen(prepend));
-  SMARTLIST_FOREACH(lst, const char *, cp,
-                    crypto_digest_add_bytes(d, cp, strlen(cp)));
-  if (append)
-    crypto_digest_add_bytes(d, append, strlen(append));
-  crypto_digest_get_digest(d, digest_out, len_out);
-  crypto_digest_free(d);
-}
-
-/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
- * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
- * result in <b>hmac_out</b>. Asserts on failure.
- */
-void
-crypto_hmac_sha256(char *hmac_out,
-                   const char *key, size_t key_len,
-                   const char *msg, size_t msg_len)
-{
-  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
-  tor_assert(key_len < INT_MAX);
-  tor_assert(msg_len < INT_MAX);
-  tor_assert(hmac_out);
-#ifdef ENABLE_NSS
-  PK11SlotInfo *slot = NULL;
-  PK11SymKey *symKey = NULL;
-  PK11Context *hmac = NULL;
-
-  int ok = 0;
-  SECStatus s;
-  SECItem keyItem, paramItem;
-  keyItem.data = (unsigned char *)key;
-  keyItem.len = (unsigned)key_len;
-  paramItem.type = siBuffer;
-  paramItem.data = NULL;
-  paramItem.len = 0;
-
-  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
-  if (!slot)
-    goto done;
-  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
-                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
-  if (!symKey)
-    goto done;
-
-  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
-                                    &paramItem);
-  if (!hmac)
-    goto done;
-  s = PK11_DigestBegin(hmac);
-  if (s != SECSuccess)
-    goto done;
-  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-  unsigned int len=0;
-  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
-  if (s != SECSuccess || len != DIGEST256_LEN)
-    goto done;
-  ok = 1;
-
- done:
-  if (hmac)
-    PK11_DestroyContext(hmac, PR_TRUE);
-  if (symKey)
-    PK11_FreeSymKey(symKey);
-  if (slot)
-    PK11_FreeSlot(slot);
-
-  tor_assert(ok);
-#else
-  unsigned char *rv = NULL;
-  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
-            (unsigned char*)hmac_out, NULL);
-  tor_assert(rv);
-#endif
-}
-
 /** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
  * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
  * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
@@ -779,7 +135,23 @@ crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
 
 /** Internal state for a eXtendable-Output Function (XOF). */
 struct crypto_xof_t {
+#ifdef OPENSSL_HAS_SHAKE3_EVP
+  /* XXXX We can't enable this yet, because OpenSSL's
+   * DigestFinalXOF function can't be called repeatedly on the same
+   * XOF.
+   *
+   * We could in theory use the undocumented SHA3_absorb and SHA3_squeeze
+   * functions, but let's not mess with undocumented OpenSSL internals any
+   * more than we have to.
+   *
+   * We could also revise our XOF code so that it only allows a single
+   * squeeze operation; we don't require streaming squeeze operations
+   * outside the tests yet.
+   */
+  EVP_MD_CTX *ctx;
+#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */
   keccak_state s;
+#endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */
 };
 
 /** Allocate a new XOF object backed by SHAKE-256.  The security level
@@ -792,7 +164,14 @@ crypto_xof_new(void)
 {
   crypto_xof_t *xof;
   xof = tor_malloc(sizeof(crypto_xof_t));
+#ifdef OPENSSL_HAS_SHAKE256
+  xof->ctx = EVP_MD_CTX_new();
+  tor_assert(xof->ctx);
+  int r = EVP_DigestInit(xof->ctx, EVP_shake256());
+  tor_assert(r == 1);
+#else /* !defined(OPENSSL_HAS_SHAKE256) */
   keccak_xof_init(&xof->s, 256);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
   return xof;
 }
 
@@ -803,8 +182,13 @@ crypto_xof_new(void)
 void
 crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestUpdate(xof->ctx, data, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_absorb(&xof->s, data, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Squeeze bytes out of a XOF object.  Calling this routine will render
@@ -813,8 +197,13 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 void
 crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestFinalXOF(xof->ctx, out, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_squeeze(&xof->s, out, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Cleanse and deallocate a XOF object. */
@@ -823,6 +212,34 @@ crypto_xof_free_(crypto_xof_t *xof)
 {
   if (!xof)
     return;
+#ifdef OPENSSL_HAS_SHAKE256
+  if (xof->ctx)
+    EVP_MD_CTX_free(xof->ctx);
+#endif
   memwipe(xof, 0, sizeof(crypto_xof_t));
   tor_free(xof);
 }
+
+/** Compute the XOF (SHAKE256) of a <b>input_len</b> bytes at <b>input</b>,
+ * putting <b>output_len</b> bytes at <b>output</b>. */
+void
+crypto_xof(uint8_t *output, size_t output_len,
+           const uint8_t *input, size_t input_len)
+{
+#ifdef OPENSSL_HAS_SHA3
+  EVP_MD_CTX *ctx = EVP_MD_CTX_new();
+  tor_assert(ctx);
+  int r = EVP_DigestInit(ctx, EVP_shake256());
+  tor_assert(r == 1);
+  r = EVP_DigestUpdate(ctx, input, input_len);
+  tor_assert(r == 1);
+  r = EVP_DigestFinalXOF(ctx, output, output_len);
+  tor_assert(r == 1);
+  EVP_MD_CTX_free(ctx);
+#else /* !defined(OPENSSL_HAS_SHA3) */
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, input, input_len);
+  crypto_xof_squeeze_bytes(xof, output, output_len);
+  crypto_xof_free(xof);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+}
diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h
index 47e60ce617..5869db7800 100644
--- a/src/lib/crypt_ops/crypto_digest.h
+++ b/src/lib/crypt_ops/crypto_digest.h
@@ -124,6 +124,8 @@ void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
 void crypto_xof_free_(crypto_xof_t *xof);
 #define crypto_xof_free(xof) \
   FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
+void crypto_xof(uint8_t *output, size_t output_len,
+                const uint8_t *input, size_t input_len);
 
 #ifdef TOR_UNIT_TESTS
 digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
diff --git a/src/lib/crypt_ops/crypto_digest_nss.c b/src/lib/crypt_ops/crypto_digest_nss.c
new file mode 100644
index 0000000000..afa8f6d5ef
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_nss.c
@@ -0,0 +1,559 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_nss.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (NSS specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+DISABLE_GCC_WARNING(strict-prototypes)
+#include <pk11pub.h>
+ENABLE_GCC_WARNING(strict-prototypes)
+
+/**
+ * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
+ * On failure, return SEC_OID_UNKNOWN. */
+static SECOidTag
+digest_alg_to_nss_oid(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: return SEC_OID_SHA1;
+    case DIGEST_SHA256: return SEC_OID_SHA256;
+    case DIGEST_SHA512: return SEC_OID_SHA512;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return SEC_OID_UNKNOWN;
+  }
+}
+
+/* Helper: get an unkeyed digest via pk11wrap */
+static int
+digest_nss_internal(SECOidTag alg,
+                    char *digest, unsigned len_out,
+                    const char *msg, size_t msg_len)
+{
+  if (alg == SEC_OID_UNKNOWN)
+    return -1;
+  tor_assert(msg_len <= UINT_MAX);
+
+  int rv = -1;
+  SECStatus s;
+  PK11Context *ctx = PK11_CreateDigestContext(alg);
+  if (!ctx)
+    return -1;
+
+  s = PK11_DigestBegin(ctx);
+  if (s != SECSuccess)
+    goto done;
+
+  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+
+  unsigned int len = 0;
+  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
+  if (s != SECSuccess)
+    goto done;
+
+  rv = 0;
+ done:
+  PK11_DestroyContext(ctx, PR_TRUE);
+  return rv;
+}
+
+/** True iff alg is implemented in our crypto library, and we want to use that
+ * implementation */
+static bool
+library_supports_digest(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return true;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return false;
+  }
+}
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
+  } else {
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
+  } else {
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    PK11Context *ctx;
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.ctx);
+    case DIGEST_SHA3_256:
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
+      if (BUG(!r->d.ctx)) {
+        tor_free(r);
+        return NULL;
+      }
+      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+  if (library_supports_digest(digest->algorithm)) {
+    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
+  }
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      tor_assert(len <= UINT_MAX);
+      SECStatus s = PK11_DigestOp(digest->d.ctx,
+                                  (const unsigned char *)data,
+                                  (unsigned int)len);
+      tor_assert(s == SECSuccess);
+      break;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+  }
+
+  /* Copy into a temporary buffer since DigestFinal (alters) the context */
+  unsigned char buf[1024];
+  unsigned int saved_len = 0;
+  unsigned rlen;
+  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
+                                               buf, sizeof(buf),
+                                               &saved_len);
+  tor_assert(saved);
+  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
+  tor_assert(s == SECSuccess);
+  tor_assert(rlen >= out_len);
+  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
+  tor_assert(s == SECSuccess);
+
+  if (saved != buf) {
+    PORT_ZFree(saved, saved_len);
+  }
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+  if (library_supports_digest(digest->algorithm)) {
+    result->d.ctx = PK11_CloneContext(digest->d.ctx);
+  }
+
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  if (library_supports_digest(digest->algorithm)) {
+    unsigned char *allocated;
+    allocated = PK11_SaveContextAlloc(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      sizeof(checkpoint->mem),
+                                      &checkpoint->bytes_used);
+    /* No allocation is allowed here. */
+    tor_assert(allocated == checkpoint->mem);
+    return;
+  }
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  if (library_supports_digest(digest->algorithm)) {
+    SECStatus s = PK11_RestoreContext(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      checkpoint->bytes_used);
+    tor_assert(s == SECSuccess);
+    return;
+  }
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+  if (library_supports_digest(from->algorithm)) {
+    PK11_DestroyContext(into->d.ctx, PR_TRUE);
+    into->d.ctx = PK11_CloneContext(from->d.ctx);
+    return;
+  }
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+
+  PK11SlotInfo *slot = NULL;
+  PK11SymKey *symKey = NULL;
+  PK11Context *hmac = NULL;
+
+  int ok = 0;
+  SECStatus s;
+  SECItem keyItem, paramItem;
+  keyItem.data = (unsigned char *)key;
+  keyItem.len = (unsigned)key_len;
+  paramItem.type = siBuffer;
+  paramItem.data = NULL;
+  paramItem.len = 0;
+
+  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
+  if (!slot)
+    goto done;
+  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
+                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
+  if (!symKey)
+    goto done;
+
+  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
+                                    &paramItem);
+  if (!hmac)
+    goto done;
+  s = PK11_DigestBegin(hmac);
+  if (s != SECSuccess)
+    goto done;
+  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+  unsigned int len=0;
+  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
+  if (s != SECSuccess || len != DIGEST256_LEN)
+    goto done;
+  ok = 1;
+
+ done:
+  if (hmac)
+    PK11_DestroyContext(hmac, PR_TRUE);
+  if (symKey)
+    PK11_FreeSymKey(symKey);
+  if (slot)
+    PK11_FreeSlot(slot);
+
+  tor_assert(ok);
+}
diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c
new file mode 100644
index 0000000000..50a8ff4d27
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_openssl.c
@@ -0,0 +1,522 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_openssl.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (OpenSSL specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
+    return -1;
+  }
+  return 0;
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST256_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL);
+#else
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
+           != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST512_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL);
+#else
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    SHA_CTX sha1; /**< state for SHA1 */
+    SHA256_CTX sha2; /**< state for SHA256 */
+    SHA512_CTX sha512; /**< state for SHA512 */
+#ifdef OPENSSL_HAS_SHA3
+    EVP_MD_CTX *md;
+#else
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+#endif
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1:
+      return END_OF_FIELD(d.sha1);
+    case DIGEST_SHA256:
+      return END_OF_FIELD(d.sha2);
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.sha512);
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.md);
+#else
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1:
+      SHA1_Init(&r->d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Init(&r->d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Init(&r->d.sha512);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_512:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    if (digest->d.md) {
+      EVP_MD_CTX_free(digest->d.md);
+    }
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Update(&digest->d.sha1, (void*)data, len);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Update(&digest->d.sha2, (void*)data, len);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Update(&digest->d.sha512, (void*)data, len);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: {
+      int r = EVP_DigestUpdate(digest->d.md, data, len);
+      tor_assert(r);
+  }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned dlen = (unsigned)
+      crypto_digest_algorithm_get_length(digest->algorithm);
+    EVP_MD_CTX *tmp = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(tmp, digest->d.md);
+    memset(r, 0xff, sizeof(r));
+    int res = EVP_DigestFinal(tmp, r, &dlen);
+    EVP_MD_CTX_free(tmp);
+    tor_assert(res == 1);
+    goto done;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    /* Tiny-Keccak handles copying into a temporary ctx, and also can handle
+     * short output buffers by truncating appropriately. */
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t tmpenv;
+  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
+  memcpy(&tmpenv, digest, alloc_bytes);
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Final(r, &tmpenv.d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Final(r, &tmpenv.d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Final(r, &tmpenv.d.sha512);
+      break;
+//LCOV_EXCL_START
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+    default:
+      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
+      /* This is fatal, because it should never happen. */
+      tor_assert_unreached();
+      break;
+//LCOV_EXCL_STOP
+  }
+#ifdef OPENSSL_HAS_SHA3
+ done:
+#endif
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    result->d.md = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(result->d.md, digest->d.md);
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (from->algorithm == DIGEST_SHA3_256 ||
+      from->algorithm == DIGEST_SHA3_512) {
+    EVP_MD_CTX_copy(into->d.md, from->d.md);
+    return;
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+  unsigned char *rv = NULL;
+  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
+            (unsigned char*)hmac_out, NULL);
+  tor_assert(rv);
+}
+
diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c
index 0a442bb739..c28111a5a5 100644
--- a/src/lib/crypt_ops/crypto_ed25519.c
+++ b/src/lib/crypt_ops/crypto_ed25519.c
@@ -226,7 +226,7 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong)
 int
 ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey)
 {
-  return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
+  return safe_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
 }
 
 /* Return a heap-allocated array that contains <b>msg</b> prefixed by the
diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c
index 84f73e5272..118cd79045 100644
--- a/src/lib/crypt_ops/crypto_format.c
+++ b/src/lib/crypt_ops/crypto_format.c
@@ -104,7 +104,7 @@ crypto_read_tagged_contents_from_file(const char *fname,
   prefix[32] = 0;
   /* Check type, extract tag. */
   if (strcmpstart(prefix, "== ") || strcmpend(prefix, " ==") ||
-      ! tor_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
+      ! fast_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
     saved_errno = EINVAL;
     goto end;
   }
@@ -131,20 +131,27 @@ crypto_read_tagged_contents_from_file(const char *fname,
   return r;
 }
 
-/** Encode <b>pkey</b> as a base64-encoded string, without trailing "="
+/** Encode <b>pkey</b> as a base64-encoded string, including trailing "="
  * characters, in the buffer <b>output</b>, which must have at least
- * CURVE25519_BASE64_PADDED_LEN+1 bytes available.  Return 0 on success, -1 on
- * failure. */
-int
+ * CURVE25519_BASE64_PADDED_LEN+1 bytes available.
+ * Can not fail.
+ *
+ * Careful! CURVE25519_BASE64_PADDED_LEN is one byte longer than
+ * ED25519_BASE64_LEN.
+ */
+void
 curve25519_public_to_base64(char *output,
                             const curve25519_public_key_t *pkey)
 {
   char buf[128];
-  base64_encode(buf, sizeof(buf),
-                (const char*)pkey->public_key, CURVE25519_PUBKEY_LEN, 0);
-  buf[CURVE25519_BASE64_PADDED_LEN] = '\0';
+  int n = base64_encode(buf, sizeof(buf),
+                        (const char*)pkey->public_key,
+                        CURVE25519_PUBKEY_LEN, 0);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode(). */
+  tor_assert(n == CURVE25519_BASE64_PADDED_LEN);
+  tor_assert(buf[CURVE25519_BASE64_PADDED_LEN] == '\0');
   memcpy(output, buf, CURVE25519_BASE64_PADDED_LEN+1);
-  return 0;
 }
 
 /** Try to decode a base64-encoded curve25519 public key from <b>input</b>
@@ -181,8 +188,7 @@ ed25519_fmt(const ed25519_public_key_t *pkey)
     if (ed25519_public_key_is_zero(pkey)) {
       strlcpy(formatted, "<unset>", sizeof(formatted));
     } else {
-      int r = ed25519_public_to_base64(formatted, pkey);
-      tor_assert(!r);
+      ed25519_public_to_base64(formatted, pkey);
     }
   } else {
     strlcpy(formatted, "<null>", sizeof(formatted));
@@ -202,28 +208,35 @@ ed25519_public_from_base64(ed25519_public_key_t *pkey,
 
 /** Encode the public key <b>pkey</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_BASE64_LEN bytes of encoded key,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
+ *
+ * Careful! ED25519_BASE64_LEN is one byte shorter than
+ * CURVE25519_BASE64_PADDED_LEN.
  */
-int
+void
 ed25519_public_to_base64(char *output,
                          const ed25519_public_key_t *pkey)
 {
-  return digest256_to_base64(output, (const char *)pkey->pubkey);
+  digest256_to_base64(output, (const char *)pkey->pubkey);
 }
 
 /** Encode the signature <b>sig</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_SIG_BASE64_LEN bytes of encoded signature,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
  */
-int
+void
 ed25519_signature_to_base64(char *output,
                             const ed25519_signature_t *sig)
 {
   char buf[256];
   int n = base64_encode_nopad(buf, sizeof(buf), sig->sig, ED25519_SIG_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
   tor_assert(n == ED25519_SIG_BASE64_LEN);
+  tor_assert(buf[ED25519_SIG_BASE64_LEN] == '\0');
   memcpy(output, buf, ED25519_SIG_BASE64_LEN+1);
-  return 0;
 }
 
 /** Try to decode the string <b>input</b> into an ed25519 signature. On
@@ -233,16 +246,11 @@ int
 ed25519_signature_from_base64(ed25519_signature_t *sig,
                               const char *input)
 {
-
   if (strlen(input) != ED25519_SIG_BASE64_LEN)
     return -1;
-  char buf[ED25519_SIG_BASE64_LEN+3];
-  memcpy(buf, input, ED25519_SIG_BASE64_LEN);
-  buf[ED25519_SIG_BASE64_LEN+0] = '=';
-  buf[ED25519_SIG_BASE64_LEN+1] = '=';
-  buf[ED25519_SIG_BASE64_LEN+2] = 0;
   char decoded[128];
-  int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf));
+  int n = base64_decode(decoded, sizeof(decoded), input,
+                        ED25519_SIG_BASE64_LEN);
   if (n < 0 || n != ED25519_SIG_LEN)
     return -1;
   memcpy(sig->sig, decoded, ED25519_SIG_LEN);
@@ -250,24 +258,26 @@ ed25519_signature_from_base64(ed25519_signature_t *sig,
   return 0;
 }
 
-/** Base64 encode DIGEST_LINE bytes from <b>digest</b>, remove the trailing =
+/** Base64 encode DIGEST_LEN bytes from <b>digest</b>, remove the trailing =
  * characters, and store the nul-terminated result in the first
- * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.  */
-/* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST_LEN, 0);
-  buf[BASE64_DIGEST_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST_LEN);
+  tor_assert(buf[BASE64_DIGEST_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest_from_base64(char *digest, const char *d64)
 {
@@ -279,22 +289,24 @@ digest_from_base64(char *digest, const char *d64)
 
 /** Base64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the
  * trailing = characters, and store the nul-terminated result in the first
- * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */
- /* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest256_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN, 0);
-  buf[BASE64_DIGEST256_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST256_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST256_LEN);
+  tor_assert(buf[BASE64_DIGEST256_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST256_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST256_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest256_from_base64(char *digest, const char *d64)
 {
diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h
index fe852e6a61..b4b3aa189c 100644
--- a/src/lib/crypt_ops/crypto_format.h
+++ b/src/lib/crypt_ops/crypto_format.h
@@ -33,18 +33,18 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname,
 
 int ed25519_public_from_base64(struct ed25519_public_key_t *pkey,
                                const char *input);
-int ed25519_public_to_base64(char *output,
-                             const struct ed25519_public_key_t *pkey);
+void ed25519_public_to_base64(char *output,
+                              const struct ed25519_public_key_t *pkey);
 const char *ed25519_fmt(const struct ed25519_public_key_t *pkey);
 
 int ed25519_signature_from_base64(struct ed25519_signature_t *sig,
                                   const char *input);
-int ed25519_signature_to_base64(char *output,
-                                const struct ed25519_signature_t *sig);
+void ed25519_signature_to_base64(char *output,
+                                 const struct ed25519_signature_t *sig);
 
-int digest_to_base64(char *d64, const char *digest);
+void digest_to_base64(char *d64, const char *digest);
 int digest_from_base64(char *digest, const char *d64);
-int digest256_to_base64(char *d64, const char *digest);
+void digest256_to_base64(char *d64, const char *digest);
 int digest256_from_base64(char *digest, const char *d64);
 
 #endif /* !defined(TOR_CRYPTO_FORMAT_H) */
diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c
index fd2e701651..e0d241d4ea 100644
--- a/src/lib/crypt_ops/crypto_hkdf.c
+++ b/src/lib/crypt_ops/crypto_hkdf.c
@@ -25,7 +25,7 @@
 #include <openssl/kdf.h>
 #define HAVE_OPENSSL_HKDF 1
 #endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <string.h>
 
@@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl(
   return 0;
 }
 
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
 
 /**
  * Perform RFC5869 HKDF computation using our own legacy implementation.
@@ -166,7 +166,7 @@ crypto_expand_key_material_rfc5869_sha256_legacy(
   memwipe(mac, 0, sizeof(mac));
   return 0;
 }
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 
 /** Expand some secret key material according to RFC5869, using SHA256 as the
  * underlying hash.  The <b>key_in_len</b> bytes at <b>key_in</b> are the
@@ -191,11 +191,11 @@ crypto_expand_key_material_rfc5869_sha256(
                                              salt_in_len, info_in,
                                              info_in_len,
                                              key_out, key_out_len);
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
   return crypto_expand_key_material_rfc5869_sha256_legacy(key_in,
                                                key_in_len, salt_in,
                                                salt_in_len, info_in,
                                                info_in_len,
                                                key_out, key_out_len);
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 }
diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c
index 329c264af6..a16bf4e11a 100644
--- a/src/lib/crypt_ops/crypto_init.c
+++ b/src/lib/crypt_ops/crypto_init.c
@@ -12,6 +12,8 @@
 
 #include "orconfig.h"
 
+#define CRYPTO_PRIVATE
+
 #include "lib/crypt_ops/crypto_init.h"
 
 #include "lib/crypt_ops/crypto_curve25519.h"
@@ -20,8 +22,11 @@
 #include "lib/crypt_ops/crypto_openssl_mgt.h"
 #include "lib/crypt_ops/crypto_nss_mgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_sys.h"
+
+#include "lib/subsys/subsys.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** Boolean: has our crypto library been initialized? (early phase) */
 static int crypto_early_initialized_ = 0;
@@ -66,6 +71,8 @@ crypto_early_init(void)
     if (crypto_init_siphash_key() < 0)
       return -1;
 
+    crypto_rand_fast_init();
+
     curve25519_init();
     ed25519_init();
   }
@@ -92,7 +99,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
     (void)useAccel;
     (void)accelName;
     (void)accelDir;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
     if (crypto_nss_late_init() < 0)
       return -1;
@@ -108,6 +115,7 @@ crypto_thread_cleanup(void)
 #ifdef ENABLE_OPENSSL
   crypto_openssl_thread_cleanup();
 #endif
+  destroy_thread_fast_rng();
 }
 
 /**
@@ -126,6 +134,8 @@ crypto_global_cleanup(void)
   crypto_nss_global_cleanup();
 #endif
 
+  crypto_rand_fast_shutdown();
+
   crypto_early_initialized_ = 0;
   crypto_global_initialized_ = 0;
   have_seeded_siphash = 0;
@@ -142,6 +152,12 @@ crypto_prefork(void)
 #ifdef ENABLE_NSS
   crypto_nss_prefork();
 #endif
+  /* It is not safe to share a fast_rng object across a fork boundary unless
+   * we actually have zero-on-fork support in map_anon.c.  If we have
+   * drop-on-fork support, we will crash; if we have neither, we will yield
+   * a copy of the parent process's rng, which is scary and insecure.
+   */
+  destroy_thread_fast_rng();
 }
 
 /** Run operations that the crypto library requires to be happy again
@@ -202,3 +218,47 @@ tor_is_using_nss(void)
   return 0;
 #endif
 }
+
+static int
+subsys_crypto_initialize(void)
+{
+  if (crypto_early_init() < 0)
+    return -1;
+  crypto_dh_init();
+  return 0;
+}
+
+static void
+subsys_crypto_shutdown(void)
+{
+  crypto_global_cleanup();
+}
+
+static void
+subsys_crypto_prefork(void)
+{
+  crypto_prefork();
+}
+
+static void
+subsys_crypto_postfork(void)
+{
+  crypto_postfork();
+}
+
+static void
+subsys_crypto_thread_cleanup(void)
+{
+  crypto_thread_cleanup();
+}
+
+const struct subsys_fns_t sys_crypto = {
+  .name = "crypto",
+  .supported = true,
+  .level = -60,
+  .initialize = subsys_crypto_initialize,
+  .shutdown = subsys_crypto_shutdown,
+  .prefork = subsys_crypto_prefork,
+  .postfork = subsys_crypto_postfork,
+  .thread_cleanup = subsys_crypto_thread_cleanup,
+};
diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h
index 540d08eb56..8de3eb03ed 100644
--- a/src/lib/crypt_ops/crypto_init.h
+++ b/src/lib/crypt_ops/crypto_init.h
@@ -33,4 +33,4 @@ const char *crypto_get_header_version_string(void);
 
 int tor_is_using_nss(void);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_INIT_H) */
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h
index 72fd2a1229..4cfa9b42a4 100644
--- a/src/lib/crypt_ops/crypto_nss_mgt.h
+++ b/src/lib/crypt_ops/crypto_nss_mgt.h
@@ -29,6 +29,6 @@ void crypto_nss_global_cleanup(void);
 
 void crypto_nss_prefork(void);
 void crypto_nss_postfork(void);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
-#endif /* !defined(TOR_CRYPTO_NSS_H) */
+#endif /* !defined(TOR_CRYPTO_NSS_MGT_H) */
diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c
index 2186d2a939..ed832d852e 100644
--- a/src/lib/crypt_ops/crypto_ope.c
+++ b/src/lib/crypt_ops/crypto_ope.c
@@ -57,9 +57,9 @@ ope_val_from_le(ope_val_t x)
     ((x) >> 8) |
     (((x)&0xff) << 8);
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
 #define ope_val_from_le(x) (x)
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
 /**
  * Return a new AES256-CTR stream cipher object for <b>ope</b>, ready to yield
diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h
index 610d956335..9778dfe0f0 100644
--- a/src/lib/crypt_ops/crypto_ope.h
+++ b/src/lib/crypt_ops/crypto_ope.h
@@ -41,6 +41,6 @@ struct aes_cnt_cipher;
 STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope,
                                               uint32_t initial_idx);
 STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n);
-#endif
+#endif /* defined(CRYPTO_OPE_PRIVATE) */
 
-#endif
+#endif /* !defined(CRYPTO_OPE_H) */
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c
index c97815f9a4..f51309219a 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.c
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.c
@@ -176,6 +176,10 @@ crypto_openssl_free_all(void)
   tor_free(crypto_openssl_version_str);
   tor_free(crypto_openssl_header_version_str);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
 #ifndef NEW_THREAD_API
   if (n_openssl_mutexes_) {
     int n = n_openssl_mutexes_;
@@ -200,10 +204,10 @@ crypto_openssl_early_init(void)
                      OPENSSL_INIT_LOAD_CRYPTO_STRINGS |
                      OPENSSL_INIT_ADD_ALL_CIPHERS |
                      OPENSSL_INIT_ADD_ALL_DIGESTS, NULL);
-#else
+#else /* !defined(OPENSSL_1_1_API) */
     ERR_load_crypto_strings();
     OpenSSL_add_all_algorithms();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
     setup_openssl_threading();
 
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h
index a3dd03aa04..111a2d12ed 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.h
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.h
@@ -84,6 +84,6 @@ int crypto_openssl_late_init(int useAccel, const char *accelName,
 void crypto_openssl_thread_cleanup(void);
 void crypto_openssl_global_cleanup(void);
 
-#endif /* ENABLE_OPENSSL */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #endif /* !defined(TOR_CRYPTO_OPENSSL_H) */
diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c
index 915fe0870d..afbafbfa35 100644
--- a/src/lib/crypt_ops/crypto_rand.c
+++ b/src/lib/crypt_ops/crypto_rand.c
@@ -11,7 +11,6 @@
  * number generators, and working with randomness.
  **/
 
-#ifndef CRYPTO_RAND_PRIVATE
 #define CRYPTO_RAND_PRIVATE
 
 #include "lib/crypt_ops/crypto_rand.h"
@@ -37,6 +36,7 @@
 
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/crypto_digest.h"
+#include "lib/ctime/di_ops.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
@@ -47,7 +47,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/rand.h>
 #include <openssl/sha.h>
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 #include <pk11pub.h>
@@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
   (void)out;
   (void)out_len;
   return -1;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   static const char *filenames[] = {
     "/dev/srandom", "/dev/urandom", "/dev/random", NULL
   };
@@ -315,7 +315,7 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
       }
     }
 
-    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
+    if ((out_len < sanity_min_size) || !safe_mem_is_zero((char*)out, out_len))
       return 0;
   }
 
@@ -419,7 +419,7 @@ crypto_seed_openssl_rng(void)
   else
     return -1;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 /**
@@ -442,7 +442,7 @@ crypto_seed_nss_rng(void)
 
   return load_entropy_ok ? 0 : -1;
 }
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 /**
  * Seed the RNG for any and all crypto libraries that we're using with bytes
@@ -520,121 +520,24 @@ crypto_rand_unmocked(char *to, size_t n)
 
 #undef BUFLEN
   }
-#else
+#else /* !defined(ENABLE_NSS) */
   int r = RAND_bytes((unsigned char*)to, (int)n);
   /* We consider a PRNG failure non-survivable. Let's assert so that we get a
    * stack trace about where it happened.
    */
   tor_assert(r >= 0);
-#endif
+#endif /* defined(ENABLE_NSS) */
 }
 
 /**
- * Return a pseudorandom integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
- * INT_MAX+1, inclusive.
+ * Draw an unsigned 32-bit integer uniformly at random.
  */
-int
-crypto_rand_int(unsigned int max)
-{
-  unsigned int val;
-  unsigned int cutoff;
-  tor_assert(max <= ((unsigned int)INT_MAX)+1);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT_MAX - (UINT_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom integer, chosen uniformly from the values i such
- * that min <= i < max.
- *
- * <b>min</b> MUST be in range [0, <b>max</b>).
- * <b>max</b> MUST be in range (min, INT_MAX].
- **/
-int
-crypto_rand_int_range(unsigned int min, unsigned int max)
-{
-  tor_assert(min < max);
-  tor_assert(max <= INT_MAX);
-
-  /* The overflow is avoided here because crypto_rand_int() returns a value
-   * between 0 and (max - min) inclusive. */
-  return min + crypto_rand_int(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports uint64_t.
- **/
-uint64_t
-crypto_rand_uint64_range(uint64_t min, uint64_t max)
+uint32_t
+crypto_rand_u32(void)
 {
-  tor_assert(min < max);
-  return min + crypto_rand_uint64(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports time_t.
- **/
-time_t
-crypto_rand_time_range(time_t min, time_t max)
-{
-  tor_assert(min < max);
-  return min + (time_t)crypto_rand_uint64(max - min);
-}
-
-/**
- * Return a pseudorandom 64-bit integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.
- **/
-uint64_t
-crypto_rand_uint64(uint64_t max)
-{
-  uint64_t val;
-  uint64_t cutoff;
-  tor_assert(max < UINT64_MAX);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT64_MAX - (UINT64_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom double d, chosen uniformly from the range
- * 0.0 <= d < 1.0.
- **/
-double
-crypto_rand_double(void)
-{
-  /* We just use an unsigned int here; we don't really care about getting
-   * more than 32 bits of resolution */
-  unsigned int u;
-  crypto_rand((char*)&u, sizeof(u));
-#if SIZEOF_INT == 4
-#define UINT_MAX_AS_DOUBLE 4294967296.0
-#elif SIZEOF_INT == 8
-#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
-#else
-#error SIZEOF_INT is neither 4 nor 8
-#endif /* SIZEOF_INT == 4 || ... */
-  return ((double)u) / UINT_MAX_AS_DOUBLE;
+  uint32_t rand;
+  crypto_rand((void*)&rand, sizeof(rand));
+  return rand;
 }
 
 /**
@@ -724,8 +627,6 @@ crypto_force_rand_ssleay(void)
     RAND_set_rand_method(default_method);
     return 1;
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   return 0;
 }
-
-#endif /* !defined(CRYPTO_RAND_PRIVATE) */
diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h
index 86fa20faa3..a019287aa9 100644
--- a/src/lib/crypt_ops/crypto_rand.h
+++ b/src/lib/crypt_ops/crypto_rand.h
@@ -16,6 +16,7 @@
 #include "lib/cc/compat_compiler.h"
 #include "lib/cc/torint.h"
 #include "lib/testsupport/testsupport.h"
+#include "lib/malloc/malloc.h"
 
 /* random numbers */
 int crypto_seed_rng(void) ATTR_WUR;
@@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n);
 void crypto_strongest_rand(uint8_t *out, size_t out_len);
 MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len));
 int crypto_rand_int(unsigned int max);
+unsigned crypto_rand_uint(unsigned limit);
 int crypto_rand_int_range(unsigned int min, unsigned int max);
 uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
 time_t crypto_rand_time_range(time_t min, time_t max);
+uint32_t crypto_rand_u32(void);
 uint64_t crypto_rand_uint64(uint64_t max);
 double crypto_rand_double(void);
 struct tor_weak_rng_t;
@@ -40,6 +43,61 @@ void *smartlist_choose(const struct smartlist_t *sl);
 void smartlist_shuffle(struct smartlist_t *sl);
 int crypto_force_rand_ssleay(void);
 
+/**
+ * A fast PRNG, for use when the PRNG provided by our crypto library isn't
+ * fast enough.  This one _should_ be cryptographically strong, but
+ * has seen less auditing than the PRNGs in OpenSSL and NSS.  Use with
+ * caution.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+typedef struct crypto_fast_rng_t crypto_fast_rng_t;
+/**
+ * Number of bytes used to seed a crypto_rand_fast_t.
+ **/
+crypto_fast_rng_t *crypto_fast_rng_new(void);
+#define CRYPTO_FAST_RNG_SEED_LEN 48
+crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed);
+void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n);
+void crypto_fast_rng_free_(crypto_fast_rng_t *);
+#define crypto_fast_rng_free(c)                                 \
+  FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c))
+
+unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit);
+uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit);
+uint32_t crypto_fast_rng_get_u32(crypto_fast_rng_t *rng);
+uint64_t crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                                      uint64_t min, uint64_t max);
+double crypto_fast_rng_get_double(crypto_fast_rng_t *rng);
+
+/**
+ * Using the fast_rng <b>rng</b>, yield true with probability
+ * 1/<b>n</b>. Otherwise yield false.
+ *
+ * <b>n</b> must not be zero.
+ **/
+#define crypto_fast_rng_one_in_n(rng, n)        \
+  (0 == (crypto_fast_rng_get_uint((rng), (n))))
+
+crypto_fast_rng_t *get_thread_fast_rng(void);
+
+#ifdef CRYPTO_PRIVATE
+/* These are only used from crypto_init.c */
+void destroy_thread_fast_rng(void);
+void crypto_rand_fast_init(void);
+void crypto_rand_fast_shutdown(void);
+#endif /* defined(CRYPTO_PRIVATE) */
+
+#if defined(TOR_UNIT_TESTS)
+/* Used for white-box testing */
+size_t crypto_fast_rng_get_bytes_used_per_stream(void);
+/* For deterministic prng implementations */
+void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng);
+/* To override the prng for testing. */
+crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng);
+#endif /* defined(TOR_UNIT_TESTS) */
+
 #ifdef CRYPTO_RAND_PRIVATE
 
 STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c
new file mode 100644
index 0000000000..e6ceb42ccb
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_fast.c
@@ -0,0 +1,439 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand_fast.c
+ *
+ * \brief A fast strong PRNG for use when our underlying cryptographic
+ *   library's PRNG isn't fast enough.
+ **/
+
+/* This library is currently implemented to use the same implementation
+ * technique as libottery, using AES-CTR-256 as our underlying stream cipher.
+ * It's backtracking-resistant immediately, and prediction-resistant after
+ * a while.
+ *
+ * Here's how it works:
+ *
+ * We generate pseudorandom bytes using AES-CTR-256.  We generate BUFLEN bytes
+ * at a time.  When we do this, we keep the first SEED_LEN bytes as the key
+ * and the IV for our next invocation of AES_CTR, and yield the remaining
+ * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG.  As we yield
+ * bytes to the user, we clear them from the buffer.
+ *
+ * After we have refilled the buffer RESEED_AFTER times, we mix in an
+ * additional SEED_LEN bytes from our strong PRNG into the seed.
+ *
+ * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN
+ * bytes from the PRNG and use them with our stream cipher to fill the user's
+ * request.
+ */
+
+#define CRYPTO_RAND_FAST_PRIVATE
+#define CRYPTO_PRIVATE
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/intmath/cmp.h"
+#include "lib/cc/ctassert.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/thread/threads.h"
+
+#include "lib/log/util_bug.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <string.h>
+
+#ifdef NOINHERIT_CAN_FAIL
+#define CHECK_PID
+#endif
+
+#ifdef CHECK_PID
+#define PID_FIELD_LEN sizeof(pid_t)
+#else
+#define PID_FIELD_LEN 0
+#endif
+
+/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter.
+ */
+#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN)
+
+/* The amount of space that we mmap for a crypto_fast_rng_t.
+ */
+#define MAPLEN 4096
+
+/* The number of random bytes that we can yield to the user after each
+ * time we fill a crypto_fast_rng_t's buffer.
+ */
+#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN - PID_FIELD_LEN)
+
+/* The number of buffer refills after which we should fetch more
+ * entropy from crypto_strongest_rand().
+ */
+#define RESEED_AFTER 16
+
+/* The length of the stream cipher key we will use for the PRNG, in bytes.
+ */
+#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN)
+/* The length of the stream cipher key we will use for the PRNG, in bits.
+ */
+#define KEY_BITS (KEY_LEN * 8)
+
+/* Make sure that we have a key length we can actually use with AES. */
+CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256);
+
+struct crypto_fast_rng_t {
+  /** How many more fills does this buffer have before we should mix
+   * in the output of crypto_strongest_rand()?
+   *
+   * This value may be negative if unit tests are enabled.  If so, it
+   * indicates that we should never mix in extra data from
+   * crypto_strongest_rand().
+   */
+  int16_t n_till_reseed;
+  /** How many bytes are remaining in cbuf.bytes? */
+  uint16_t bytes_left;
+#ifdef CHECK_PID
+  /** Which process owns this fast_rng?  If this value is zero, we do not
+   * need to test the owner. */
+  pid_t owner;
+#endif
+  struct cbuf {
+    /** The seed (key and IV) that we will use the next time that we refill
+     * cbuf. */
+    uint8_t seed[SEED_LEN];
+    /**
+     * Bytes that we are yielding to the user.  The next byte to be
+     * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this
+     * array are set to zero.
+     */
+    uint8_t bytes[BUFLEN];
+  } buf;
+};
+
+/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf.
+ */
+CTASSERT(sizeof(struct cbuf) == BUFLEN+SEED_LEN);
+/* We're trying to fit all of the RNG state into a nice mmapable chunk.
+ */
+CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN);
+
+/**
+ * Initialize and return a new fast PRNG, using a strong random seed.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new(void)
+{
+  uint8_t seed[SEED_LEN];
+  crypto_strongest_rand(seed, sizeof(seed));
+  crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed);
+  memwipe(seed, 0, sizeof(seed));
+  return result;
+}
+
+/**
+ * Initialize and return a new fast PRNG, using a seed value specified
+ * in <b>seed</b>.  This value must be CRYPTO_FAST_RNG_SEED_LEN bytes
+ * long.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, you should probably look at get_thread_fast_rng().  Alternatively,
+ * use crypto_rand(), wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new_from_seed(const uint8_t *seed)
+{
+  unsigned inherit = INHERIT_RES_KEEP;
+  /* We try to allocate this object as securely as we can, to avoid
+   * having it get dumped, swapped, or shared after fork.
+   */
+  crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result),
+                                ANONMAP_PRIVATE | ANONMAP_NOINHERIT,
+                                &inherit);
+  memcpy(result->buf.seed, seed, SEED_LEN);
+  /* Causes an immediate refill once the user asks for data. */
+  result->bytes_left = 0;
+  result->n_till_reseed = RESEED_AFTER;
+#ifdef CHECK_PID
+  if (inherit == INHERIT_RES_KEEP) {
+    /* This value will neither be dropped nor zeroed after fork, so we need to
+     * check our pid to make sure we are not sharing it across a fork.  This
+     * can be expensive if the pid value isn't cached, sadly.
+     */
+    result->owner = getpid();
+  }
+#elif defined(_WIN32)
+  /* Windows can't fork(), so there's no need to noinherit. */
+#else
+  /* We decided above that noinherit would always do _something_. Assert here
+   * that we were correct. */
+  tor_assertf(inherit != INHERIT_RES_KEEP,
+              "We failed to create a non-inheritable memory region, even "
+              "though we believed such a failure to be impossible! This is "
+              "probably a bug in Tor support for your platform; please report "
+              "it.");
+#endif /* defined(CHECK_PID) || ... */
+  return result;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more
+ * entropy.
+ */
+void
+crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed = -1;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Helper: create a crypto_cipher_t object from SEED_LEN bytes of
+ * input.  The first KEY_LEN bytes are used as the stream cipher's key,
+ * and the remaining CIPHER_IV_LEN bytes are used as its IV.
+ **/
+static inline crypto_cipher_t *
+cipher_from_seed(const uint8_t *seed)
+{
+  return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS);
+}
+
+/**
+ * Helper: mix additional entropy into <b>rng</b> by using our XOF to mix the
+ * old value for the seed with some additional bytes from
+ * crypto_strongest_rand().
+ **/
+static void
+crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng)
+{
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN);
+  {
+    uint8_t seedbuf[SEED_LEN];
+    crypto_strongest_rand(seedbuf, SEED_LEN);
+    crypto_xof_add_bytes(xof, seedbuf, SEED_LEN);
+    memwipe(seedbuf, 0, SEED_LEN);
+  }
+  crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN);
+  crypto_xof_free(xof);
+}
+
+/**
+ * Helper: refill the seed bytes and output buffer of <b>rng</b>, using
+ * the input seed bytes as input (key and IV) for the stream cipher.
+ *
+ * If the n_till_reseed counter has reached zero, mix more random bytes into
+ * the seed before refilling the buffer.
+ **/
+static void
+crypto_fast_rng_refill(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed--;
+  if (rng->n_till_reseed == 0) {
+    /* It's time to reseed the RNG. */
+    crypto_fast_rng_add_entopy(rng);
+    rng->n_till_reseed = RESEED_AFTER;
+  } else if (rng->n_till_reseed < 0) {
+#ifdef TOR_UNIT_TESTS
+    /* Reseeding is disabled for testing; never do it on this prng. */
+    rng->n_till_reseed = -1;
+#else
+    /* If testing is disabled, this shouldn't be able to become negative. */
+    tor_assert_unreached();
+#endif /* defined(TOR_UNIT_TESTS) */
+  }
+  /* Now fill rng->buf with output from our stream cipher, initialized from
+   * that seed value. */
+  crypto_cipher_t *c = cipher_from_seed(rng->buf.seed);
+  memset(&rng->buf, 0, sizeof(rng->buf));
+  crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf));
+  crypto_cipher_free(c);
+
+  rng->bytes_left = sizeof(rng->buf.bytes);
+}
+
+/**
+ * Release all storage held by <b>rng</b>.
+ **/
+void
+crypto_fast_rng_free_(crypto_fast_rng_t *rng)
+{
+  if (!rng)
+    return;
+  memwipe(rng, 0, sizeof(*rng));
+  tor_munmap_anonymous(rng, sizeof(*rng));
+}
+
+/**
+ * Helper: extract bytes from the PRNG, refilling it as necessary. Does not
+ * optimize the case when the user has asked for a huge output.
+ **/
+static void
+crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out,
+                              const size_t n)
+{
+#ifdef CHECK_PID
+  if (rng->owner) {
+    /* Note that we only need to do this check when we have owner set: that
+     * is, when our attempt to block inheriting failed, and the result was
+     * INHERIT_RES_KEEP.
+     *
+     * If the result was INHERIT_RES_DROP, then any attempt to access the rng
+     * memory after forking will crash.
+     *
+     * If the result was INHERIT_RES_ZERO, then forking will set the bytes_left
+     * and n_till_reseed fields to zero.  This function will call
+     * crypto_fast_rng_refill(), which will in turn reseed the PRNG.
+     *
+     * So we only need to do this test in the case when mmap_anonymous()
+     * returned INHERIT_KEEP.  We avoid doing it needlessly, since getpid() is
+     * often a system call, and that can be slow.
+     */
+    tor_assert(rng->owner == getpid());
+  }
+#endif /* defined(CHECK_PID) */
+
+  size_t bytes_to_yield = n;
+
+  while (bytes_to_yield) {
+    if (rng->bytes_left == 0)
+      crypto_fast_rng_refill(rng);
+
+    const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield);
+
+    tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left);
+    uint8_t *copy_from = rng->buf.bytes +
+      (sizeof(rng->buf.bytes) - rng->bytes_left);
+    memcpy(out, copy_from, to_copy);
+    memset(copy_from, 0, to_copy);
+
+    out += to_copy;
+    bytes_to_yield -= to_copy;
+    rng->bytes_left -= to_copy;
+  }
+}
+
+/**
+ * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>.
+ **/
+void
+crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n)
+{
+  if (PREDICT_UNLIKELY(n > BUFLEN)) {
+    /* The user has asked for a lot of output; generate it from a stream
+     * cipher seeded by the PRNG rather than by pulling it out of the PRNG
+     * directly.
+     */
+    uint8_t seed[SEED_LEN];
+    crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN);
+    crypto_cipher_t *c = cipher_from_seed(seed);
+    memset(out, 0, n);
+    crypto_cipher_crypt_inplace(c, (char*)out, n);
+    crypto_cipher_free(c);
+    memwipe(seed, 0, sizeof(seed));
+    return;
+  }
+
+  crypto_fast_rng_getbytes_impl(rng, out, n);
+}
+
+#if defined(TOR_UNIT_TESTS)
+/** for white-box testing: return the number of bytes that are returned from
+ * the user for each invocation of the stream cipher in this RNG. */
+size_t
+crypto_fast_rng_get_bytes_used_per_stream(void)
+{
+  return BUFLEN;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Thread-local instance for our fast RNG.
+ **/
+static tor_threadlocal_t thread_rng;
+
+/**
+ * Return a per-thread fast RNG, initializing it if necessary.
+ *
+ * You do not need to free this yourself.
+ *
+ * It is NOT safe to share this value across threads.
+ **/
+crypto_fast_rng_t *
+get_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+
+  if (PREDICT_UNLIKELY(rng == NULL)) {
+    rng = crypto_fast_rng_new();
+    tor_threadlocal_set(&thread_rng, rng);
+  }
+
+  return rng;
+}
+
+/**
+ * Used when a thread is exiting: free the per-thread fast RNG if needed.
+ * Invoked from the crypto subsystem's thread-cleanup code.
+ **/
+void
+destroy_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+  if (!rng)
+    return;
+  crypto_fast_rng_free(rng);
+  tor_threadlocal_set(&thread_rng, NULL);
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Replace the current thread's rng with <b>rng</b>. For use by the
+ * unit tests only.  Returns the previous thread rng.
+ **/
+crypto_fast_rng_t *
+crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng)
+{
+  crypto_fast_rng_t *old_rng =  tor_threadlocal_get(&thread_rng);
+  tor_threadlocal_set(&thread_rng, rng);
+  return old_rng;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * initialization code.
+ **/
+void
+crypto_rand_fast_init(void)
+{
+  tor_threadlocal_init(&thread_rng);
+}
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * shutdown code.
+ **/
+void
+crypto_rand_fast_shutdown(void)
+{
+  destroy_thread_fast_rng();
+  tor_threadlocal_destroy(&thread_rng);
+}
diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c
new file mode 100644
index 0000000000..ffbfa2d56c
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_numeric.c
@@ -0,0 +1,194 @@
+/**
+ * \file crypto_rand_numeric.c
+ *
+ * \brief Functions for retrieving uniformly distributed numbers
+ *   from our PRNGs.
+ **/
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/log/util_bug.h"
+
+/**
+ * Implementation macro: yields code that returns a uniform unbiased
+ * random number between 0 and limit.  "type" is the type of the number to
+ * return; "maxval" is the largest possible value of "type"; and "fill_stmt"
+ * is a code snippet that fills an object named "val" with random bits.
+ **/
+#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt)         \
+  do {                                                                  \
+    type val;                                                           \
+    type cutoff;                                                        \
+    tor_assert((limit) > 0);                                            \
+                                                                        \
+    /* We ignore any values that are >= 'cutoff,' to avoid biasing */   \
+    /* the distribution with clipping at the upper end of the type's */ \
+    /* range. */                                                        \
+    cutoff = (maxval) - ((maxval)%(limit));                             \
+    while (1) {                                                         \
+      fill_stmt;                                                        \
+      if (val < cutoff)                                                 \
+        return val % (limit);                                           \
+    }                                                                   \
+  } while (0)
+
+/**
+ * Return a pseudorandom integer chosen uniformly from the values between 0
+ * and <b>limit</b>-1 inclusive. limit must be strictly between 0 and
+ * UINT_MAX. */
+unsigned
+crypto_rand_uint(unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
+ * INT_MAX+1, inclusive.
+ */
+int
+crypto_rand_int(unsigned int max)
+{
+  /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying
+   * to return a signed type. Instead we make sure that the range is
+   * reasonable for a nonnegative int, use crypto_rand_uint(), and cast.
+   */
+  tor_assert(max <= ((unsigned int)INT_MAX)+1);
+
+  return (int)crypto_rand_uint(max);
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values i such
+ * that min <= i < max.
+ *
+ * <b>min</b> MUST be in range [0, <b>max</b>).
+ * <b>max</b> MUST be in range (min, INT_MAX].
+ **/
+int
+crypto_rand_int_range(unsigned int min, unsigned int max)
+{
+  tor_assert(min < max);
+  tor_assert(max <= INT_MAX);
+
+  /* The overflow is avoided here because crypto_rand_int() returns a value
+   * between 0 and (max - min) inclusive. */
+  return min + crypto_rand_int(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports uint64_t.
+ **/
+uint64_t
+crypto_rand_uint64_range(uint64_t min, uint64_t max)
+{
+  tor_assert(min < max);
+  return min + crypto_rand_uint64(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports time_t.
+ **/
+time_t
+crypto_rand_time_range(time_t min, time_t max)
+{
+  tor_assert(min < max);
+  return min + (time_t)crypto_rand_uint64(max - min);
+}
+
+/**
+ * Return a pseudorandom 64-bit integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.
+ **/
+uint64_t
+crypto_rand_uint64(uint64_t max)
+{
+  tor_assert(max < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+#if SIZEOF_INT == 4
+#define UINT_MAX_AS_DOUBLE 4294967296.0
+#elif SIZEOF_INT == 8
+#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
+#else
+#error SIZEOF_INT is neither 4 nor 8
+#endif /* SIZEOF_INT == 4 || ... */
+
+/**
+ * Return a pseudorandom double d, chosen uniformly from the range
+ * 0.0 <= d < 1.0.
+ **/
+double
+crypto_rand_double(void)
+{
+  /* We just use an unsigned int here; we don't really care about getting
+   * more than 32 bits of resolution */
+  unsigned int u;
+  crypto_rand((char*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
+/**
+ * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t
+ */
+unsigned
+crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit)
+{
+  tor_assert(limit < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_u32, but extract the result from a crypto_fast_rng_t.
+ */
+uint32_t
+crypto_fast_rng_get_u32(crypto_fast_rng_t *rng)
+{
+  uint32_t val;
+  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val));
+  return val;
+}
+
+/**
+ * As crypto_rand_uint64_range(), but extract the result from a
+ * crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                             uint64_t min, uint64_t max)
+{
+  /* Handle corrupted input */
+  if (BUG(min >= max)) {
+    return min;
+  }
+
+  return min + crypto_fast_rng_get_uint64(rng, max - min);
+}
+
+/**
+ * As crypto_rand_get_double() but extract the result from a crypto_fast_rng_t.
+ */
+double
+crypto_fast_rng_get_double(crypto_fast_rng_t *rng)
+{
+  unsigned int u;
+  crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c
index 8fd8a8aa7b..1289a82e62 100644
--- a/src/lib/crypt_ops/crypto_rsa.c
+++ b/src/lib/crypt_ops/crypto_rsa.c
@@ -59,7 +59,7 @@ crypto_get_rsa_padding(int padding)
     default: tor_assert(0); return -1; // LCOV_EXCL_LINE
     }
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Compare the public-key components of a and b.  Return non-zero iff
  * a==b.  A NULL key is considered to be distinct from all non-NULL
diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h
index 6d9cc8d30e..01e6d10906 100644
--- a/src/lib/crypt_ops/crypto_rsa.h
+++ b/src/lib/crypt_ops/crypto_rsa.h
@@ -122,7 +122,7 @@ struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env);
 crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa);
 MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,(
                                  crypto_pk_t *env,int private));
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 struct SECKEYPublicKeyStr;
@@ -132,7 +132,7 @@ const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey(
                                            const crypto_pk_t *key);
 const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey(
                                            const crypto_pk_t *key);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src);
 void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src);
@@ -143,6 +143,6 @@ struct SECItemStr;
 STATIC int secitem_uint_cmp(const struct SECItemStr *a,
                             const struct SECItemStr *b);
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_CRYPTO_RSA_H) */
diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c
index 7abf6716f0..fd8fda486e 100644
--- a/src/lib/crypt_ops/crypto_rsa_nss.c
+++ b/src/lib/crypt_ops/crypto_rsa_nss.c
@@ -156,7 +156,7 @@ crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private))
   tor_free(buf);
   return result;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Allocate and return storage for a public key.  The key itself will not yet
  * be set.
diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c
index 17eae24cc2..41d936d25b 100644
--- a/src/lib/crypt_ops/crypto_rsa_openssl.c
+++ b/src/lib/crypt_ops/crypto_rsa_openssl.c
@@ -54,7 +54,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k)
   const BIGNUM *p, *q;
   RSA_get0_factors(k->key, &p, &q);
   return p != NULL; /* XXX/yawning: Should we check q? */
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   return k && k->key && k->key->p;
 #endif /* defined(OPENSSL_1_1_API) */
 }
@@ -288,7 +288,7 @@ crypto_pk_num_bits(crypto_pk_t *env)
   tor_assert(n != NULL);
 
   return RSA_bits(env->key);
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   tor_assert(env->key->n);
   return BN_num_bits(env->key->n);
 #endif /* defined(OPENSSL_1_1_API) */
diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c
index 42276597d4..361db18927 100644
--- a/src/lib/crypt_ops/crypto_s2k.c
+++ b/src/lib/crypt_ops/crypto_s2k.c
@@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv < 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
       SECItem passItem = { .type = siBuffer,
                            .data = (unsigned char *) secret,
                            .len = (int)secret_len };
@@ -325,7 +325,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (alg)
         SECOID_DestroyAlgorithmID(alg, PR_TRUE);
       return rv;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
     }
 
     case S2K_TYPE_SCRYPT: {
@@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv != 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else /* !(defined(HAVE_SCRYPT)) */
+#else /* !defined(HAVE_SCRYPT) */
       return S2K_NO_SCRYPT_SUPPORT;
 #endif /* defined(HAVE_SCRYPT) */
     }
diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h
new file mode 100644
index 0000000000..894243b175
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_crypto.h
+ * \brief Declare subsystem object for the crypto module.
+ **/
+
+#ifndef TOR_CRYPTO_SYS_H
+#define TOR_CRYPTO_SYS_H
+
+extern const struct subsys_fns_t sys_crypto;
+
+#endif /* !defined(TOR_CRYPTO_SYS_H) */
diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c
index 67a1a9eb92..5e3f4a87a1 100644
--- a/src/lib/crypt_ops/crypto_util.c
+++ b/src/lib/crypt_ops/crypto_util.c
@@ -30,7 +30,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/err.h>
 #include <openssl/crypto.h>
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c
index 0dba64d595..c931b58369 100644
--- a/src/lib/crypt_ops/digestset.c
+++ b/src/lib/crypt_ops/digestset.c
@@ -11,7 +11,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/digestset.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /* Wrap our hash function to have the signature that the bloom filter
  * needs. */
diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h
index 91d53a0542..7d6d687342 100644
--- a/src/lib/crypt_ops/digestset.h
+++ b/src/lib/crypt_ops/digestset.h
@@ -26,4 +26,4 @@ void digestset_add(digestset_t *set, const char *addr);
 int digestset_probably_contains(const digestset_t *set,
                                 const char *addr);
 
-#endif
+#endif /* !defined(TOR_DIGESTSET_H) */
diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am
index 1022096fdc..1f58a33d38 100644
--- a/src/lib/crypt_ops/include.am
+++ b/src/lib/crypt_ops/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_cipher.c		\
 	src/lib/crypt_ops/crypto_curve25519.c		\
@@ -17,6 +18,8 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_ope.c          	\
 	src/lib/crypt_ops/crypto_pwbox.c		\
 	src/lib/crypt_ops/crypto_rand.c			\
+	src/lib/crypt_ops/crypto_rand_fast.c		\
+	src/lib/crypt_ops/crypto_rand_numeric.c		\
 	src/lib/crypt_ops/crypto_rsa.c			\
 	src/lib/crypt_ops/crypto_s2k.c			\
 	src/lib/crypt_ops/crypto_util.c                 \
@@ -25,12 +28,14 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 if USE_NSS
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_nss.c			\
+	src/lib/crypt_ops/crypto_digest_nss.c		\
 	src/lib/crypt_ops/crypto_dh_nss.c		\
 	src/lib/crypt_ops/crypto_nss_mgt.c		\
 	src/lib/crypt_ops/crypto_rsa_nss.c
 else
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_openssl.c			\
+	src/lib/crypt_ops/crypto_digest_openssl.c	\
 	src/lib/crypt_ops/crypto_rsa_openssl.c
 endif
 
@@ -48,6 +53,7 @@ src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_crypt_ops_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/crypt_ops/aes.h				\
 	src/lib/crypt_ops/compat_openssl.h		\
@@ -66,5 +72,6 @@ noinst_HEADERS +=					\
 	src/lib/crypt_ops/crypto_rand.h			\
 	src/lib/crypt_ops/crypto_rsa.h			\
 	src/lib/crypt_ops/crypto_s2k.h			\
+	src/lib/crypt_ops/crypto_sys.h			\
 	src/lib/crypt_ops/crypto_util.h                 \
 	src/lib/crypt_ops/digestset.h
diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am
index b46c43ba0c..83942ca4e0 100644
--- a/src/lib/ctime/include.am
+++ b/src/lib/ctime/include.am
@@ -11,6 +11,7 @@ else
 mulodi4_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_ctime_a_SOURCES =			\
 	$(mulodi4_source) 				\
 	src/ext/csiphash.c   				\
@@ -21,5 +22,6 @@ src_lib_libtor_ctime_testing_a_SOURCES = \
 src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@
 src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/ctime/di_ops.h
diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h
index a2ffbc51c2..b0d1eba0c5 100644
--- a/src/lib/defs/dh_sizes.h
+++ b/src/lib/defs/dh_sizes.h
@@ -19,4 +19,4 @@
 /** Length of our legacy DH keys. */
 #define DH1024_KEY_LEN (1024/8)
 
-#endif
+#endif /* !defined(TOR_DH_SIZES_H) */
diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h
index 525e5209d6..a0dd97a74d 100644
--- a/src/lib/defs/digest_sizes.h
+++ b/src/lib/defs/digest_sizes.h
@@ -24,4 +24,4 @@
 /** Length of the output of our 64-bit optimized message digests (SHA512). */
 #define DIGEST512_LEN 64
 
-#endif
+#endif /* !defined(TOR_DIGEST_SIZES_H) */
diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am
index 48ee7f29fc..84ee403771 100644
--- a/src/lib/defs/include.am
+++ b/src/lib/defs/include.am
@@ -1,5 +1,8 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += 			\
 	src/lib/defs/dh_sizes.h 	\
 	src/lib/defs/digest_sizes.h	\
+	src/lib/defs/logging_types.h	\
+	src/lib/defs/time.h      	\
 	src/lib/defs/x25519_sizes.h
diff --git a/src/lib/defs/logging_types.h b/src/lib/defs/logging_types.h
new file mode 100644
index 0000000000..d3eacde464
--- /dev/null
+++ b/src/lib/defs/logging_types.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file logging_types.h
+ *
+ * \brief Global definition for types used by logging systems.
+ **/
+
+#ifndef TOR_LOGGING_TYPES_H
+#define TOR_LOGGING_TYPES_H
+
+/* We define this here so that it can be used both by backtrace.h and
+ * log.h.
+ */
+
+/** Mask of zero or more log domains, OR'd together. */
+typedef uint64_t log_domain_mask_t;
+
+#endif /* !defined(TOR_LOGGING_TYPES_H) */
diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h
new file mode 100644
index 0000000000..459afbf42d
--- /dev/null
+++ b/src/lib/defs/time.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TIME_DEFS_H
+#define TOR_TIME_DEFS_H
+
+/**
+ * \file time.h
+ *
+ * \brief Definitions for timing-related constants.
+ **/
+
+/** How many microseconds per second */
+#define TOR_USEC_PER_SEC (1000000)
+/** How many nanoseconds per microsecond */
+#define TOR_NSEC_PER_USEC (1000)
+/* How many nanoseconds per millisecond */
+#define TOR_NSEC_PER_MSEC (1000*1000)
+
+#endif /* !defined(TOR_TIME_DEFS_H) */
diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h
index 8933a8866b..6431f0a2dd 100644
--- a/src/lib/defs/x25519_sizes.h
+++ b/src/lib/defs/x25519_sizes.h
@@ -33,4 +33,4 @@
 #define ED25519_BASE64_LEN 43
 #define ED25519_SIG_BASE64_LEN 86
 
-#endif
+#endif /* !defined(TOR_X25519_SIZES_H) */
diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include
new file mode 100644
index 0000000000..884f4c0dbc
--- /dev/null
+++ b/src/lib/dispatch/.may_include
@@ -0,0 +1,11 @@
+orconfig.h
+
+ext/tor_queue.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
\ No newline at end of file
diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h
new file mode 100644
index 0000000000..a9e655409a
--- /dev/null
+++ b/src/lib/dispatch/dispatch.h
@@ -0,0 +1,114 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_H
+#define TOR_DISPATCH_H
+
+#include "lib/dispatch/msgtypes.h"
+
+/**
+ * \file dispatch.h
+ * \brief Low-level APIs for message-passing system.
+ *
+ * This module implements message dispatch based on a set of short integer
+ * identifiers.  For a higher-level interface, see pubsub.h.
+ *
+ * Each message is represented as a generic msg_t object, and is discriminated
+ * by its message_id_t.  Messages are delivered by a dispatch_t object, which
+ * delivers each message to its recipients by a configured "channel".
+ *
+ * A "channel" is a means of delivering messages.  Every message_id_t must
+ * be associated with exactly one channel, identified by channel_id_t.
+ * When a channel receives messages, a callback is invoked to either process
+ * the messages immediately, or to cause them to be processed later.
+ *
+ * Every message_id_t has zero or more associated receiver functions set up in
+ * the dispatch_t object.  Once the dispatch_t object is created, receivers
+ * can be enabled or disabled [TODO], but not added or removed.
+ *
+ * Every message_id_t has an associated datatype, identified by a
+ * msg_type_id_t.  These datatypes can be associated with functions to
+ * (for example) free them, or format them for debugging.
+ *
+ * To setup a dispatch_t object, first create a dispatch_cfg_t object, and
+ * configure messages with their types, channels, and receivers.  Then, use
+ * dispatch_new() with that dispatch_cfg_t to create the dispatch_t object.
+ *
+ * (We use a two-phase contruction procedure here to enable better static
+ * reasoning about publish/subscribe relationships.)
+ *
+ * Once you have a dispatch_t, you can queue messages on it with
+ * dispatch_send*(), and cause those messages to be delivered with
+ * dispatch_flush().
+ **/
+
+/**
+ * A "dispatcher" is the highest-level object; it handles making sure that
+ * messages are received and delivered properly.  Only the mainloop
+ * should handle this type directly.
+ */
+typedef struct dispatch_t dispatch_t;
+
+struct dispatch_cfg_t;
+
+dispatch_t *dispatch_new(const struct dispatch_cfg_t *cfg);
+
+/**
+ * Free a dispatcher.  Tor does this at exit.
+ */
+#define dispatch_free(d) \
+  FREE_AND_NULL(dispatch_t, dispatch_free_, (d))
+
+void dispatch_free_(dispatch_t *);
+
+int dispatch_send(dispatch_t *d,
+                  subsys_id_t sender,
+                  channel_id_t channel,
+                  message_id_t msg,
+                  msg_type_id_t type,
+                  msg_aux_data_t auxdata);
+
+int dispatch_send_msg(dispatch_t *d, msg_t *m);
+
+int dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m);
+
+/* Flush up to <b>max_msgs</b> currently pending messages from the
+ * dispatcher.  Messages that are not pending when this function are
+ * called, are not flushed by this call.  Return 0 on success, -1 on
+ * unrecoverable error.
+ */
+int dispatch_flush(dispatch_t *, channel_id_t chan, int max_msgs);
+
+/**
+ * Function callback type used to alert some other module when a channel's
+ * queue changes from empty to nonempty.
+ *
+ * Ex 1: To cause messages to be processed immediately on-stack, this callback
+ * should invoke dispatch_flush() directly.
+ *
+ * Ex 2: To cause messages to be processed very soon, from the event queue,
+ * this callback should schedule an event callback to run dispatch_flush().
+ *
+ * Ex 3: To cause messages to be processed periodically, this function should
+ * do nothing, and a periodic event should invoke dispatch_flush().
+ **/
+typedef void (*dispatch_alertfn_t)(struct dispatch_t *,
+                                   channel_id_t, void *);
+
+int dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                          dispatch_alertfn_t fn, void *userdata);
+
+#define dispatch_free_msg(d,msg)                                \
+  STMT_BEGIN {                                                  \
+    msg_t **msg_tmp_ptr__ = &(msg);                             \
+    dispatch_free_msg_((d), *msg_tmp_ptr__);                    \
+    *msg_tmp_ptr__= NULL;                                       \
+  } STMT_END
+void dispatch_free_msg_(const dispatch_t *d, msg_t *msg);
+
+char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg);
+
+#endif /* !defined(TOR_DISPATCH_H) */
diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c
new file mode 100644
index 0000000000..b3a72ec22f
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.c
@@ -0,0 +1,141 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_cfg.c
+ * \brief Create and configure a dispatch_cfg_t.
+ *
+ * A dispatch_cfg_t object is used to configure a set of messages and
+ * associated information before creating a dispatch_t.
+ */
+
+#define DISPATCH_PRIVATE
+
+#include "orconfig.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/malloc/malloc.h"
+
+/**
+ * Create and return a new dispatch_cfg_t.
+ **/
+dispatch_cfg_t *
+dcfg_new(void)
+{
+  dispatch_cfg_t *cfg = tor_malloc(sizeof(dispatch_cfg_t));
+  cfg->type_by_msg = smartlist_new();
+  cfg->chan_by_msg = smartlist_new();
+  cfg->fns_by_type = smartlist_new();
+  cfg->recv_by_msg = smartlist_new();
+  return cfg;
+}
+
+/**
+ * Associate a message with a datatype.  Return 0 on success, -1 if a
+ * different type was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                  msg_type_id_t type)
+{
+  smartlist_grow(cfg->type_by_msg, msg+1);
+  msg_type_id_t *oldval = smartlist_get(cfg->type_by_msg, msg);
+  if (oldval != NULL && *oldval != type) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->type_by_msg, msg, tor_memdup(&type, sizeof(type)));
+  return 0;
+}
+
+/**
+ * Associate a message with a channel.  Return 0 on success, -1 if a
+ * different channel was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                  channel_id_t chan)
+{
+  smartlist_grow(cfg->chan_by_msg, msg+1);
+  channel_id_t *oldval = smartlist_get(cfg->chan_by_msg, msg);
+  if (oldval != NULL && *oldval != chan) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->chan_by_msg, msg, tor_memdup(&chan, sizeof(chan)));
+  return 0;
+}
+
+/**
+ * Associate a set of functions with a datatype. Return 0 on success, -1 if
+ * different functions were previously associated with the type.
+ **/
+int
+dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                  const dispatch_typefns_t *fns)
+{
+  smartlist_grow(cfg->fns_by_type, type+1);
+  dispatch_typefns_t *oldfns = smartlist_get(cfg->fns_by_type, type);
+  if (oldfns && (oldfns->free_fn != fns->free_fn ||
+                 oldfns->fmt_fn != fns->fmt_fn))
+    return -1;
+  if (!oldfns)
+    smartlist_set(cfg->fns_by_type, type, tor_memdup(fns, sizeof(*fns)));
+  return 0;
+}
+
+/**
+ * Associate a receiver with a message ID.  Multiple receivers may be
+ * associated with a single messasge ID.
+ *
+ * Return 0 on success, on failure.
+ **/
+int
+dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+              subsys_id_t sys, recv_fn_t fn)
+{
+  smartlist_grow(cfg->recv_by_msg, msg+1);
+  smartlist_t *receivers = smartlist_get(cfg->recv_by_msg, msg);
+  if (!receivers) {
+    receivers = smartlist_new();
+    smartlist_set(cfg->recv_by_msg, msg, receivers);
+  }
+
+  dispatch_rcv_t *rcv = tor_malloc(sizeof(dispatch_rcv_t));
+  rcv->sys = sys;
+  rcv->enabled = true;
+  rcv->fn = fn;
+  smartlist_add(receivers, (void*)rcv);
+  return 0;
+}
+
+/** Helper: release all storage held by <b>cfg</b>. */
+void
+dcfg_free_(dispatch_cfg_t *cfg)
+{
+  if (!cfg)
+    return;
+
+  SMARTLIST_FOREACH(cfg->type_by_msg, msg_type_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->chan_by_msg, channel_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->fns_by_type, dispatch_typefns_t *, f, tor_free(f));
+  smartlist_free(cfg->type_by_msg);
+  smartlist_free(cfg->chan_by_msg);
+  smartlist_free(cfg->fns_by_type);
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, receivers) {
+    if (!receivers)
+      continue;
+    SMARTLIST_FOREACH(receivers, dispatch_rcv_t *, rcv, tor_free(rcv));
+    smartlist_free(receivers);
+  } SMARTLIST_FOREACH_END(receivers);
+  smartlist_free(cfg->recv_by_msg);
+
+  tor_free(cfg);
+}
diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h
new file mode 100644
index 0000000000..348dce8d40
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_CFG_H
+#define TOR_DISPATCH_CFG_H
+
+#include "lib/dispatch/msgtypes.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * A "dispatch_cfg" is the configuration used to set up a dispatcher.
+ * It is created and accessed with a set of dcfg_* functions, and then
+ * used with dispatcher_new() to make the dispatcher.
+ */
+typedef struct dispatch_cfg_t dispatch_cfg_t;
+
+dispatch_cfg_t *dcfg_new(void);
+
+int dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                      msg_type_id_t type);
+
+int dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                      channel_id_t chan);
+
+int dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                      const dispatch_typefns_t *fns);
+
+int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+                  subsys_id_t sys, recv_fn_t fn);
+
+/** Free a dispatch_cfg_t. */
+#define dcfg_free(cfg) \
+  FREE_AND_NULL(dispatch_cfg_t, dcfg_free_, (cfg))
+
+void dcfg_free_(dispatch_cfg_t *cfg);
+
+#ifdef DISPATCH_NEW_PRIVATE
+struct smartlist_t;
+STATIC int max_in_u16_sl(const struct smartlist_t *sl, int dflt);
+#endif
+
+#endif /* !defined(TOR_DISPATCH_CFG_H) */
diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h
new file mode 100644
index 0000000000..57b6f0347f
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg_st.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_CFG_ST_H
+#define TOR_DISPATCH_CFG_ST_H
+
+struct smartlist_t;
+
+/* Information needed to create a dispatcher, but in a less efficient, more
+ * mutable format. */
+struct dispatch_cfg_t {
+  /** A list of msg_type_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *type_by_msg;
+  /** A list of channel_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *chan_by_msg;
+  /** A list of dispatch_rcv_t, indexed by msg_type_id_t. */
+  struct smartlist_t *fns_by_type;
+  /** A list of dispatch_typefns_t, indexed by msg_t. */
+  struct smartlist_t *recv_by_msg;
+};
+
+#endif /* !defined(TOR_DISPATCH_CFG_ST_H) */
diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c
new file mode 100644
index 0000000000..da54f9b437
--- /dev/null
+++ b/src/lib/dispatch/dispatch_core.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_core.c
+ * \brief Core module for sending and receiving messages.
+ */
+
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_naming.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Use <b>d</b> to drop all storage held for <b>msg</b>.
+ *
+ * (We need the dispatcher so we know how to free the auxiliary data.)
+ **/
+void
+dispatch_free_msg_(const dispatch_t *d, msg_t *msg)
+{
+  if (!msg)
+    return;
+
+  d->typefns[msg->type].free_fn(msg->aux_data__);
+  tor_free(msg);
+}
+
+/**
+ * Format the auxiliary data held by msg.
+ **/
+char *
+dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg)
+{
+  if (!msg)
+    return NULL;
+
+  return d->typefns[msg->type].fmt_fn(msg->aux_data__);
+}
+
+/**
+ * Release all storage held by <b>d</b>.
+ **/
+void
+dispatch_free_(dispatch_t *d)
+{
+  if (d == NULL)
+    return;
+
+  size_t n_queues = d->n_queues;
+  for (size_t i = 0; i < n_queues; ++i) {
+    msg_t *m, *mtmp;
+    TOR_SIMPLEQ_FOREACH_SAFE(m, &d->queues[i].queue, next, mtmp) {
+      dispatch_free_msg(d, m);
+    }
+  }
+
+  size_t n_msgs = d->n_msgs;
+
+  for (size_t i = 0; i < n_msgs; ++i) {
+    tor_free(d->table[i]);
+  }
+  tor_free(d->table);
+  tor_free(d->typefns);
+  tor_free(d->queues);
+
+  // This is the only time we will treat d->cfg as non-const.
+  //dispatch_cfg_free_((dispatch_items_t *) d->cfg);
+
+  tor_free(d);
+}
+
+/**
+ * Tell the dispatcher to call <b>fn</b> with <b>userdata</b> whenever
+ * <b>chan</b> becomes nonempty.  Return 0 on success, -1 on error.
+ **/
+int
+dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                      dispatch_alertfn_t fn, void *userdata)
+{
+  if (BUG(chan >= d->n_queues))
+    return -1;
+
+  dqueue_t *q = &d->queues[chan];
+  q->alert_fn = fn;
+  q->alert_fn_arg = userdata;
+  return 0;
+}
+
+/**
+ * Send a message on the appropriate channel notifying that channel if
+ * necessary.
+ *
+ * This function takes ownership of the auxiliary data; it can't be static or
+ * stack-allocated, and the caller is not allowed to use it afterwards.
+ *
+ * This function does not check the various vields of the message object for
+ * consistency.
+ **/
+int
+dispatch_send(dispatch_t *d,
+              subsys_id_t sender,
+              channel_id_t channel,
+              message_id_t msg,
+              msg_type_id_t type,
+              msg_aux_data_t auxdata)
+{
+  if (!d->table[msg]) {
+    /* Fast path: nobody wants this data. */
+
+    d->typefns[type].free_fn(auxdata);
+    return 0;
+  }
+
+  msg_t *m = tor_malloc(sizeof(msg_t));
+
+  m->sender = sender;
+  m->channel = channel;
+  m->msg = msg;
+  m->type = type;
+  memcpy(&m->aux_data__, &auxdata, sizeof(msg_aux_data_t));
+
+  return dispatch_send_msg(d, m);
+}
+
+int
+dispatch_send_msg(dispatch_t *d, msg_t *m)
+{
+  if (BUG(!d))
+    goto err;
+  if (BUG(!m))
+    goto err;
+  if (BUG(m->channel >= d->n_queues))
+    goto err;
+  if (BUG(m->msg >= d->n_msgs))
+    goto err;
+
+  dtbl_entry_t *ent = d->table[m->msg];
+  if (ent) {
+    if (BUG(m->type != ent->type))
+      goto err;
+    if (BUG(m->channel != ent->channel))
+      goto err;
+  }
+
+  return dispatch_send_msg_unchecked(d, m);
+ err:
+  /* Probably it isn't safe to free m, since type could be wrong. */
+  return -1;
+}
+
+/**
+ * Send a message on the appropriate queue, notifying that queue if necessary.
+ *
+ * This function takes ownership of the message object and its auxiliary data;
+ * it can't be static or stack-allocated, and the caller isn't allowed to use
+ * it afterwards.
+ *
+ * This function does not check the various fields of the message object for
+ * consistency, and can crash if they are out of range.  Only functions that
+ * have already constructed the message in a safe way, or checked it for
+ * correctness themselves, should call this function.
+ **/
+int
+dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m)
+{
+  /* Find the right queue. */
+  dqueue_t *q = &d->queues[m->channel];
+  bool was_empty = TOR_SIMPLEQ_EMPTY(&q->queue);
+
+  /* Append the message. */
+  TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next);
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Queued: %s (%s) from %s, on %s.",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  /* If we just made the queue nonempty for the first time, call the alert
+   * function. */
+  if (was_empty) {
+    q->alert_fn(d, m->channel, q->alert_fn_arg);
+  }
+
+  return 0;
+}
+
+/**
+ * Run all of the callbacks on <b>d</b> associated with <b>m</b>.
+ **/
+static void
+dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m)
+{
+  tor_assert(m->msg <= d->n_msgs);
+  dtbl_entry_t *ent = d->table[m->msg];
+  int n_fns = ent->n_fns;
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Delivering: %s (%s) from %s, on %s:",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  int i;
+  for (i=0; i < n_fns; ++i) {
+    if (ent->rcv[i].enabled) {
+      log_debug(LD_MESG, "  Delivering to %s.",
+                get_subsys_id_name(ent->rcv[i].sys));
+      ent->rcv[i].fn(m);
+    }
+  }
+}
+
+/**
+ * Run up to <b>max_msgs</b> callbacks for messages on the channel <b>ch</b>
+ * on the given dispatcher.  Return 0 on success or recoverable failure,
+ * -1 on unrecoverable error.
+ **/
+int
+dispatch_flush(dispatch_t *d, channel_id_t ch, int max_msgs)
+{
+  if (BUG(ch >= d->n_queues))
+    return 0;
+
+  int n_flushed = 0;
+  dqueue_t *q = &d->queues[ch];
+
+  while (n_flushed < max_msgs) {
+    msg_t *m = TOR_SIMPLEQ_FIRST(&q->queue);
+    if (!m)
+      break;
+    TOR_SIMPLEQ_REMOVE_HEAD(&q->queue, next);
+    dispatcher_run_msg_cbs(d, m);
+    dispatch_free_msg(d, m);
+    ++n_flushed;
+  }
+
+  return 0;
+}
diff --git a/src/lib/dispatch/dispatch_naming.c b/src/lib/dispatch/dispatch_naming.c
new file mode 100644
index 0000000000..83d9a2d604
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.c
@@ -0,0 +1,63 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "lib/cc/compat_compiler.h"
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+
+#include "lib/log/util_bug.h"
+#include "lib/log/log.h"
+
+#include <stdlib.h>
+
+/** Global namemap for message IDs. */
+static namemap_t message_id_map = NAMEMAP_INIT();
+/** Global namemap for subsystem IDs. */
+static namemap_t subsys_id_map = NAMEMAP_INIT();
+/** Global namemap for channel IDs. */
+static namemap_t channel_id_map = NAMEMAP_INIT();
+/** Global namemap for message type IDs. */
+static namemap_t msg_type_id_map = NAMEMAP_INIT();
+
+void
+dispatch_naming_init(void)
+{
+}
+
+/* Helper macro: declare functions to map IDs to and from names for a given
+ * type in a namemap_t.
+ */
+#define DECLARE_ID_MAP_FNS(type)                                        \
+  type##_id_t                                                           \
+  get_##type##_id(const char *name)                                     \
+  {                                                                     \
+    unsigned u = namemap_get_or_create_id(&type##_id_map, name);        \
+    tor_assert(u != NAMEMAP_ERR);                                       \
+    tor_assert(u != ERROR_ID);                                          \
+    return (type##_id_t) u;                                             \
+  }                                                                     \
+  const char *                                                          \
+  get_##type##_id_name(type##_id_t id)                                  \
+  {                                                                     \
+    return namemap_fmt_name(&type##_id_map, id);                        \
+  }                                                                     \
+  size_t                                                                \
+  get_num_##type##_ids(void)                                            \
+  {                                                                     \
+    return namemap_get_size(&type##_id_map);                            \
+  }                                                                     \
+  EAT_SEMICOLON
+
+DECLARE_ID_MAP_FNS(message);
+DECLARE_ID_MAP_FNS(channel);
+DECLARE_ID_MAP_FNS(subsys);
+DECLARE_ID_MAP_FNS(msg_type);
diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h
new file mode 100644
index 0000000000..fd6c83cc12
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_NAMING_H
+#define TOR_DISPATCH_NAMING_H
+
+#include "lib/dispatch/msgtypes.h"
+#include <stddef.h>
+
+/**
+ * Return an existing channel ID by name, allocating the channel ID if
+ * if necessary.  Returns ERROR_ID if we have run out of
+ * channels
+ */
+channel_id_t get_channel_id(const char *);
+/**
+ * Return the name corresponding to a given channel ID.
+ **/
+const char *get_channel_id_name(channel_id_t);
+/**
+ * Return the total number of _named_ channel IDs.
+ **/
+size_t get_num_channel_ids(void);
+
+/* As above, but for messages. */
+message_id_t get_message_id(const char *);
+const char *get_message_id_name(message_id_t);
+size_t get_num_message_ids(void);
+
+/* As above, but for subsystems */
+subsys_id_t get_subsys_id(const char *);
+const char *get_subsys_id_name(subsys_id_t);
+size_t get_num_subsys_ids(void);
+
+/* As above, but for types. Note that types additionally must be
+ * "defined", if any message is to use them. */
+msg_type_id_t get_msg_type_id(const char *);
+const char *get_msg_type_id_name(msg_type_id_t);
+size_t get_num_msg_type_ids(void);
+
+void dispatch_naming_init(void);
+
+#endif /* !defined(TOR_DISPATCH_NAMING_H) */
diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c
new file mode 100644
index 0000000000..d8e59d610a
--- /dev/null
+++ b/src/lib/dispatch/dispatch_new.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_new.c
+ * \brief Code to construct a dispatch_t from a dispatch_cfg_t.
+ **/
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+
+#include "lib/cc/ctassert.h"
+#include "lib/intmath/cmp.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/** Given a smartlist full of (possibly NULL) pointers to uint16_t values,
+ * return the largest value, or dflt if the list is empty. */
+STATIC int
+max_in_u16_sl(const smartlist_t *sl, int dflt)
+{
+  uint16_t *maxptr = NULL;
+  SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) {
+    if (!maxptr)
+      maxptr = u;
+    else if (u && *u > *maxptr)
+      maxptr = u;
+  } SMARTLIST_FOREACH_END(u);
+
+  return maxptr ? *maxptr : dflt;
+}
+
+/* The above function is only safe to call if we are sure that channel_id_t
+ * and msg_type_id_t are really uint16_t.  They should be so defined in
+ * msgtypes.h, but let's be extra cautious.
+ */
+CTASSERT(sizeof(uint16_t) == sizeof(msg_type_id_t));
+CTASSERT(sizeof(uint16_t) == sizeof(channel_id_t));
+
+/** Helper: Format an unformattable message auxiliary data item: just return a
+* copy of the string <>. */
+static char *
+type_fmt_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+  return tor_strdup("<>");
+}
+
+/** Helper: Free an unfreeable message auxiliary data item: do nothing. */
+static void
+type_free_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+}
+
+/** Type functions to use when no type functions are provided. */
+static dispatch_typefns_t nop_typefns = {
+  .free_fn = type_free_nop,
+  .fmt_fn = type_fmt_nop
+};
+
+/**
+ * Alert function to use when none is configured: do nothing.
+ **/
+static void
+alert_fn_nop(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)d;
+  (void)ch;
+  (void)arg;
+}
+
+/**
+ * Given a list of recvfn_t, create and return a new dtbl_entry_t mapping
+ * to each of those functions.
+ **/
+static dtbl_entry_t *
+dtbl_entry_from_lst(smartlist_t *receivers)
+{
+  if (!receivers)
+    return NULL;
+
+  size_t n_recv = smartlist_len(receivers);
+  dtbl_entry_t *ent;
+  ent = tor_malloc_zero(offsetof(dtbl_entry_t, rcv) +
+                        sizeof(dispatch_rcv_t) * n_recv);
+
+  ent->n_fns = n_recv;
+
+  SMARTLIST_FOREACH_BEGIN(receivers, const dispatch_rcv_t *, rcv) {
+    memcpy(&ent->rcv[rcv_sl_idx], rcv, sizeof(*rcv));
+    if (rcv->enabled) {
+      ++ent->n_enabled;
+    }
+  } SMARTLIST_FOREACH_END(rcv);
+
+  return ent;
+}
+
+/** Create and return a new dispatcher from a given dispatch_cfg_t. */
+dispatch_t *
+dispatch_new(const dispatch_cfg_t *cfg)
+{
+  dispatch_t *d = tor_malloc_zero(sizeof(dispatch_t));
+
+  /* Any message that has a type or a receiver counts towards our messages */
+  const size_t n_msgs = MAX(smartlist_len(cfg->type_by_msg),
+                            smartlist_len(cfg->recv_by_msg)) + 1;
+
+  /* Any channel that any message has counts towards the number of channels. */
+  const size_t n_chans = (size_t)
+    MAX(1, max_in_u16_sl(cfg->chan_by_msg,0)) + 1;
+
+  /* Any type that a message has, or that has functions, counts towards
+   * the number of types. */
+  const size_t n_types = (size_t) MAX(max_in_u16_sl(cfg->type_by_msg,0),
+                                      smartlist_len(cfg->fns_by_type)) + 1;
+
+  d->n_msgs = n_msgs;
+  d->n_queues = n_chans;
+  d->n_types = n_types;
+
+  /* Initialize the array of type-functions. */
+  d->typefns = tor_calloc(n_types, sizeof(dispatch_typefns_t));
+  for (size_t i = 0; i < n_types; ++i) {
+    /* Default to no-op for everything... */
+    memcpy(&d->typefns[i], &nop_typefns, sizeof(dispatch_typefns_t));
+  }
+  SMARTLIST_FOREACH_BEGIN(cfg->fns_by_type, dispatch_typefns_t *, fns) {
+    /* Set the functions if they are provided. */
+    if (fns) {
+      if (fns->free_fn)
+        d->typefns[fns_sl_idx].free_fn = fns->free_fn;
+      if (fns->fmt_fn)
+        d->typefns[fns_sl_idx].fmt_fn = fns->fmt_fn;
+    }
+  } SMARTLIST_FOREACH_END(fns);
+
+  /* Initialize the message queues: one for each channel. */
+  d->queues = tor_calloc(d->n_queues, sizeof(dqueue_t));
+  for (size_t i = 0; i < d->n_queues; ++i) {
+    TOR_SIMPLEQ_INIT(&d->queues[i].queue);
+    d->queues[i].alert_fn = alert_fn_nop;
+  }
+
+  /* Build the dispatch tables mapping message IDs to receivers. */
+  d->table = tor_calloc(d->n_msgs, sizeof(dtbl_entry_t *));
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, rcv) {
+    d->table[rcv_sl_idx] = dtbl_entry_from_lst(rcv);
+  } SMARTLIST_FOREACH_END(rcv);
+
+  /* Fill in the empty entries in the dispatch tables:
+   * types and channels for each message. */
+  SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, msg_type_id_t *, type) {
+    if (d->table[type_sl_idx])
+      d->table[type_sl_idx]->type = *type;
+  } SMARTLIST_FOREACH_END(type);
+
+  SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, channel_id_t *, chan) {
+    if (d->table[chan_sl_idx])
+      d->table[chan_sl_idx]->channel = *chan;
+  } SMARTLIST_FOREACH_END(chan);
+
+  return d;
+}
diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h
new file mode 100644
index 0000000000..ee42518b5a
--- /dev/null
+++ b/src/lib/dispatch/dispatch_st.h
@@ -0,0 +1,108 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_st.h
+ *
+ * \brief private structures used for the dispatcher module
+ */
+
+#ifndef TOR_DISPATCH_ST_H
+#define TOR_DISPATCH_ST_H
+
+#ifdef DISPATCH_PRIVATE
+
+#include "lib/container/smartlist.h"
+
+/**
+ * Information about the recipient of a message.
+ **/
+typedef struct dispatch_rcv_t {
+  /** The subsystem receiving a message. */
+  subsys_id_t sys;
+  /** True iff this recipient is enabled. */
+  bool enabled;
+  /** The function that will handle the message. */
+  recv_fn_t fn;
+} dispatch_rcv_t;
+
+/**
+ * Information used by a dispatcher to handle and dispatch a single message
+ * ID.  It maps that message ID to its type, channel, and list of receiver
+ * functions.
+ *
+ * This structure is used when the dispatcher is running.
+ **/
+typedef struct dtbl_entry_t {
+  /** The number of enabled non-stub subscribers for this message.
+   *
+   * Note that for now, this will be the same as <b>n_fns</b>, since there is
+   * no way to turn these subscribers on an off yet. */
+  uint16_t n_enabled;
+  /** The channel that handles this message. */
+  channel_id_t channel;
+  /** The associated C type for this message. */
+  msg_type_id_t type;
+  /**
+   * The number of functions pointers for subscribers that receive this
+   * message, in rcv. */
+  uint16_t n_fns;
+  /**
+   * The recipients for this message.
+   */
+  dispatch_rcv_t rcv[FLEXIBLE_ARRAY_MEMBER];
+} dtbl_entry_t;
+
+/**
+ * A queue of messages for a given channel, used by a live dispatcher.
+ */
+typedef struct dqueue_t {
+  /** The queue of messages itself. */
+  TOR_SIMPLEQ_HEAD( , msg_t) queue;
+  /** A function to be called when the queue becomes nonempty. */
+  dispatch_alertfn_t alert_fn;
+  /** An argument for the alert_fn. */
+  void *alert_fn_arg;
+} dqueue_t ;
+
+/**
+ * A single dispatcher for cross-module messages.
+ */
+struct dispatch_t {
+  /**
+   * The length of <b>table</b>: the number of message IDs that this
+   * dispatcher can handle.
+   */
+  size_t n_msgs;
+  /**
+   * The length of <b>queues</b>: the number of channels that this dispatcher
+   * has configured.
+   */
+  size_t n_queues;
+  /**
+   * The length of <b>typefns</b>: the number of C type IDs that this
+   * dispatcher has configured.
+   */
+  size_t n_types;
+  /**
+   * An array of message queues, indexed by channel ID.
+   */
+  dqueue_t *queues;
+  /**
+   * An array of entries about how to handle particular message types, indexed
+   * by message ID.
+   */
+  dtbl_entry_t **table;
+  /**
+   * An array of function tables for manipulating types, index by message
+   * type ID.
+   **/
+  dispatch_typefns_t *typefns;
+};
+
+#endif /* defined(DISPATCH_PRIVATE) */
+
+#endif /* !defined(TOR_DISPATCH_ST_H) */
diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am
new file mode 100644
index 0000000000..4a0e0dfd90
--- /dev/null
+++ b/src/lib/dispatch/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-dispatch.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_dispatch_a_SOURCES =			\
+	src/lib/dispatch/dispatch_cfg.c			\
+	src/lib/dispatch/dispatch_core.c		\
+	src/lib/dispatch/dispatch_naming.c		\
+	src/lib/dispatch/dispatch_new.c
+
+src_lib_libtor_dispatch_testing_a_SOURCES = \
+	$(src_lib_libtor_dispatch_a_SOURCES)
+src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/dispatch/dispatch.h			\
+	src/lib/dispatch/dispatch_cfg.h 		\
+	src/lib/dispatch/dispatch_cfg_st.h 		\
+	src/lib/dispatch/dispatch_naming.h		\
+	src/lib/dispatch/dispatch_st.h			\
+	src/lib/dispatch/msgtypes.h
diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h
new file mode 100644
index 0000000000..b4c4a10248
--- /dev/null
+++ b/src/lib/dispatch/msgtypes.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file msgtypes.h
+ * \brief Types used for messages in the dispatcher code.
+ **/
+
+#ifndef TOR_DISPATCH_MSGTYPES_H
+#define TOR_DISPATCH_MSGTYPES_H
+
+#include <stdint.h>
+
+#include "ext/tor_queue.h"
+
+/**
+ * These types are aliases for subsystems, channels, and message IDs.
+ **/
+typedef uint16_t subsys_id_t;
+typedef uint16_t channel_id_t;
+typedef uint16_t message_id_t;
+
+/**
+ * This identifies a C type that can be sent along with a message.
+ **/
+typedef uint16_t msg_type_id_t;
+
+/**
+ * An ID value returned for *_type_t when none exists.
+ */
+#define ERROR_ID 65535
+
+/**
+ * Auxiliary (untyped) data sent along with a message.
+ *
+ * We define this as a union of a pointer and a u64, so that the integer
+ * types will have the same range across platforms.
+ **/
+typedef union {
+  void *ptr;
+  uint64_t u64;
+} msg_aux_data_t;
+
+/**
+ * Structure of a received message.
+ **/
+typedef struct msg_t {
+  TOR_SIMPLEQ_ENTRY(msg_t) next;
+  subsys_id_t sender;
+  channel_id_t channel;
+  message_id_t msg;
+  /** We could omit this field, since it is implicit in the message type, but
+   * IMO let's leave it in for safety. */
+  msg_type_id_t type;
+  /** Untyped auxiliary data. You shouldn't have to mess with this
+   * directly. */
+  msg_aux_data_t aux_data__;
+} msg_t;
+
+/**
+ * A function that a subscriber uses to receive a message.
+ **/
+typedef void (*recv_fn_t)(const msg_t *m);
+
+/**
+ * Table of functions to use for a given C type.  Any omitted (NULL) functions
+ * will be treated as no-ops.
+ **/
+typedef struct dispatch_typefns_t {
+  /** Release storage held for the auxiliary data of this type. */
+  void (*free_fn)(msg_aux_data_t);
+  /** Format and return a newly allocated string describing the contents
+   * of this data element. */
+  char *(*fmt_fn)(msg_aux_data_t);
+} dispatch_typefns_t;
+
+#endif /* !defined(TOR_DISPATCH_MSGTYPES_H) */
diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include
index 7c2ef36929..c9bf4b1786 100644
--- a/src/lib/encoding/.may_include
+++ b/src/lib/encoding/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/container/*.h
 lib/ctime/*.h
 lib/encoding/*.h
 lib/intmath/*.h
diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c
index bd063440d6..fc64e014e7 100644
--- a/src/lib/encoding/binascii.c
+++ b/src/lib/encoding/binascii.c
@@ -84,7 +84,7 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
 }
 
 /** Implements base32 decoding as in RFC 4648.
- * Returns 0 if successful, -1 otherwise.
+ * Return the number of bytes decoded if successful; -1 otherwise.
  */
 int
 base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
@@ -147,7 +147,7 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
   memset(tmp, 0, srclen); /* on the heap, this should be safe */
   tor_free(tmp);
   tmp = NULL;
-  return 0;
+  return i;
 }
 
 #define BASE64_OPENSSL_LINELEN 64
@@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags)
   return enclen;
 }
 
+/** Return an upper bound on the number of bytes that might be needed to hold
+ * the data from decoding the base64 string <b>srclen</b>.  This is only an
+ * upper bound, since some part of the base64 string might be padding or
+ * space. */
+size_t
+base64_decode_maxsize(size_t srclen)
+{
+  tor_assert(srclen < INT_MAX / 3);
+
+  return CEIL_DIV(srclen * 3, 4);
+}
+
 /** Internal table mapping 6 bit values to the Base64 alphabet. */
 static const char base64_encode_table[64] = {
   'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
@@ -309,8 +321,10 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
   return (int) enclen;
 }
 
-/** As base64_encode, but do not add any internal spaces or external padding
- * to the output stream. */
+/** As base64_encode, but do not add any internal spaces, and remove external
+ * padding from the output stream.
+ * dest must be at least base64_encode_size(srclen, 0), including space for
+ * the removed external padding. */
 int
 base64_encode_nopad(char *dest, size_t destlen,
                     const uint8_t *src, size_t srclen)
diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h
index 7e3cc04f09..40c5593b11 100644
--- a/src/lib/encoding/binascii.h
+++ b/src/lib/encoding/binascii.h
@@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen);
 
 #define BASE64_ENCODE_MULTILINE 1
 size_t base64_encode_size(size_t srclen, int flags);
+size_t base64_decode_maxsize(size_t srclen);
 int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
                   int flags);
 int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
@@ -57,4 +58,4 @@ size_t base32_encoded_size(size_t srclen);
 void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 
-#endif /* !defined(TOR_UTIL_FORMAT_H) */
+#endif /* !defined(TOR_BINASCII_H) */
diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c
index 8110f3dd9c..0d8384db13 100644
--- a/src/lib/encoding/confline.c
+++ b/src/lib/encoding/confline.c
@@ -82,6 +82,19 @@ config_line_find(const config_line_t *lines,
   return NULL;
 }
 
+/** As config_line_find(), but perform a case-insensitive comparison. */
+const config_line_t *
+config_line_find_case(const config_line_t *lines,
+                      const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcasecmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
 /** Auxiliary function that does all the work of config_get_lines.
  * <b>recursion_level</b> is the count of how many nested %includes we have.
  * <b>opened_lst</b> will have a list of opened files if provided.
@@ -243,7 +256,7 @@ config_lines_dup_and_filter(const config_line_t *inp,
 /** Return true iff a and b contain identical keys and values in identical
  * order. */
 int
-config_lines_eq(config_line_t *a, config_line_t *b)
+config_lines_eq(const config_line_t *a, const config_line_t *b)
 {
   while (a && b) {
     if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h
index 3d9ae8a662..12c554c6e7 100644
--- a/src/lib/encoding/confline.h
+++ b/src/lib/encoding/confline.h
@@ -48,7 +48,9 @@ config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
                                            const char *key);
 const config_line_t *config_line_find(const config_line_t *lines,
                                       const char *key);
-int config_lines_eq(config_line_t *a, config_line_t *b);
+const config_line_t *config_line_find_case(const config_line_t *lines,
+                                           const char *key);
+int config_lines_eq(const config_line_t *a, const config_line_t *b);
 int config_count_key(const config_line_t *a, const char *key);
 void config_free_lines_(config_line_t *front);
 #define config_free_lines(front) \
diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am
index 2d2aa3988a..48d0120bfc 100644
--- a/src/lib/encoding/include.am
+++ b/src/lib/encoding/include.am
@@ -4,12 +4,15 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-encoding-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_encoding_a_SOURCES =			\
 	src/lib/encoding/binascii.c			\
 	src/lib/encoding/confline.c			\
 	src/lib/encoding/cstring.c			\
 	src/lib/encoding/keyval.c			\
+	src/lib/encoding/kvline.c			\
 	src/lib/encoding/pem.c				\
+	src/lib/encoding/qstring.c			\
 	src/lib/encoding/time_fmt.c
 
 src_lib_libtor_encoding_testing_a_SOURCES = \
@@ -17,10 +20,13 @@ src_lib_libtor_encoding_testing_a_SOURCES = \
 src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/encoding/binascii.h			\
 	src/lib/encoding/confline.h			\
 	src/lib/encoding/cstring.h			\
 	src/lib/encoding/keyval.h			\
+	src/lib/encoding/kvline.h			\
 	src/lib/encoding/pem.h				\
+	src/lib/encoding/qstring.h			\
 	src/lib/encoding/time_fmt.h
diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h
index cd327b7a82..dcddfa3396 100644
--- a/src/lib/encoding/keyval.h
+++ b/src/lib/encoding/keyval.h
@@ -14,4 +14,4 @@
 
 int string_is_key_value(int severity, const char *string);
 
-#endif
+#endif /* !defined(TOR_KEYVAL_H) */
diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c
new file mode 100644
index 0000000000..d4a8f510ba
--- /dev/null
+++ b/src/lib/encoding/kvline.c
@@ -0,0 +1,289 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.c
+ *
+ * \brief Manipulating lines of key-value pairs.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
+
+/** Return true iff we need to quote and escape the string <b>s</b> to encode
+ * it. */
+static bool
+needs_escape(const char *s, bool as_keyless_val)
+{
+  if (as_keyless_val && *s == 0)
+    return true;
+
+  for (; *s; ++s) {
+    if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) ||
+        *s == '\'' || *s == '\"') {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff the key in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_key(const config_line_t *line)
+{
+  return line->key == NULL || strlen(line->key) == 0;
+}
+
+/**
+ * Return true iff the value in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_val(const config_line_t *line)
+{
+  return line->value == NULL || strlen(line->value) == 0;
+}
+
+/**
+ * Return true iff the all the lines in <b>line</b> can be encoded
+ * using <b>flags</b>.
+ **/
+static bool
+kvline_can_encode_lines(const config_line_t *line, unsigned flags)
+{
+  for ( ; line; line = line->next) {
+    const bool keyless = line_has_no_key(line);
+    if (keyless) {
+      if (! (flags & KV_OMIT_KEYS)) {
+        /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */
+        return false;
+      }
+      if (strchr(line->value, '=') && !( flags & KV_QUOTED)) {
+        /* We can't have a keyless value with = without quoting it. */
+        return false;
+      }
+    }
+
+    if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) {
+      /* If KV_QUOTED is false, we can't encode a value that needs quotes. */
+      return false;
+    }
+    if (line->key && strlen(line->key) &&
+        (needs_escape(line->key, false) || strchr(line->key, '='))) {
+      /* We can't handle keys that need quoting. */
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value'
+ * pairs, using the provided <b>flags</b> to encode it.  Return a newly
+ * allocated string on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then all values that contain
+ * spaces or unusual characters are escaped and quoted.  Otherwise, such
+ * values are not allowed.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are
+ * allowed, and are encoded as 'Value'.  Otherwise, such pairs are not
+ * allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then an empty value is
+ * encoded as 'Key', not as 'Key=' or 'Key=""'.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * KV_QUOTED_QSTRING is not supported.
+ */
+char *
+kvline_encode(const config_line_t *line,
+              unsigned flags)
+{
+  tor_assert(! (flags & KV_QUOTED_QSTRING));
+
+  if (!kvline_can_encode_lines(line, flags))
+    return NULL;
+
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+
+  smartlist_t *elements = smartlist_new();
+
+  for (; line; line = line->next) {
+
+    const char *k = "";
+    const char *eq = "=";
+    const char *v = "";
+    const bool keyless = line_has_no_key(line);
+    bool esc = needs_escape(line->value, keyless);
+    char *tmp = NULL;
+
+    if (! keyless) {
+      k = line->key;
+    } else {
+      eq = "";
+      if (strchr(line->value, '=')) {
+        esc = true;
+      }
+    }
+
+    if ((flags & KV_OMIT_VALS) && line_has_no_val(line)) {
+      eq = "";
+      v = "";
+    } else if (esc) {
+      tmp = esc_for_log(line->value);
+      v = tmp;
+    } else {
+      v = line->value;
+    }
+
+    smartlist_add_asprintf(elements, "%s%s%s", k, eq, v);
+    tor_free(tmp);
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/**
+ * Decode a <b>line</b> containing a series of space-separated 'Key=Value'
+ * pairs, using the provided <b>flags</b> to decode it.  Return a newly
+ * allocated list of pairs on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are
+ * allowed and handled as C strings. Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are
+ * allowed.  Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then keys without values are
+ * allowed.  Otherwise, such keys are not allowed.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * If KV_QUOTED_QSTRING is set in <b>flags</b>, then double-quoted values
+ * are allowed and handled as QuotedStrings per qstring.c.  Do not add
+ * new users of this flag.
+ */
+config_line_t *
+kvline_parse(const char *line, unsigned flags)
+{
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+
+  const char *cp = line, *cplast = NULL;
+  const bool omit_keys = (flags & KV_OMIT_KEYS) != 0;
+  const bool omit_vals = (flags & KV_OMIT_VALS) != 0;
+  const bool quoted = (flags & (KV_QUOTED|KV_QUOTED_QSTRING)) != 0;
+  const bool c_quoted = (flags & (KV_QUOTED)) != 0;
+
+  config_line_t *result = NULL;
+  config_line_t **next_line = &result;
+
+  char *key = NULL;
+  char *val = NULL;
+
+  while (*cp) {
+    key = val = NULL;
+    /* skip all spaces */
+    {
+      size_t idx = strspn(cp, " \t\r\v\n");
+      cp += idx;
+    }
+    if (BUG(cp == cplast)) {
+      /* If we didn't parse anything since the last loop, this code is
+       * broken. */
+      goto err; // LCOV_EXCL_LINE
+    }
+    cplast = cp;
+    if (! *cp)
+      break; /* End of string; we're done. */
+
+    /* Possible formats are K=V, K="V", K, V, and "V", depending on flags. */
+
+    /* Find where the key ends */
+    if (*cp != '\"') {
+      size_t idx = strcspn(cp, " \t\r\v\n=");
+
+      if (cp[idx] == '=') {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx + 1;
+      } else if (omit_vals) {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx;
+        goto commit;
+      } else {
+        if (!omit_keys)
+          goto err;
+      }
+    }
+
+    if (*cp == '\"') {
+      /* The type is "V". */
+      if (!quoted)
+        goto err;
+      size_t len=0;
+      if (c_quoted) {
+        cp = unescape_string(cp, &val, &len);
+      } else {
+        cp = decode_qstring(cp, strlen(cp), &val, &len);
+      }
+      if (cp == NULL || len != strlen(val)) {
+        // The string contains a NUL or is badly coded.
+        goto err;
+      }
+    } else {
+      size_t idx = strcspn(cp, " \t\r\v\n");
+      val = tor_memdup_nulterm(cp, idx);
+      cp += idx;
+    }
+
+  commit:
+    if (key && strlen(key) == 0) {
+      /* We don't allow empty keys. */
+      goto err;
+    }
+
+    *next_line = tor_malloc_zero(sizeof(config_line_t));
+    (*next_line)->key = key ? key : tor_strdup("");
+    (*next_line)->value = val ? val : tor_strdup("");
+    next_line = &(*next_line)->next;
+    key = val = NULL;
+  }
+
+  if (! (flags & KV_QUOTED_QSTRING)) {
+    if (!kvline_can_encode_lines(result, flags)) {
+      goto err;
+    }
+  }
+  return result;
+
+ err:
+  tor_free(key);
+  tor_free(val);
+  config_free_lines(result);
+  return NULL;
+}
diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h
new file mode 100644
index 0000000000..dea2ce1809
--- /dev/null
+++ b/src/lib/encoding/kvline.h
@@ -0,0 +1,26 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.h
+ *
+ * \brief Header for kvline.c
+ **/
+
+#ifndef TOR_KVLINE_H
+#define TOR_KVLINE_H
+
+struct config_line_t;
+
+#define KV_QUOTED    (1u<<0)
+#define KV_OMIT_KEYS (1u<<1)
+#define KV_OMIT_VALS (1u<<2)
+#define KV_QUOTED_QSTRING (1u<<3)
+
+struct config_line_t *kvline_parse(const char *line, unsigned flags);
+char *kvline_encode(const struct config_line_t *line, unsigned flags);
+
+#endif /* !defined(TOR_KVLINE_H) */
diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h
index 0bbb06a794..6b20350aa8 100644
--- a/src/lib/encoding/pem.h
+++ b/src/lib/encoding/pem.h
@@ -23,4 +23,4 @@ int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen,
 int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen,
                const char *objtype);
 
-#endif
+#endif /* !defined(TOR_PEM_H) */
diff --git a/src/lib/encoding/qstring.c b/src/lib/encoding/qstring.c
new file mode 100644
index 0000000000..a92d28c706
--- /dev/null
+++ b/src/lib/encoding/qstring.c
@@ -0,0 +1,90 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.c
+ * \brief Implement QuotedString parsing.
+ *
+ * Note that this is only used for controller authentication; do not
+ * create new users for this.  Instead, prefer the cstring.c functions.
+ **/
+
+#include "orconfig.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
+ * QuotedString, return the length of that
+ * string (as encoded, including quotes).  Otherwise return -1. */
+static inline int
+get_qstring_length(const char *start, size_t in_len_max,
+                          int *chars_out)
+{
+  const char *cp, *end;
+  int chars = 0;
+
+  if (*start != '\"')
+    return -1;
+
+  cp = start+1;
+  end = start+in_len_max;
+
+  /* Calculate length. */
+  while (1) {
+    if (cp >= end) {
+      return -1; /* Too long. */
+    } else if (*cp == '\\') {
+      if (++cp == end)
+        return -1; /* Can't escape EOS. */
+      ++cp;
+      ++chars;
+    } else if (*cp == '\"') {
+      break;
+    } else {
+      ++cp;
+      ++chars;
+    }
+  }
+  if (chars_out)
+    *chars_out = chars;
+  return (int)(cp - start+1);
+}
+
+/** Given a pointer to a string starting at <b>start</b> containing
+ * <b>in_len_max</b> characters, decode a string beginning with one double
+ * quote, containing any number of non-quote characters or characters escaped
+ * with a backslash, and ending with a final double quote.  Place the resulting
+ * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
+ * store its length in <b>out_len</b>.  On success, return a pointer to the
+ * character immediately following the escaped string.  On failure, return
+ * NULL. */
+const char *
+decode_qstring(const char *start, size_t in_len_max,
+               char **out, size_t *out_len)
+{
+  const char *cp, *end;
+  char *outp;
+  int len, n_chars = 0;
+
+  len = get_qstring_length(start, in_len_max, &n_chars);
+  if (len<0)
+    return NULL;
+
+  end = start+len-1; /* Index of last quote. */
+  tor_assert(*end == '\"');
+  outp = *out = tor_malloc(len+1);
+  *out_len = n_chars;
+
+  cp = start+1;
+  while (cp < end) {
+    if (*cp == '\\')
+      ++cp;
+    *outp++ = *cp++;
+  }
+  *outp = '\0';
+  tor_assert((outp - *out) == (int)*out_len);
+
+  return end+1;
+}
diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h
new file mode 100644
index 0000000000..840e1044ce
--- /dev/null
+++ b/src/lib/encoding/qstring.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.h
+ * \brief Header for qstring.c
+ */
+
+#ifndef TOR_ENCODING_QSTRING_H
+#define TOR_ENCODING_QSTRING_H
+
+#include <stddef.h>
+
+const char *decode_qstring(const char *start, size_t in_len_max,
+                           char **out, size_t *out_len);
+
+#endif /* !defined(TOR_ENCODING_QSTRING_H) */
diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h
index 0ddeca57fc..d14bc1f902 100644
--- a/src/lib/encoding/time_fmt.h
+++ b/src/lib/encoding/time_fmt.h
@@ -41,4 +41,4 @@ int parse_iso_time_nospace(const char *cp, time_t *t);
 int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
-#endif
+#endif /* !defined(TOR_TIME_FMT_H) */
diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include
index 48cc0ef088..314424545e 100644
--- a/src/lib/err/.may_include
+++ b/src/lib/err/.may_include
@@ -1,3 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/defs/*.h
 lib/err/*.h
+lib/subsys/*.h
+lib/version/*.h
diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c
index 8606f42177..ce8ddcd7c0 100644
--- a/src/lib/err/backtrace.c
+++ b/src/lib/err/backtrace.c
@@ -52,12 +52,14 @@
 #include <pthread.h>
 #endif
 
+#include "lib/cc/ctassert.h"
+
 #define EXPOSE_CLEAN_BACKTRACE
 #include "lib/err/backtrace.h"
-#include "lib/err/torerr.h"
 
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
-  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
+  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \
+  defined(HAVE_PTHREAD_H)
 #define USE_BACKTRACE
 #endif
 
@@ -72,15 +74,40 @@
 static char bt_version[128] = "";
 
 #ifdef USE_BACKTRACE
+
 /** Largest stack depth to try to dump. */
 #define MAX_DEPTH 256
-/** Static allocation of stack to dump. This is static so we avoid stack
- * pressure. */
-static void *cb_buf[MAX_DEPTH];
+/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */
+#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *))
 /** Protects cb_buf from concurrent access. Pthreads, since this code
  * is Unix-only, and since this code needs to be lowest-level. */
 static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+/** Lock and return a static stack pointer buffer that can hold up to
+ *  MAX_DEPTH function pointers. */
+static void **
+lock_cb_buf(void)
+{
+  /* Lock the mutex first, before even declaring the buffer. */
+  pthread_mutex_lock(&cb_buf_mutex);
+
+  /** Static allocation of stack to dump. This is static so we avoid stack
+   * pressure. */
+  static void *cb_buf[MAX_DEPTH];
+  CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf));
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+
+  return cb_buf;
+}
+
+/** Unlock the static stack pointer buffer. */
+static void
+unlock_cb_buf(void **cb_buf)
+{
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+  pthread_mutex_unlock(&cb_buf_mutex);
+}
+
 /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will
  * log the correct function from which a signal was received with context
  * <b>ctx</b>.  (When we get a signal, the current function will not have
@@ -104,7 +131,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
     return;
 
   stack[n] = (void*) ctx->PC_FROM_UCONTEXT;
-#else /* !(defined(PC_FROM_UCONTEXT)) */
+#else /* !defined(PC_FROM_UCONTEXT) */
   (void) depth;
   (void) ctx;
   (void) stack;
@@ -115,14 +142,14 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
  * that with a backtrace log.  Send messages via the tor_log function at
  * logger". */
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   size_t depth;
   char **symbols;
   size_t i;
 
-  pthread_mutex_lock(&cb_buf_mutex);
+  void **cb_buf = lock_cb_buf();
 
   depth = backtrace(cb_buf, MAX_DEPTH);
   symbols = backtrace_symbols(cb_buf, (int)depth);
@@ -140,7 +167,7 @@ log_backtrace_impl(int severity, int domain, const char *msg,
   raw_free(symbols);
 
  done:
-  pthread_mutex_unlock(&cb_buf_mutex);
+  unlock_cb_buf(cb_buf);
 }
 
 static void crash_handler(int sig, siginfo_t *si, void *ctx_)
@@ -156,6 +183,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   int n_fds, i;
   const int *fds = NULL;
 
+  void **cb_buf = lock_cb_buf();
+
   (void) si;
 
   depth = backtrace(cb_buf, MAX_DEPTH);
@@ -172,7 +201,9 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
 
-  abort();
+  unlock_cb_buf(cb_buf);
+
+  tor_raw_abort_();
 }
 
 /** Write a backtrace to all of the emergency-error fds. */
@@ -183,20 +214,26 @@ dump_stack_symbols_to_error_fds(void)
   const int *fds = NULL;
   size_t depth;
 
+  void **cb_buf = lock_cb_buf();
+
   depth = backtrace(cb_buf, MAX_DEPTH);
 
   n_fds = tor_log_get_sigsafe_err_fds(&fds);
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
+
+  unlock_cb_buf(cb_buf);
 }
 
+/* The signals that we want our backtrace handler to trap */
+static int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
+  SIGIO, -1 };
+
 /** Install signal handlers as needed so that when we crash, we produce a
  * useful stack trace. Return 0 on success, -errno on failure. */
 static int
 install_bt_handler(void)
 {
-  int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
-                         SIGIO, -1 };
   int i, rv=0;
 
   struct sigaction sa;
@@ -219,10 +256,12 @@ install_bt_handler(void)
      * libc has pre-loaded the symbols we need to dump things, so that later
      * reads won't be denied by the sandbox code */
     char **symbols;
+    void **cb_buf = lock_cb_buf();
     size_t depth = backtrace(cb_buf, MAX_DEPTH);
     symbols = backtrace_symbols(cb_buf, (int) depth);
     if (symbols)
       raw_free(symbols);
+    unlock_cb_buf(cb_buf);
   }
 
   return rv;
@@ -232,12 +271,29 @@ install_bt_handler(void)
 static void
 remove_bt_handler(void)
 {
+  int i;
+
+  struct sigaction sa;
+
+  memset(&sa, 0, sizeof(sa));
+  sa.sa_handler = SIG_DFL;
+  sigfillset(&sa.sa_mask);
+
+  for (i = 0; trap_signals[i] >= 0; ++i) {
+    /* remove_bt_handler() is called on shutdown, from low-level code.
+     * It's not a fatal error, so we just ignore it. */
+    (void)sigaction(trap_signals[i], &sa, NULL);
+  }
+
+  /* cb_buf_mutex is statically initialised, so we can not destroy it.
+   * If we destroy it, and then re-initialise tor, all our backtraces will
+   * fail. */
 }
 #endif /* defined(USE_BACKTRACE) */
 
 #ifdef NO_BACKTRACE_IMPL
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   logger(severity, domain, "%s: %s. (Stack trace not available)",
diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h
index 48b41fca02..7e09a0a5a7 100644
--- a/src/lib/err/backtrace.h
+++ b/src/lib/err/backtrace.h
@@ -12,11 +12,14 @@
 
 #include "orconfig.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/defs/logging_types.h"
 
-typedef void (*tor_log_fn)(int, unsigned, const char *fmt, ...)
+typedef void (*tor_log_fn)(int, log_domain_mask_t, const char *fmt, ...)
   CHECK_PRINTF(3,4);
 
-void log_backtrace_impl(int severity, int domain, const char *msg,
+void log_backtrace_impl(int severity, log_domain_mask_t domain,
+                        const char *msg,
                         tor_log_fn logger);
 int configure_backtrace_handler(const char *tor_version);
 void clean_up_backtrace_handler(void);
diff --git a/src/lib/err/include.am b/src/lib/err/include.am
index f2a409c51e..883ac91511 100644
--- a/src/lib/err/include.am
+++ b/src/lib/err/include.am
@@ -5,15 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-err-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_err_a_SOURCES =			\
-	src/lib/err/backtrace.c				\
-	src/lib/err/torerr.c
+	src/lib/err/backtrace.c			\
+	src/lib/err/torerr.c			\
+	src/lib/err/torerr_sys.c
 
 src_lib_libtor_err_testing_a_SOURCES = \
 	$(src_lib_libtor_err_a_SOURCES)
 src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/err/backtrace.h				\
-	src/lib/err/torerr.h
+	src/lib/err/torerr.h				\
+	src/lib/err/torerr_sys.h
diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c
index 6b5224273a..a5c00ca389 100644
--- a/src/lib/err/torerr.c
+++ b/src/lib/err/torerr.c
@@ -110,6 +110,14 @@ tor_log_get_sigsafe_err_fds(const int **out)
  * Update the list of fds that get errors from inside a signal handler or
  * other emergency condition. Ignore any beyond the first
  * TOR_SIGSAFE_LOG_MAX_FDS.
+ *
+ * These fds must remain open even after the log module has shut down. (And
+ * they should remain open even while logs are being reconfigured.) Therefore,
+ * any fds closed by the log module should be dup()ed, and the duplicate fd
+ * should be given to the err module in fds. In particular, the log module
+ * closes the file log fds, but does not close the stdio log fds.
+ *
+ * If fds is NULL or n is 0, clears the list of error fds.
  */
 void
 tor_log_set_sigsafe_err_fds(const int *fds, int n)
@@ -118,8 +126,28 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n)
     n = TOR_SIGSAFE_LOG_MAX_FDS;
   }
 
-  memcpy(sigsafe_log_fds, fds, n * sizeof(int));
-  n_sigsafe_log_fds = n;
+  /* Clear the entire array. This code mitigates against some race conditions,
+   * but there are still some races here:
+   * - err logs are disabled while the array is cleared, and
+   * - a thread can read the old value of n_sigsafe_log_fds, then read a
+   *   partially written array.
+   * We could fix these races using atomics, but atomics use the err module. */
+  n_sigsafe_log_fds = 0;
+  memset(sigsafe_log_fds, 0, sizeof(sigsafe_log_fds));
+  if (fds && n > 0) {
+    memcpy(sigsafe_log_fds, fds, n * sizeof(int));
+    n_sigsafe_log_fds = n;
+  }
+}
+
+/**
+ * Reset the list of emergency error fds to its default.
+ */
+void
+tor_log_reset_sigsafe_err_fds(void)
+{
+  int fds[] = { STDERR_FILENO };
+  tor_log_set_sigsafe_err_fds(fds, 1);
 }
 
 /**
@@ -161,6 +189,17 @@ tor_raw_assertion_failed_msg_(const char *file, int line, const char *expr,
   tor_log_err_sigsafe_write("\n");
 }
 
+/**
+ * Call the abort() function to kill the current process with a fatal
+ * error. This is a separate function, so that log users don't have to include
+ * the header for abort().
+ **/
+void
+tor_raw_abort_(void)
+{
+  abort();
+}
+
 /* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument
  * in range 2..16 inclusive. */
 static int
@@ -195,7 +234,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
     unsigned digit = (unsigned) (x % radix);
     if (cp <= buf) {
       /* Not tor_assert(); see above. */
-      abort();
+      tor_raw_abort_();
     }
     --cp;
     *cp = "0123456789ABCDEF"[digit];
@@ -204,7 +243,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
 
   /* NOT tor_assert; see above. */
   if (cp != buf) {
-    abort(); // LCOV_EXCL_LINE
+    tor_raw_abort_(); // LCOV_EXCL_LINE
   }
 
   return len;
@@ -224,8 +263,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
  * does not guarantee that an int is wider than a char (an int must be at
  * least 16 bits but it is permitted for a char to be that wide as well), we
  * can't assume a signed int is sufficient to accommodate an unsigned char.
- * Thus, format_helper_exit_status() will still need to emit any require '-'
- * on its own.
+ * Thus, callers will still need to add any required '-' to the final string.
  *
  * For most purposes, you'd want to use tor_snprintf("%x") instead of this
  * function; it's designed to be used in code paths where you can't call
diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h
index 6ae91fbe85..a1822d9c9a 100644
--- a/src/lib/err/torerr.h
+++ b/src/lib/err/torerr.h
@@ -20,13 +20,13 @@
 #define raw_assert(expr) STMT_BEGIN                                     \
     if (!(expr)) {                                                      \
       tor_raw_assertion_failed_msg_(__FILE__, __LINE__, #expr, NULL);   \
-      abort();                                                          \
+      tor_raw_abort_();                                                 \
     }                                                                   \
   STMT_END
 #define raw_assert_unreached(expr) raw_assert(0)
 #define raw_assert_unreached_msg(msg) STMT_BEGIN                    \
     tor_raw_assertion_failed_msg_(__FILE__, __LINE__, "0", (msg));  \
-    abort();                                                        \
+    tor_raw_abort_();                                               \
   STMT_END
 
 void tor_raw_assertion_failed_msg_(const char *file, int line,
@@ -39,9 +39,12 @@ void tor_raw_assertion_failed_msg_(const char *file, int line,
 void tor_log_err_sigsafe(const char *m, ...);
 int tor_log_get_sigsafe_err_fds(const int **out);
 void tor_log_set_sigsafe_err_fds(const int *fds, int n);
+void tor_log_reset_sigsafe_err_fds(void);
 void tor_log_sigsafe_err_set_granularity(int ms);
 
+void tor_raw_abort_(void) ATTR_NORETURN;
+
 int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len);
 int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_TORERR_H) */
diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c
new file mode 100644
index 0000000000..c42e958093
--- /dev/null
+++ b/src/lib/err/torerr_sys.c
@@ -0,0 +1,43 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.c
+ * \brief Subsystem object for the error handling subsystem.
+ **/
+
+#include "orconfig.h"
+#include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/subsys/subsys.h"
+#include "lib/version/torversion.h"
+
+#include <stddef.h>
+
+static int
+subsys_torerr_initialize(void)
+{
+  if (configure_backtrace_handler(get_version()) < 0)
+    return -1;
+  tor_log_reset_sigsafe_err_fds();
+
+  return 0;
+}
+static void
+subsys_torerr_shutdown(void)
+{
+  /* Stop handling signals with backtraces. */
+  clean_up_backtrace_handler();
+}
+
+const subsys_fns_t sys_torerr = {
+  .name = "err",
+  /* Low-level error handling is a diagnostic feature, we want it to init
+   * right after windows process security, and shutdown last.
+   * (Security never shuts down.) */
+  .level = -99,
+  .supported = true,
+  .initialize = subsys_torerr_initialize,
+  .shutdown = subsys_torerr_shutdown
+};
diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h
new file mode 100644
index 0000000000..c947695689
--- /dev/null
+++ b/src/lib/err/torerr_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.h
+ * \brief Declare subsystem object for torerr.c
+ **/
+
+#ifndef TOR_TORERR_SYS_H
+#define TOR_TORERR_SYS_H
+
+extern const struct subsys_fns_t sys_torerr;
+
+#endif /* !defined(TOR_TORERR_SYS_H) */
diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include
index 30af508914..54aa75fbff 100644
--- a/src/lib/evloop/.may_include
+++ b/src/lib/evloop/.may_include
@@ -8,9 +8,10 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/time/*.h
 
-src/ext/timeouts/timeout.c
-tor_queue.h
\ No newline at end of file
+ext/timeouts/timeout.c
+ext/tor_queue.h
\ No newline at end of file
diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c
new file mode 100644
index 0000000000..56641a3175
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.c
@@ -0,0 +1,49 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.c
+ * @brief Subsystem definition for the event loop module
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/evloop_sys.h"
+#include "lib/log/log.h"
+
+static int
+subsys_evloop_initialize(void)
+{
+  if (tor_init_libevent_rng() < 0) {
+    log_warn(LD_NET, "Problem initializing libevent RNG.");
+    return -1;
+  }
+  return 0;
+}
+
+static void
+subsys_evloop_postfork(void)
+{
+#ifdef TOR_UNIT_TESTS
+  tor_libevent_postfork();
+#endif
+}
+
+static void
+subsys_evloop_shutdown(void)
+{
+  tor_libevent_free_all();
+}
+
+const struct subsys_fns_t sys_evloop = {
+  .name = "evloop",
+  .supported = true,
+  .level = -20,
+  .initialize = subsys_evloop_initialize,
+  .shutdown = subsys_evloop_shutdown,
+  .postfork = subsys_evloop_postfork,
+};
diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h
new file mode 100644
index 0000000000..e6155c25b0
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.h
+ * @brief Declare subsystem object for the event loop module.
+ **/
+
+#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H
+#define TOR_LIB_EVLOOP_EVLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_evloop;
+
+#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */
diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am
index 6b0076272a..41cd2f45c5 100644
--- a/src/lib/evloop/include.am
+++ b/src/lib/evloop/include.am
@@ -5,21 +5,24 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-evloop-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_evloop_a_SOURCES =			\
 	src/lib/evloop/compat_libevent.c		\
+	src/lib/evloop/evloop_sys.c			\
 	src/lib/evloop/procmon.c			\
 	src/lib/evloop/timers.c				\
 	src/lib/evloop/token_bucket.c			\
 	src/lib/evloop/workqueue.c
 
-
 src_lib_libtor_evloop_testing_a_SOURCES = \
 	$(src_lib_libtor_evloop_a_SOURCES)
 src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/evloop/compat_libevent.h		\
+	src/lib/evloop/evloop_sys.h			\
 	src/lib/evloop/procmon.h			\
 	src/lib/evloop/timers.h				\
 	src/lib/evloop/token_bucket.h			\
diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c
index 52469fa5fc..b2d81fc14b 100644
--- a/src/lib/evloop/procmon.c
+++ b/src/lib/evloop/procmon.c
@@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_)
       tor_free(errmsg);
     }
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   /* Unix makes this part easy, if a bit racy. */
   its_dead_jim = kill(procmon->pid, 0);
   its_dead_jim = its_dead_jim && (errno == ESRCH);
diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c
index e46d2635a8..4b2a96ef7d 100644
--- a/src/lib/evloop/timers.c
+++ b/src/lib/evloop/timers.c
@@ -80,7 +80,8 @@ struct timeout_cb {
  * use 32-bit math. */
 #define WHEEL_BIT 5
 #endif
-#include "src/ext/timeouts/timeout.c"
+
+#include "ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
 static struct mainloop_event_t *global_timer_event = NULL;
diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c
index ee6d631e3b..ec62d1b018 100644
--- a/src/lib/evloop/token_bucket.c
+++ b/src/lib/evloop/token_bucket.c
@@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket,
     flags |= TB_WRITE;
   return flags;
 }
+
+/** Initialize a token bucket in <b>bucket</b>, set up to allow <b>rate</b>
+ * per second, with a maximum burst of <b>burst</b>. The bucket is created
+ * such that <b>now_ts</b> is the current timestamp. The bucket starts out
+ * full. */
+void
+token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                      uint32_t burst, uint32_t now_ts)
+{
+  memset(bucket, 0, sizeof(token_bucket_ctr_t));
+  token_bucket_ctr_adjust(bucket, rate, burst);
+  token_bucket_ctr_reset(bucket, now_ts);
+}
+
+/** Change the configured rate and burst of the given token bucket object in
+ * <b>bucket</b>. */
+void
+token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                        uint32_t burst)
+{
+  token_bucket_cfg_init(&bucket->cfg, rate, burst);
+  token_bucket_raw_adjust(&bucket->counter, &bucket->cfg);
+}
+
+/** Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. */
+void
+token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  token_bucket_raw_reset(&bucket->counter, &bucket->cfg);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
+
+/** Refill <b>bucket</b> as appropriate, given that the current timestamp is
+ * <b>now_ts</b>. */
+void
+token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  const uint32_t elapsed_ticks =
+    (now_ts - bucket->last_refilled_at_timestamp);
+  if (elapsed_ticks > UINT32_MAX-(300*1000)) {
+    /* Either about 48 days have passed since the last refill, or the
+     * monotonic clock has somehow moved backwards. (We're looking at you,
+     * Windows.).  We accept up to a 5 minute jump backwards as
+     * "unremarkable".
+     */
+    return;
+  }
+
+  token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg,
+                                elapsed_ticks);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h
index 9398d2baa3..dde9bd65a4 100644
--- a/src/lib/evloop/token_bucket.h
+++ b/src/lib/evloop/token_bucket.h
@@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
   return token_bucket_raw_get(&bucket->write_bucket);
 }
 
+/**
+ * A specialized bucket containing a single counter.
+ */
+
+typedef struct token_bucket_ctr_t {
+  token_bucket_cfg_t cfg;
+  token_bucket_raw_t counter;
+  uint32_t last_refilled_at_timestamp;
+} token_bucket_ctr_t;
+
+void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                           uint32_t burst, uint32_t now_ts);
+void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                             uint32_t burst);
+void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts);
+void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts);
+
+static inline bool
+token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->counter, n);
+}
+
+static inline size_t
+token_bucket_ctr_get(const token_bucket_ctr_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->counter);
+}
+
 #ifdef TOKEN_BUCKET_PRIVATE
 
 /* To avoid making the rates too small, we consider units of "steps",
@@ -112,6 +141,6 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
 
 STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate);
 
-#endif
+#endif /* defined(TOKEN_BUCKET_PRIVATE) */
 
-#endif /* TOR_TOKEN_BUCKET_H */
+#endif /* !defined(TOR_TOKEN_BUCKET_H) */
diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c
index 931f65e710..015b694290 100644
--- a/src/lib/evloop/workqueue.c
+++ b/src/lib/evloop/workqueue.c
@@ -15,7 +15,7 @@
  *
  * The main thread informs the worker threads of pending work by using a
  * condition variable.  The workers inform the main process of completed work
- * by using an alert_sockets_t object, as implemented in compat_threads.c.
+ * by using an alert_sockets_t object, as implemented in net/alertsock.c.
  *
  * The main thread can also queue an "update" that will be handled by all the
  * workers.  This is useful for updating state that all the workers share.
@@ -36,7 +36,7 @@
 #include "lib/net/socket.h"
 #include "lib/thread/threads.h"
 
-#include "tor_queue.h"
+#include "ext/tor_queue.h"
 #include <event2/event.h>
 #include <string.h>
 
@@ -59,9 +59,6 @@ struct threadpool_s {
    * <b>p</b> is work[p]. */
   work_tailq_t work[WORKQUEUE_N_PRIORITIES];
 
-  /** Weak RNG, used to decide when to ignore priority. */
-  tor_weak_rng_t weak_rng;
-
   /** The current 'update generation' of the threadpool.  Any thread that is
    * at an earlier generation needs to run the update function. */
   unsigned generation;
@@ -238,7 +235,7 @@ worker_thread_extract_next_work(workerthread_t *thread)
     this_queue = &pool->work[i];
     if (!TOR_TAILQ_EMPTY(this_queue)) {
       queue = this_queue;
-      if (! tor_weak_random_one_in_n(&pool->weak_rng,
+      if (! crypto_fast_rng_one_in_n(get_thread_fast_rng(),
                                      thread->lower_priority_chance)) {
         /* Usually we'll just break now, so that we can get out of the loop
          * and use the queue where we found work. But with a small
@@ -555,11 +552,6 @@ threadpool_new(int n_threads,
   for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
     TOR_TAILQ_INIT(&pool->work[i]);
   }
-  {
-    unsigned seed;
-    crypto_rand((void*)&seed, sizeof(seed));
-    tor_init_weak_random(&pool->weak_rng, seed);
-  }
 
   pool->new_thread_state_fn = new_thread_state_fn;
   pool->new_thread_state_arg = arg;
@@ -622,8 +614,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg)
     tp->reply_cb(tp);
 }
 
-/** Register the threadpool <b>tp</b>'s reply queue with the libevent
- * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after
+/** Register the threadpool <b>tp</b>'s reply queue with Tor's global
+ * libevent mainloop. If <b>cb</b> is provided, it is run after
  * each time there is work to process from the reply queue. Return 0 on
  * success, -1 on failure.
  */
diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h
index 333a3f6dde..d0ee8f2be2 100644
--- a/src/lib/evloop/workqueue.h
+++ b/src/lib/evloop/workqueue.h
@@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp);
 replyqueue_t *replyqueue_new(uint32_t alertsocks_flags);
 void replyqueue_process(replyqueue_t *queue);
 
-struct event_base;
 int threadpool_register_reply_event(threadpool_t *tp,
                                     void (*cb)(threadpool_t *tp));
 
diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c
index d723d04d2a..2d0864cc19 100644
--- a/src/lib/fdio/fdio.c
+++ b/src/lib/fdio/fdio.c
@@ -17,12 +17,16 @@
 #ifdef _WIN32
 #include <windows.h>
 #endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 
 #include "lib/fdio/fdio.h"
 #include "lib/cc/torint.h"
 #include "lib/err/torerr.h"
 
 #include <stdlib.h>
+#include <stdio.h>
 
 /** @{ */
 /** Some old versions of Unix didn't define constants for these values,
diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am
index 6c18f00a0d..545bbc929e 100644
--- a/src/lib/fdio/include.am
+++ b/src/lib/fdio/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fdio-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fdio_a_SOURCES =			\
 	src/lib/fdio/fdio.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_fdio_testing_a_SOURCES = \
 src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fdio/fdio.h
diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include
index b1e49fc891..c192e6181c 100644
--- a/src/lib/fs/.may_include
+++ b/src/lib/fs/.may_include
@@ -13,4 +13,4 @@ lib/malloc/*.h
 lib/memarea/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h
index 7af9119dbb..29115e1085 100644
--- a/src/lib/fs/conffile.h
+++ b/src/lib/fs/conffile.h
@@ -20,4 +20,4 @@ int config_get_lines_include(const char *string, struct config_line_t **result,
                              int extended, int *has_include,
                              struct smartlist_t *opened_lst);
 
-#endif /* !defined(TOR_CONFLINE_H) */
+#endif /* !defined(TOR_CONFFILE_H) */
diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c
index 3c31e00d99..390836b048 100644
--- a/src/lib/fs/dir.c
+++ b/src/lib/fs/dir.c
@@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check,
     }
   }
   close(fd);
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   /* Win32 case: we can't open() a directory. */
   (void)effective_user;
 
@@ -347,7 +347,7 @@ tor_listdir, (const char *dirname))
   }
   FindClose(handle);
   tor_free(pattern);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *prot_dname = sandbox_intern_string(dirname);
   DIR *d;
   struct dirent *de;
diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h
index 826bc2dfc5..9ff81faa42 100644
--- a/src/lib/fs/dir.h
+++ b/src/lib/fs/dir.h
@@ -30,4 +30,4 @@ MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check,
 
 MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
 
-#endif
+#endif /* !defined(TOR_DIR_H) */
diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h
index 52c94c914f..ed983f3b3c 100644
--- a/src/lib/fs/files.h
+++ b/src/lib/fs/files.h
@@ -27,7 +27,7 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -108,7 +108,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
  * Tor is built for unit tests, or when Tor is built on an operating system
  * without its own getdelim(). */
 ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
-#endif
+#endif /* !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS) */
 
 #ifdef HAVE_GETDELIM
 /**
@@ -123,10 +123,10 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getdelim(lineptr, n, delim, stream) \
   getdelim((lineptr), (n), (delim), (stream))
-#else
+#else /* !defined(HAVE_GETDELIM) */
 #define tor_getdelim(lineptr, n, delim, stream) \
   compat_getdelim_((lineptr), (n), (delim), (stream))
-#endif
+#endif /* defined(HAVE_GETDELIM) */
 
 #ifdef HAVE_GETLINE
 /**
@@ -137,9 +137,9 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getline(lineptr, n, stream) \
   getline((lineptr), (n), (stream))
-#else
+#else /* !defined(HAVE_GETLINE) */
 #define tor_getline(lineptr, n, stream) \
   tor_getdelim((lineptr), (n), '\n', (stream))
-#endif
+#endif /* defined(HAVE_GETLINE) */
 
-#endif
+#endif /* !defined(TOR_FS_H) */
diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am
index f33e4d6430..493db8f044 100644
--- a/src/lib/fs/include.am
+++ b/src/lib/fs/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fs-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fs_a_SOURCES =			\
 	src/lib/fs/conffile.c			\
 	src/lib/fs/dir.c			\
@@ -25,6 +26,7 @@ src_lib_libtor_fs_testing_a_SOURCES = \
 src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fs/conffile.h				\
 	src/lib/fs/dir.h				\
diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h
index 8aeee4cc7f..fc0281e253 100644
--- a/src/lib/fs/lockfile.h
+++ b/src/lib/fs/lockfile.h
@@ -17,4 +17,4 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
                                   int *locked_out);
 void tor_lockfile_unlock(tor_lockfile_t *lockfile);
 
-#endif
+#endif /* !defined(TOR_LOCKFILE_H) */
diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c
index daaee1f9b1..f71c0cff7a 100644
--- a/src/lib/fs/mmap.c
+++ b/src/lib/fs/mmap.c
@@ -237,4 +237,4 @@ tor_munmap_file(tor_mmap_t *handle)
 }
 #else
 #error "cannot implement tor_mmap_file"
-#endif /* defined(HAVE_MMAP) || ... || ... */
+#endif /* defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) || ... */
diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h
index 18fb18a13c..61aad544b2 100644
--- a/src/lib/fs/mmap.h
+++ b/src/lib/fs/mmap.h
@@ -38,4 +38,4 @@ typedef struct tor_mmap_t {
 tor_mmap_t *tor_mmap_file(const char *filename);
 int tor_munmap_file(tor_mmap_t *handle);
 
-#endif
+#endif /* !defined(TOR_MMAP_H) */
diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c
index b3ef61979d..28dde62aea 100644
--- a/src/lib/fs/path.c
+++ b/src/lib/fs/path.c
@@ -72,7 +72,7 @@ expand_filename(const char *filename)
    *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
    */
   return tor_strdup(filename);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   if (*filename == '~') {
     char *home, *result=NULL;
     const char *rest;
@@ -102,7 +102,7 @@ expand_filename(const char *filename)
       }
       tor_free(username);
       rest = slash ? (slash+1) : "";
-#else /* !(defined(HAVE_PWD_H)) */
+#else /* !defined(HAVE_PWD_H) */
       log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
       return tor_strdup(filename);
 #endif /* defined(HAVE_PWD_H) */
@@ -153,7 +153,7 @@ clean_fname_for_stat(char *name)
       return;
     name[len-1]='\0';
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)name;
 #endif /* defined(_WIN32) */
 }
@@ -233,7 +233,7 @@ alloc_getcwd(void)
     raw_free(cwd); // alias for free to avoid tripping check-spaces.
   }
   return result;
-#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */
   size_t size = 1024;
   char *buf = NULL;
   char *ptr = NULL;
@@ -268,7 +268,7 @@ make_path_absolute(char *fname)
   if (absfname_malloced) raw_free(absfname_malloced);
 
   return absfname;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   char *absfname = NULL, *path = NULL;
 
   tor_assert(fname);
diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h
index 4675ac84e8..28a1838b88 100644
--- a/src/lib/fs/path.h
+++ b/src/lib/fs/path.h
@@ -27,4 +27,4 @@ void clean_fname_for_stat(char *name);
 int get_parent_directory(char *fname);
 char *make_path_absolute(char *fname);
 
-#endif
+#endif /* !defined(TOR_PATH_H) */
diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h
index 5c39794873..5e5ddb89a3 100644
--- a/src/lib/fs/userdb.h
+++ b/src/lib/fs/userdb.h
@@ -21,6 +21,6 @@ struct passwd;
 const struct passwd *tor_getpwnam(const char *username);
 const struct passwd *tor_getpwuid(uid_t uid);
 char *get_user_homedir(const char *username);
-#endif
+#endif /* !defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_USERDB_H) */
diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h
index 64a22439e5..7237226c76 100644
--- a/src/lib/fs/winlib.h
+++ b/src/lib/fs/winlib.h
@@ -17,6 +17,6 @@
 #include <tchar.h>
 
 HANDLE load_windows_system_library(const TCHAR *library_name);
-#endif
+#endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_WINLIB_H) */
diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h
index 9a8911d494..a24a1c4c0d 100644
--- a/src/lib/geoip/country.h
+++ b/src/lib/geoip/country.h
@@ -13,4 +13,4 @@ typedef int16_t country_t;
 
 #define COUNTRY_MAX INT16_MAX
 
-#endif
+#endif /* !defined(TOR_COUNTRY_H) */
diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am
index 9710d75ac7..ea426d14bc 100644
--- a/src/lib/geoip/include.am
+++ b/src/lib/geoip/include.am
@@ -4,6 +4,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-geoip-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_geoip_a_SOURCES =			\
 	src/lib/geoip/geoip.c
 
@@ -12,6 +13,7 @@ src_lib_libtor_geoip_testing_a_SOURCES = \
 src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/geoip/geoip.h   			\
 	src/lib/geoip/country.h
diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h
index 83efa82919..3f745d457d 100644
--- a/src/lib/intmath/addsub.h
+++ b/src/lib/intmath/addsub.h
@@ -16,4 +16,4 @@
 
 uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
 
-#endif /* !defined(TOR_INTMATH_MULDIV_H) */
+#endif /* !defined(TOR_INTMATH_ADDSUB_H) */
diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h
index d0b0e8b954..67a738861b 100644
--- a/src/lib/intmath/cmp.h
+++ b/src/lib/intmath/cmp.h
@@ -36,4 +36,7 @@
     ((v) > (max)) ? (max) :                     \
     (v) )
 
+/** Give the absolute value of <b>x</b>, independent of its type. */
+#define ABS(x) ( ((x)<0) ? -(x) : (x) )
+
 #endif /* !defined(TOR_INTMATH_CMP_H) */
diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am
index 45ee3bd53b..155ffa145a 100644
--- a/src/lib/intmath/include.am
+++ b/src/lib/intmath/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-intmath-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_intmath_a_SOURCES =			\
 	src/lib/intmath/addsub.c			\
 	src/lib/intmath/bits.c				\
@@ -16,6 +17,7 @@ src_lib_libtor_intmath_testing_a_SOURCES = \
 src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/intmath/addsub.h			\
 	src/lib/intmath/cmp.h				\
diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h
index a4cecd69cc..b2f77462e1 100644
--- a/src/lib/intmath/logic.h
+++ b/src/lib/intmath/logic.h
@@ -17,4 +17,4 @@
 /** Macro: true if two values have different boolean values. */
 #define bool_neq(a,b) (!(a)!=!(b))
 
-#endif
+#endif /* !defined(HAVE_TOR_LOGIC_H) */
diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h
index e26bf58cbb..40941e59b2 100644
--- a/src/lib/intmath/weakrng.h
+++ b/src/lib/intmath/weakrng.h
@@ -28,4 +28,4 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
  * <b>n</b> */
 #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n)))
 
-#endif
+#endif /* !defined(TOR_WEAKRNG_H) */
diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c
index 4ad5929715..670bd0174c 100644
--- a/src/lib/lock/compat_mutex.c
+++ b/src/lib/lock/compat_mutex.c
@@ -29,7 +29,15 @@ tor_mutex_new_nonrecursive(void)
   tor_mutex_init_nonrecursive(m);
   return m;
 }
-/** Release all storage and system resources held by <b>m</b>. */
+/** Release all storage and system resources held by <b>m</b>.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_free_(tor_mutex_t *m)
 {
diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h
index b63ce24024..e0c3d7cb78 100644
--- a/src/lib/lock/compat_mutex.h
+++ b/src/lib/lock/compat_mutex.h
@@ -48,7 +48,7 @@ typedef struct tor_mutex_t {
 #else
   /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
   int _unused;
-#endif /* defined(USE_WIN32_MUTEX) || ... */
+#endif /* defined(USE_WIN32_THREADS) || ... */
 } tor_mutex_t;
 
 tor_mutex_t *tor_mutex_new(void);
diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c
index ee5f520cd0..f82ad9f0e8 100644
--- a/src/lib/lock/compat_mutex_pthreads.c
+++ b/src/lib/lock/compat_mutex_pthreads.c
@@ -88,12 +88,26 @@ tor_mutex_release(tor_mutex_t *m)
 }
 /** Clean up the mutex <b>m</b> so that it no longer uses any system
  * resources.  Does not free <b>m</b>.  This function must only be called on
- * mutexes from tor_mutex_init(). */
+ * mutexes from tor_mutex_init().
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_uninit(tor_mutex_t *m)
 {
   int err;
   raw_assert(m);
+  /* If the mutex is already locked, wait until after it is unlocked to destroy
+   * it. Locking and releasing the mutex makes undefined behaviour less likely,
+   * but does not prevent it. Another thread can lock the mutex between release
+   * and destroy. */
+  tor_mutex_acquire(m);
+  tor_mutex_release(m);
   err = pthread_mutex_destroy(&m->mutex);
   if (PREDICT_UNLIKELY(err)) {
     // LCOV_EXCL_START
diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am
index 4e6f444347..1475b9911b 100644
--- a/src/lib/lock/include.am
+++ b/src/lib/lock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-lock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_lock_a_SOURCES =			\
 	src/lib/lock/compat_mutex.c
 
@@ -20,5 +21,6 @@ src_lib_libtor_lock_testing_a_SOURCES = \
 src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/lock/compat_mutex.h
diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include
index 852173aab3..54d96324db 100644
--- a/src/lib/log/.may_include
+++ b/src/lib/log/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/cc/*.h
+lib/defs/*.h
 lib/smartlist_core/*.h
 lib/err/*.h
 lib/fdio/*.h
@@ -9,7 +10,7 @@ lib/lock/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
+lib/version/*.h
 lib/wallclock/*.h
-
-micro-revision.i
\ No newline at end of file
diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h
index 2f726186c5..0b9fc3406b 100644
--- a/src/lib/log/escape.h
+++ b/src/lib/log/escape.h
@@ -20,4 +20,4 @@ char *esc_for_log(const char *string) ATTR_MALLOC;
 char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC;
 const char *escaped(const char *string);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_ESCAPE_H) */
diff --git a/src/lib/log/include.am b/src/lib/log/include.am
index 4a6c9b3686..5b9f7113ba 100644
--- a/src/lib/log/include.am
+++ b/src/lib/log/include.am
@@ -5,11 +5,12 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-log-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_log_a_SOURCES =			\
 	src/lib/log/escape.c			\
-        src/lib/log/git_revision.c              \
 	src/lib/log/ratelim.c			\
 	src/lib/log/log.c			\
+	src/lib/log/log_sys.c			\
 	src/lib/log/util_bug.c
 
 if WIN32
@@ -21,16 +22,11 @@ src_lib_libtor_log_testing_a_SOURCES = \
 src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
-# Declare that these object files depend on micro-revision.i.  Without this
-# rule, we could try to build them before micro-revision.i was created.
-src/lib/log/git_revision.$(OBJEXT) \
-  src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \
-	micro-revision.i
-
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/log/escape.h				\
-        src/lib/log/git_revision.h                      \
 	src/lib/log/ratelim.h				\
 	src/lib/log/log.h				\
+	src/lib/log/log_sys.h				\
 	src/lib/log/util_bug.h				\
 	src/lib/log/win32err.h
diff --git a/src/lib/log/log.c b/src/lib/log/log.c
index a9ad38fb25..c53db46a8c 100644
--- a/src/lib/log/log.c
+++ b/src/lib/log/log.c
@@ -32,7 +32,8 @@
 
 #define LOG_PRIVATE
 #include "lib/log/log.h"
-#include "lib/log/git_revision.h"
+#include "lib/log/log_sys.h"
+#include "lib/version/git_revision.h"
 #include "lib/log/ratelim.h"
 #include "lib/lock/compat_mutex.h"
 #include "lib/smartlist_core/smartlist_core.h"
@@ -48,15 +49,12 @@
 #include "lib/wallclock/approx_time.h"
 #include "lib/wallclock/time_to_tm.h"
 #include "lib/fdio/fdio.h"
+#include "lib/cc/ctassert.h"
 
 #ifdef HAVE_ANDROID_LOG_H
 #include <android/log.h>
 #endif // HAVE_ANDROID_LOG_H.
 
-/** Given a severity, yields an index into log_severity_list_t.masks to use
- * for that severity. */
-#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
-
 /** @{ */
 /** The string we stick at the end of a log message when it is too long,
  * and its length. */
@@ -153,7 +151,7 @@ severity_to_android_log_priority(int severity)
       // LCOV_EXCL_STOP
   }
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** A mutex to guard changes to logfiles and logging. */
 static tor_mutex_t log_mutex;
@@ -223,6 +221,7 @@ int log_global_min_severity_ = LOG_NOTICE;
 
 static void delete_log(logfile_t *victim);
 static void close_log(logfile_t *victim);
+static void close_log_sigsafe(logfile_t *victim);
 
 static char *domain_to_string(log_domain_mask_t domain,
                              char *buf, size_t buflen);
@@ -533,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
     if (m != msg_after_prefix) {
       tor_free(m);
     }
-#else /* !(defined(MAXLINE)) */
+#else /* !defined(MAXLINE) */
     /* We have syslog but not MAXLINE.  That's promising! */
     syslog(severity, "%s", msg_after_prefix);
 #endif /* defined(MAXLINE) */
@@ -585,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname,
   /* check that severity is sane.  Overrunning the masks array leads to
    * interesting and hard to diagnose effects */
   raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG);
-  /* check that we've initialised the log mutex before we try to lock it */
-  raw_assert(log_mutex_initialized);
+
   LOCK_LOGS();
 
   if ((! (domain & LD_NOCB)) && pending_cb_messages
@@ -630,6 +628,10 @@ void
 tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
 {
   va_list ap;
+
+  /* check that domain is composed of known domains and flags */
+  raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain);
+
   if (severity > log_global_min_severity_)
     return;
   va_start(ap,format);
@@ -663,18 +665,26 @@ tor_log_update_sigsafe_err_fds(void)
   const logfile_t *lf;
   int found_real_stderr = 0;
 
-  int fds[TOR_SIGSAFE_LOG_MAX_FDS];
+  /* log_fds and err_fds contain matching entries: log_fds are the fds used by
+   * the log module, and err_fds are the fds used by the err module.
+   * For stdio logs, the log_fd and err_fd values are identical.
+   * For file logs, the err_fd is a dup() of the log_fd.
+   */
+  int log_fds[TOR_SIGSAFE_LOG_MAX_FDS];
+  int err_fds[TOR_SIGSAFE_LOG_MAX_FDS];
   int n_fds;
 
   LOCK_LOGS();
   /* Reserve the first one for stderr. This is safe because when we daemonize,
-   * we dup2 /dev/null to stderr, */
-  fds[0] = STDERR_FILENO;
+   * we dup2 /dev/null to stderr.
+   * For stderr, log_fds and err_fds are the same. */
+  log_fds[0] = err_fds[0] = STDERR_FILENO;
   n_fds = 1;
 
   for (lf = logfiles; lf; lf = lf->next) {
-     /* Don't try callback to the control port, or syslogs: We can't
-      * do them from a signal handler. Don't try stdout: we always do stderr.
+     /* Don't try callback to the control port, syslogs, android logs, or any
+      * other non-file descriptor log: We can't call arbitrary functions from a
+      * signal handler.
       */
     if (lf->is_temporary || logfile_is_external(lf)
         || lf->seems_dead || lf->fd < 0)
@@ -683,25 +693,40 @@ tor_log_update_sigsafe_err_fds(void)
         (LD_BUG|LD_GENERAL)) {
       if (lf->fd == STDERR_FILENO)
         found_real_stderr = 1;
-      /* Avoid duplicates */
-      if (int_array_contains(fds, n_fds, lf->fd))
+      /* Avoid duplicates by checking the log module fd against log_fds */
+      if (int_array_contains(log_fds, n_fds, lf->fd))
         continue;
-      fds[n_fds++] = lf->fd;
+      /* Update log_fds using the log module's fd */
+      log_fds[n_fds] = lf->fd;
+      if (lf->needs_close) {
+        /* File log fds are duplicated, because close_log() closes the log
+         * module's fd. Both refer to the same file. */
+        err_fds[n_fds] = dup(lf->fd);
+      } else {
+        /* stdio log fds are not closed by the log module. */
+        err_fds[n_fds] = lf->fd;
+      }
+      n_fds++;
       if (n_fds == TOR_SIGSAFE_LOG_MAX_FDS)
         break;
     }
   }
 
   if (!found_real_stderr &&
-      int_array_contains(fds, n_fds, STDOUT_FILENO)) {
-    /* Don't use a virtual stderr when we're also logging to stdout. */
+      int_array_contains(log_fds, n_fds, STDOUT_FILENO)) {
+    /* Don't use a virtual stderr when we're also logging to stdout.
+     * If we reached max_fds logs, we'll now have (max_fds - 1) logs.
+     * That's ok, max_fds is large enough that most tor instances don't exceed
+     * it. */
     raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */
-    fds[0] = fds[--n_fds];
+    --n_fds;
+    log_fds[0] = log_fds[n_fds];
+    err_fds[0] = err_fds[n_fds];
   }
 
   UNLOCK_LOGS();
 
-  tor_log_set_sigsafe_err_fds(fds, n_fds);
+  tor_log_set_sigsafe_err_fds(err_fds, n_fds);
 }
 
 /** Add to <b>out</b> a copy of every currently configured log file name. Used
@@ -727,7 +752,7 @@ tor_log_get_logfile_names(smartlist_t *out)
 
 /** Implementation of the log_fn backend, used when we have
  * variadic macros. All arguments are as for log_fn, except for
- * <b>fn</b>, which is the name of the calling functions. */
+ * <b>fn</b>, which is the name of the calling function. */
 void
 log_fn_(int severity, log_domain_mask_t domain, const char *fn,
         const char *format, ...)
@@ -804,16 +829,16 @@ logs_free_all(void)
   }
 
   /* We _could_ destroy the log mutex here, but that would screw up any logs
-   * that happened between here and the end of execution. */
+   * that happened between here and the end of execution.
+   * If tor is re-initialized, log_mutex_initialized will still be 1. So we
+   * won't trigger any undefined behaviour by trying to re-initialize the
+   * log mutex. */
 }
 
 /** Remove and free the log entry <b>victim</b> from the linked-list
  * logfiles (it is probably present, but it might not be due to thread
  * racing issues). After this function is called, the caller shouldn't
  * refer to <b>victim</b> anymore.
- *
- * Long-term, we need to do something about races in the log subsystem
- * in general. See bug 222 for more details.
  */
 static void
 delete_log(logfile_t *victim)
@@ -833,13 +858,26 @@ delete_log(logfile_t *victim)
 }
 
 /** Helper: release system resources (but not memory) held by a single
- * logfile_t. */
+ * signal-safe logfile_t. If the log's resources can not be released in
+ * a signal handler, does nothing. */
 static void
-close_log(logfile_t *victim)
+close_log_sigsafe(logfile_t *victim)
 {
   if (victim->needs_close && victim->fd >= 0) {
+    /* We can't do anything useful here if close() fails: we're shutting
+     * down logging, and the err module only does fatal errors. */
     close(victim->fd);
     victim->fd = -1;
+  }
+}
+
+/** Helper: release system resources (but not memory) held by a single
+ * logfile_t. */
+static void
+close_log(logfile_t *victim)
+{
+  if (victim->needs_close) {
+    close_log_sigsafe(victim);
   } else if (victim->is_syslog) {
 #ifdef HAVE_SYSLOG_H
     if (--syslog_count == 0) {
@@ -863,7 +901,7 @@ set_log_severity_config(int loglevelMin, int loglevelMax,
   raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
   memset(severity_out, 0, sizeof(log_severity_list_t));
   for (i = loglevelMin; i >= loglevelMax; --i) {
-    severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+    severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
 }
 
@@ -1020,7 +1058,7 @@ flush_pending_log_callbacks(void)
   do {
     SMARTLIST_FOREACH_BEGIN(messages, pending_log_message_t *, msg) {
       const int severity = msg->severity;
-      const int domain = msg->domain;
+      const log_domain_mask_t domain = msg->domain;
       for (lf = logfiles; lf; lf = lf->next) {
         if (! lf->callback || lf->seems_dead ||
             ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) {
@@ -1231,7 +1269,7 @@ add_android_log(const log_severity_list_t *severity,
   UNLOCK_LOGS();
   return 0;
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** If <b>level</b> is a valid log severity, return the corresponding
  * numeric value.  Otherwise, return -1. */
@@ -1267,9 +1305,16 @@ static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
   "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL",
-  "SCHED", "GUARD", "CONSDIFF", "DOS", NULL
+  "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", "MESG",
+  NULL
 };
 
+CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1);
+
+CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS);
+CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_);
+CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS);
+
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,
  * or 0 if there is no such name. */
 static log_domain_mask_t
@@ -1278,7 +1323,7 @@ parse_log_domain(const char *domain)
   int i;
   for (i=0; domain_list[i]; ++i) {
     if (!strcasecmp(domain, domain_list[i]))
-      return (1u<<i);
+      return (UINT64_C(1)<<i);
   }
   return 0;
 }
@@ -1350,7 +1395,7 @@ parse_log_severity_config(const char **cfg_ptr,
     const char *dash, *space;
     char *sev_lo, *sev_hi;
     int low, high, i;
-    log_domain_mask_t domains = ~0u;
+    log_domain_mask_t domains = LD_ALL_DOMAINS;
 
     if (*cfg == '[') {
       int err = 0;
@@ -1368,9 +1413,9 @@ parse_log_severity_config(const char **cfg_ptr,
       tor_free(domains_str);
       SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) {
             if (!strcmp(domain, "*")) {
-              domains = ~0u;
+              domains = LD_ALL_DOMAINS;
             } else {
-              int d;
+              log_domain_mask_t d;
               int negate=0;
               if (*domain == '~') {
                 negate = 1;
@@ -1464,7 +1509,7 @@ switch_logs_debug(void)
   LOCK_LOGS();
   for (lf = logfiles; lf; lf=lf->next) {
     for (i = LOG_DEBUG; i >= LOG_ERR; --i)
-      lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+      lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
   log_global_min_severity_ = get_min_log_level();
   UNLOCK_LOGS();
diff --git a/src/lib/log/log.h b/src/lib/log/log.h
index d7a5070610..f09c63cdd9 100644
--- a/src/lib/log/log.h
+++ b/src/lib/log/log.h
@@ -11,10 +11,12 @@
  **/
 
 #ifndef TOR_TORLOG_H
+#define TOR_TORLOG_H
 
 #include <stdarg.h>
 #include "lib/cc/torint.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/logging_types.h"
 #include "lib/testsupport/testsupport.h"
 
 #ifdef HAVE_SYSLOG_H
@@ -24,7 +26,7 @@
 #error "Your syslog.h thinks high numbers are more important.  " \
        "We aren't prepared to deal with that."
 #endif
-#else /* !(defined(HAVE_SYSLOG_H)) */
+#else /* !defined(HAVE_SYSLOG_H) */
 /* Note: Syslog's logging code refers to priorities, with 0 being the most
  * important.  Thus, all our comparisons needed to be reversed when we added
  * syslog support.
@@ -55,75 +57,92 @@
 /* Logging domains */
 
 /** Catch-all for miscellaneous events and fatal errors. */
-#define LD_GENERAL  (1u<<0)
+#define LD_GENERAL  (UINT64_C(1)<<0)
 /** The cryptography subsystem. */
-#define LD_CRYPTO   (1u<<1)
+#define LD_CRYPTO   (UINT64_C(1)<<1)
 /** Networking. */
-#define LD_NET      (1u<<2)
+#define LD_NET      (UINT64_C(1)<<2)
 /** Parsing and acting on our configuration. */
-#define LD_CONFIG   (1u<<3)
+#define LD_CONFIG   (UINT64_C(1)<<3)
 /** Reading and writing from the filesystem. */
-#define LD_FS       (1u<<4)
+#define LD_FS       (UINT64_C(1)<<4)
 /** Other servers' (non)compliance with the Tor protocol. */
-#define LD_PROTOCOL (1u<<5)
+#define LD_PROTOCOL (UINT64_C(1)<<5)
 /** Memory management. */
-#define LD_MM       (1u<<6)
+#define LD_MM       (UINT64_C(1)<<6)
 /** HTTP implementation. */
-#define LD_HTTP     (1u<<7)
+#define LD_HTTP     (UINT64_C(1)<<7)
 /** Application (socks) requests. */
-#define LD_APP      (1u<<8)
+#define LD_APP      (UINT64_C(1)<<8)
 /** Communication via the controller protocol. */
-#define LD_CONTROL  (1u<<9)
+#define LD_CONTROL  (UINT64_C(1)<<9)
 /** Building, using, and managing circuits. */
-#define LD_CIRC     (1u<<10)
+#define LD_CIRC     (UINT64_C(1)<<10)
 /** Hidden services. */
-#define LD_REND     (1u<<11)
+#define LD_REND     (UINT64_C(1)<<11)
 /** Internal errors in this Tor process. */
-#define LD_BUG      (1u<<12)
+#define LD_BUG      (UINT64_C(1)<<12)
 /** Learning and using information about Tor servers. */
-#define LD_DIR      (1u<<13)
+#define LD_DIR      (UINT64_C(1)<<13)
 /** Learning and using information about Tor servers. */
-#define LD_DIRSERV  (1u<<14)
+#define LD_DIRSERV  (UINT64_C(1)<<14)
 /** Onion routing protocol. */
-#define LD_OR       (1u<<15)
+#define LD_OR       (UINT64_C(1)<<15)
 /** Generic edge-connection functionality. */
-#define LD_EDGE     (1u<<16)
+#define LD_EDGE     (UINT64_C(1)<<16)
 #define LD_EXIT     LD_EDGE
 /** Bandwidth accounting. */
-#define LD_ACCT     (1u<<17)
+#define LD_ACCT     (UINT64_C(1)<<17)
 /** Router history */
-#define LD_HIST     (1u<<18)
+#define LD_HIST     (UINT64_C(1)<<18)
 /** OR handshaking */
-#define LD_HANDSHAKE (1u<<19)
+#define LD_HANDSHAKE (UINT64_C(1)<<19)
 /** Heartbeat messages */
-#define LD_HEARTBEAT (1u<<20)
+#define LD_HEARTBEAT (UINT64_C(1)<<20)
 /** Abstract channel_t code */
-#define LD_CHANNEL   (1u<<21)
+#define LD_CHANNEL   (UINT64_C(1)<<21)
 /** Scheduler */
-#define LD_SCHED     (1u<<22)
+#define LD_SCHED     (UINT64_C(1)<<22)
 /** Guard nodes */
-#define LD_GUARD     (1u<<23)
+#define LD_GUARD     (UINT64_C(1)<<23)
 /** Generation and application of consensus diffs. */
-#define LD_CONSDIFF  (1u<<24)
+#define LD_CONSDIFF  (UINT64_C(1)<<24)
 /** Denial of Service mitigation. */
-#define LD_DOS       (1u<<25)
-/** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 26
+#define LD_DOS       (UINT64_C(1)<<25)
+/** Processes */
+#define LD_PROCESS   (UINT64_C(1)<<26)
+/** Pluggable Transports. */
+#define LD_PT        (UINT64_C(1)<<27)
+/** Bootstrap tracker. */
+#define LD_BTRACK    (UINT64_C(1)<<28)
+/** Message-passing backend. */
+#define LD_MESG      (UINT64_C(1)<<29)
 
-/** This log message is not safe to send to a callback-based logger
- * immediately.  Used as a flag, not a log domain. */
-#define LD_NOCB (1u<<31)
-/** This log message should not include a function name, even if it otherwise
- * would. Used as a flag, not a log domain. */
-#define LD_NOFUNCNAME (1u<<30)
+/** The number of log domains. */
+#define N_LOGGING_DOMAINS 30
+/** The highest log domain */
+#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1))
+/** All log domains. */
+#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS))
+
+/** The number of log flags. */
+#define N_LOGGING_FLAGS 3
+/** First bit that is reserved in log_domain_mask_t for non-domain flags. */
+#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS))
+/** All log flags. */
+#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS))
 
 #ifdef TOR_UNIT_TESTS
 /** This log message should not be intercepted by mock_saving_logv */
-#define LD_NO_MOCK (1u<<29)
+#define LD_NO_MOCK (UINT64_C(1)<<61)
 #endif
 
-/** Mask of zero or more log domains, OR'd together. */
-typedef uint32_t log_domain_mask_t;
+/** This log message is not safe to send to a callback-based logger
+ * immediately.  Used as a flag, not a log domain. */
+#define LD_NOCB (UINT64_C(1)<<62)
+/** This log message should not include a function name, even if it otherwise
+ * would. Used as a flag, not a log domain. */
+#define LD_NOFUNCNAME (UINT64_C(1)<<63)
 
 /** Configures which severities are logged for each logging domain for a given
  * log target. */
@@ -134,7 +153,8 @@ typedef struct log_severity_list_t {
 } log_severity_list_t;
 
 /** Callback type used for add_callback_log. */
-typedef void (*log_callback)(int severity, uint32_t domain, const char *msg);
+typedef void (*log_callback)(int severity, log_domain_mask_t domain,
+                             const char *msg);
 
 void init_logging(int disable_startup_queue);
 int parse_log_level(const char *level);
@@ -186,6 +206,21 @@ void tor_log_get_logfile_names(struct smartlist_t *out);
 
 extern int log_global_min_severity_;
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we try to avoid our log_debug optimization, since it
+ * can have weird effects on internal macro coverage. */
+#define debug_logging_enabled() (1)
+#else
+static inline bool debug_logging_enabled(void);
+/**
+ * Return true iff debug logging is enabled for at least one domain.
+ */
+static inline bool debug_logging_enabled(void)
+{
+  return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG);
+}
+#endif /* defined(TOR_COVERAGE) */
+
 void log_fn_(int severity, log_domain_mask_t domain,
              const char *funcname, const char *format, ...)
   CHECK_PRINTF(4,5);
@@ -215,8 +250,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
   log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, args)
 #define log_debug(domain, args...)                                      \
   STMT_BEGIN                                                            \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))        \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);            \
+    if (debug_logging_enabled())                                        \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);                   \
   STMT_END
 #define log_info(domain, args...)                           \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args)
@@ -233,8 +268,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
 
 #define log_debug(domain, args, ...)                                        \
   STMT_BEGIN                                                                \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))            \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \
+    if (debug_logging_enabled())                                            \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__);        \
   STMT_END
 #define log_info(domain, args,...)                                      \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args, ##__VA_ARGS__)
@@ -272,5 +307,10 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
     va_list ap) CHECK_PRINTF(5,0));
 #endif
 
-# define TOR_TORLOG_H
+#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS)
+/** Given a severity, yields an index into log_severity_list_t.masks to use
+ * for that severity. */
+#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
+#endif
+
 #endif /* !defined(TOR_TORLOG_H) */
diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c
new file mode 100644
index 0000000000..826358546a
--- /dev/null
+++ b/src/lib/log/log_sys.c
@@ -0,0 +1,37 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.c
+ * \brief Setup and tear down the logging module.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/log_sys.h"
+
+static int
+subsys_logging_initialize(void)
+{
+  init_logging(0);
+  return 0;
+}
+
+static void
+subsys_logging_shutdown(void)
+{
+  logs_free_all();
+  escaped(NULL);
+}
+
+const subsys_fns_t sys_logging = {
+  .name = "log",
+  .supported = true,
+  /* Logging depends on threads, approx time, raw logging, and security.
+   * Most other lib modules depend on logging. */
+  .level = -90,
+  .initialize = subsys_logging_initialize,
+  .shutdown = subsys_logging_shutdown,
+};
diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h
new file mode 100644
index 0000000000..7043253066
--- /dev/null
+++ b/src/lib/log/log_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.h
+ * \brief Declare subsystem object for the logging module.
+ **/
+
+#ifndef TOR_LOG_SYS_H
+#define TOR_LOG_SYS_H
+
+extern const struct subsys_fns_t sys_logging;
+
+#endif /* !defined(TOR_LOG_SYS_H) */
diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h
index 48edd7c849..1db54ba726 100644
--- a/src/lib/log/ratelim.h
+++ b/src/lib/log/ratelim.h
@@ -50,4 +50,4 @@ typedef struct ratelim_t {
 
 char *rate_limit_log(ratelim_t *lim, time_t now);
 
-#endif
+#endif /* !defined(TOR_RATELIM_H) */
diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c
index c65a91ae9e..af1ac40732 100644
--- a/src/lib/log/util_bug.c
+++ b/src/lib/log/util_bug.c
@@ -11,6 +11,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/log/log.h"
 #include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
 #ifdef TOR_UNIT_TESTS
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -63,32 +64,52 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 {
   failed_assertion_cb = fn;
 }
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
 #define capturing_bugs() (0)
 #define add_captured_bug(s) do { } while (0)
 #endif /* defined(TOR_UNIT_TESTS) */
 
 /** Helper for tor_assert: report the assertion failure. */
 void
+CHECK_PRINTF(5, 6)
 tor_assertion_failed_(const char *fname, unsigned int line,
-                      const char *func, const char *expr)
+                      const char *func, const char *expr,
+                      const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
+  char *extra = NULL;
+  va_list ap;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+  if (fmt) {
+    va_start(ap,fmt);
+    tor_vasprintf(&extra, fmt, ap);
+    va_end(ap);
+  }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
   log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.",
           fname, line, func, expr);
-  tor_snprintf(buf, sizeof(buf),
-               "Assertion %s failed in %s at %s:%u",
-               expr, func, fname, line);
+  tor_asprintf(&buf, "Assertion %s failed in %s at %s:%u: %s",
+               expr, func, fname, line, extra ? extra : "");
+  tor_free(extra);
   log_backtrace(LOG_ERR, LD_BUG, buf);
+  tor_free(buf);
 }
 
 /** Helper for tor_assert_nonfatal: report the assertion failure. */
 void
+CHECK_PRINTF(6, 7)
 tor_bug_occurred_(const char *fname, unsigned int line,
                   const char *func, const char *expr,
-                  int once)
+                  int once, const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
   const char *once_str = once ?
     " (Future instances of this warning will be silenced.)": "";
   if (! expr) {
@@ -98,7 +119,7 @@ tor_bug_occurred_(const char *fname, unsigned int line,
     }
     log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s",
              fname, line, func, once_str);
-    tor_snprintf(buf, sizeof(buf),
+    tor_asprintf(&buf,
                  "Line unexpectedly reached at %s at %s:%u",
                  func, fname, line);
   } else {
@@ -106,13 +127,32 @@ tor_bug_occurred_(const char *fname, unsigned int line,
       add_captured_bug(expr);
       return;
     }
+
+    va_list ap;
+    char *extra = NULL;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+    if (fmt) {
+      va_start(ap,fmt);
+      tor_vasprintf(&extra, fmt, ap);
+      va_end(ap);
+    }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
     log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s",
              fname, line, func, expr, once_str);
-    tor_snprintf(buf, sizeof(buf),
-                 "Non-fatal assertion %s failed in %s at %s:%u",
-                 expr, func, fname, line);
+    tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s",
+                 expr, func, fname, line, fmt ? " : " : "",
+                 extra ? extra : "");
+    tor_free(extra);
   }
   log_backtrace(LOG_WARN, LD_BUG, buf);
+  tor_free(buf);
 
 #ifdef TOR_UNIT_TESTS
   if (failed_assertion_cb) {
@@ -122,16 +162,16 @@ tor_bug_occurred_(const char *fname, unsigned int line,
 }
 
 /**
- * Call the abort() function to kill the current process with a fatal
- * error.
+ * Call the tor_raw_abort_() function to close raw logs, then kill the current
+ * process with a fatal error.
  *
  * (This is a separate function so that we declare it in util_bug.h without
- * including stdlib in all the users of util_bug.h)
+ * including torerr.h in all the users of util_bug.h)
  **/
 void
 tor_abort_(void)
 {
-  abort();
+  tor_raw_abort_();
 }
 
 #ifdef _WIN32
diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h
index 2a4d68127e..7eaa6173c8 100644
--- a/src/lib/log/util_bug.h
+++ b/src/lib/log/util_bug.h
@@ -80,10 +80,10 @@
     tor__assert_tmp_value__;                    \
   } )
 #define ASSERT_PREDICT_LIKELY_(e) ASSERT_PREDICT_UNLIKELY_(e)
-#else
+#else /* !(defined(TOR_UNIT_TESTS) && defined(__GNUC__)) */
 #define ASSERT_PREDICT_UNLIKELY_(e) PREDICT_UNLIKELY(e)
 #define ASSERT_PREDICT_LIKELY_(e) PREDICT_LIKELY(e)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) && defined(__GNUC__) */
 
 /* Sometimes we don't want to use assertions during branch coverage tests; it
  * leads to tons of unreached branches which in reality are only assertions we
@@ -92,21 +92,28 @@
 #define tor_assert(a) STMT_BEGIN                                        \
   (void)(a);                                                            \
   STMT_END
-#else
+#define tor_assertf(a, fmt, ...) STMT_BEGIN                             \
+  (void)(a);                                                            \
+  (void)(fmt);                                                          \
+  STMT_END
+#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */
 /** Like assert(3), but send assertion failures to the log as well as to
  * stderr. */
-#define tor_assert(expr) STMT_BEGIN                                     \
+#define tor_assert(expr) tor_assertf(expr, NULL)
+
+#define tor_assertf(expr, fmt, ...) STMT_BEGIN                          \
   if (ASSERT_PREDICT_LIKELY_(expr)) {                                   \
   } else {                                                              \
-    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr);     \
-    tor_abort_();                                                       \
+    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr,      \
+                          fmt, ##__VA_ARGS__);                          \
+    tor_abort_();                                                        \
   } STMT_END
 #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */
 
 #define tor_assert_unreached()                                  \
   STMT_BEGIN {                                                  \
     tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__,     \
-                          "line should be unreached");          \
+                          "line should be unreached", NULL);    \
     tor_abort_();                                               \
   } STMT_END
 
@@ -136,34 +143,47 @@
 #ifdef ALL_BUGS_ARE_FATAL
 #define tor_assert_nonfatal_unreached() tor_assert(0)
 #define tor_assert_nonfatal(cond) tor_assert((cond))
+#define tor_assertf_nonfatal(cond, fmt, ...)    \
+  tor_assertf(cond, fmt, ##__VA_ARGS__)
 #define tor_assert_nonfatal_unreached_once() tor_assert(0)
 #define tor_assert_nonfatal_once(cond) tor_assert((cond))
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \
+   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",NULL), \
     tor_abort_(), 1)                                                    \
    : 0)
 #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS)
 #define tor_assert_nonfatal_unreached() STMT_NIL
 #define tor_assert_nonfatal(cond) ((void)(cond))
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  (void)cond;                                                           \
+  (void)fmt;                                                            \
+  STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_NIL
 #define tor_assert_nonfatal_once(cond) ((void)(cond))
 #define BUG(cond) (ASSERT_PREDICT_UNLIKELY_(cond) ? 1 : 0)
 #else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */
 #define tor_assert_nonfatal_unreached() STMT_BEGIN                      \
-  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0);         \
+  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0, NULL);   \
   STMT_END
 #define tor_assert_nonfatal(cond) STMT_BEGIN                            \
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else {                                                              \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, NULL);\
+  }                                                                     \
+  STMT_END
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  if (ASSERT_PREDICT_UNLIKELY_(cond)) {                                 \
+  } else {                                                              \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0,        \
+                      fmt, ##__VA_ARGS__);                               \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_BEGIN                 \
   static int warning_logged__ = 0;                                      \
   if (!warning_logged__) {                                              \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1);       \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1, NULL); \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_once(cond) STMT_BEGIN                       \
@@ -171,12 +191,12 @@
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else if (!warning_logged__) {                                       \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1, NULL);\
   }                                                                     \
   STMT_END
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \
+  (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0,NULL),1) \
    : 0)
 #endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
 
@@ -188,17 +208,17 @@
       if (bool_result && !var) {                                        \
         var = 1;                                                        \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                          "!("#cond")", 1);                             \
+                          "!("#cond")", 1, NULL);                       \
       }                                                                 \
       bool_result; } ))
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 #define IF_BUG_ONCE__(cond,var)                                         \
   static int var = 0;                                                   \
   if ((cond) ?                                                          \
       (var ? 1 :                                                        \
        (var=1,                                                          \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                           "!("#cond")", 1),                            \
+                           "!("#cond")", 1, NULL),                      \
         1))                                                             \
       : 0)
 #endif /* defined(__GNUC__) */
@@ -221,10 +241,13 @@
 #define tor_fragile_assert() tor_assert_nonfatal_unreached_once()
 
 void tor_assertion_failed_(const char *fname, unsigned int line,
-                           const char *func, const char *expr);
+                           const char *func, const char *expr,
+                           const char *fmt, ...)
+    CHECK_PRINTF(5,6);
 void tor_bug_occurred_(const char *fname, unsigned int line,
                        const char *func, const char *expr,
-                       int once);
+                       int once, const char *fmt, ...)
+  CHECK_PRINTF(6,7);
 
 void tor_abort_(void) ATTR_NORETURN;
 
diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c
index dc45cb4c3d..03d5c9fad2 100644
--- a/src/lib/log/win32err.c
+++ b/src/lib/log/win32err.c
@@ -47,7 +47,7 @@ format_win32_error(DWORD err)
     result = tor_malloc(len);
     wcstombs(result,str,len);
     result[len-1] = '\0';
-#else /* !(defined(UNICODE)) */
+#else /* !defined(UNICODE) */
     result = tor_strdup(str);
 #endif /* defined(UNICODE) */
   } else {
diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h
index 33413dfd15..ecfa88792d 100644
--- a/src/lib/log/win32err.h
+++ b/src/lib/log/win32err.h
@@ -19,4 +19,4 @@
 char *format_win32_error(DWORD err);
 #endif
 
-#endif
+#endif /* !defined(TOR_WIN32ERR_H) */
diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include
index cc62bb1013..7686bf862a 100644
--- a/src/lib/malloc/.may_include
+++ b/src/lib/malloc/.may_include
@@ -3,4 +3,4 @@ orconfig.h
 lib/cc/*.h
 lib/err/*.h
 lib/malloc/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am
index 502cc1c6b7..b74292bc6e 100644
--- a/src/lib/malloc/include.am
+++ b/src/lib/malloc/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-malloc-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_malloc_a_SOURCES =			\
-	src/lib/malloc/malloc.c
+	src/lib/malloc/malloc.c				\
+	src/lib/malloc/map_anon.c
 
 if USE_OPENBSD_MALLOC
 src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c
@@ -17,5 +19,7 @@ src_lib_libtor_malloc_testing_a_SOURCES = \
 src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/malloc/malloc.h
+	src/lib/malloc/malloc.h				\
+	src/lib/malloc/map_anon.h
diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h
index ef6b509ca4..39a45901a1 100644
--- a/src/lib/malloc/malloc.h
+++ b/src/lib/malloc/malloc.h
@@ -48,12 +48,12 @@ void tor_free_(void *mem);
     raw_free(*tor_free__tmpvar);                               \
     *tor_free__tmpvar=NULL;                                    \
   STMT_END
-#else
+#else /* !defined(__GNUC__) */
 #define tor_free(p) STMT_BEGIN                                 \
   raw_free(p);                                                 \
   (p)=NULL;                                                    \
   STMT_END
-#endif
+#endif /* defined(__GNUC__) */
 
 #define tor_malloc(size)       tor_malloc_(size)
 #define tor_malloc_zero(size)  tor_malloc_zero_(size)
diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c
new file mode 100644
index 0000000000..9559cbe2d4
--- /dev/null
+++ b/src/lib/malloc/map_anon.c
@@ -0,0 +1,271 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.c
+ * \brief Manage anonymous mappings.
+ **/
+
+#include "orconfig.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/malloc/malloc.h"
+#include "lib/err/torerr.h"
+
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_MACH_VM_INHERIT_H
+#include <mach/vm_inherit.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <string.h>
+#include <errno.h>
+
+/**
+ * Macro to get the high bytes of a size_t, if there are high bytes.
+ * Windows needs this; other operating systems define a size_t that does
+ * what it should.
+ */
+#if SIZEOF_SIZE_T > 4
+#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32)
+#else
+#define HIGH_SIZE_T_BYTES(sz) (0)
+#endif
+
+/* Here we define a MINHERIT macro that is minherit() or madvise(), depending
+ * on what we actually want.
+ *
+ * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO
+ * to be that flag.  If there's a flag unmaps pages after fork, we define
+ * FLAG_NOINHERIT to be that flag.
+ */
+#if defined(HAVE_MINHERIT)
+#define MINHERIT minherit
+
+#ifdef INHERIT_ZERO
+#define FLAG_ZERO INHERIT_ZERO
+#elif defined(MAP_INHERIT_ZERO)
+#define FLAG_ZERO MAP_INHERIT_ZERO
+#endif
+#ifdef INHERIT_NONE
+#define FLAG_NOINHERIT INHERIT_NONE
+#elif defined(VM_INHERIT_NONE)
+#define FLAG_NOINHERIT VM_INHERIT_NONE
+#elif defined(MAP_INHERIT_NONE)
+#define FLAG_NOINHERIT MAP_INHERIT_NONE
+#endif /* defined(INHERIT_NONE) || ... */
+
+#elif defined(HAVE_MADVISE)
+
+#define MINHERIT madvise
+
+#ifdef MADV_WIPEONFORK
+#define FLAG_ZERO MADV_WIPEONFORK
+#endif
+#ifdef MADV_DONTFORK
+#define FLAG_NOINHERIT MADV_DONTFORK
+#endif
+
+#endif /* defined(HAVE_MINHERIT) || ... */
+
+#if defined(HAVE_MINHERIT) && !defined(FLAG_ZERO) && !defined(FLAG_NOINHERIT)
+#warn "minherit() is defined, but we couldn't find the right flag for it."
+#warn "This is probably a bug in Tor's support for this platform."
+#endif
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped
+ * to disk.  Return 0 on success or if the facility is not available on this
+ * OS; return -1 on failure.
+ */
+static int
+lock_mem(void *mem, size_t sz)
+{
+#ifdef _WIN32
+  return VirtualLock(mem, sz) ? 0 : -1;
+#elif defined(HAVE_MLOCK)
+  return mlock(mem, sz);
+#else
+  (void) mem;
+  (void) sz;
+
+  return 0;
+#endif /* defined(_WIN32) || ... */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in
+ * a core dump.  Return 0 on success or if the facility is not available on
+ * this OS; return -1 on failure.
+ */
+static int
+nodump_mem(void *mem, size_t sz)
+{
+#if defined(MADV_DONTDUMP)
+  int rv = madvise(mem, sz, MADV_DONTDUMP);
+  if (rv == 0) {
+    return 0;
+  } else if (errno == ENOSYS || errno == EINVAL) {
+    return 0; // syscall not supported, or flag not supported.
+  } else {
+    tor_log_err_sigsafe("Unexpected error from madvise: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !defined(MADV_DONTDUMP) */
+  (void) mem;
+  (void) sz;
+  return 0;
+#endif /* defined(MADV_DONTDUMP) */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being
+ * accessible in child processes -- ideally by having them set to 0 after a
+ * fork, and if that doesn't work, by having them unmapped after a fork.
+ * Return 0 on success or if the facility is not available on this OS; return
+ * -1 on failure.
+ *
+ * If we successfully make the memory uninheritable, adjust the value of
+ * *<b>inherit_result_out</b>.
+ */
+static int
+noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out)
+{
+#ifdef FLAG_ZERO
+  int r = MINHERIT(mem, sz, FLAG_ZERO);
+  if (r == 0) {
+    *inherit_result_out = INHERIT_RES_ZERO;
+    return 0;
+  }
+#endif /* defined(FLAG_ZERO) */
+
+#ifdef FLAG_NOINHERIT
+  int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT);
+  if (r2 == 0) {
+    *inherit_result_out = INHERIT_RES_DROP;
+    return 0;
+  }
+#endif /* defined(FLAG_NOINHERIT) */
+
+#if defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)
+  /* At least one operation was tried, and neither succeeded. */
+
+  if (errno == ENOSYS || errno == EINVAL) {
+    /* Syscall not supported, or flag not supported. */
+    return 0;
+  } else {
+    tor_log_err_sigsafe("Unexpected error from minherit: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !(defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)) */
+  (void)inherit_result_out;
+  (void)mem;
+  (void)sz;
+  return 0;
+#endif /* defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) */
+}
+
+/**
+ * Return a new anonymous memory mapping that holds <b>sz</b> bytes.
+ *
+ * Memory mappings are unlike the results from malloc() in that they are
+ * handled separately by the operating system, and as such can have different
+ * kernel-level flags set on them.
+ *
+ * The "flags" argument may be zero or more of ANONMAP_PRIVATE and
+ * ANONMAP_NOINHERIT.
+ *
+ * Memory returned from this function must be released with
+ * tor_munmap_anonymous().
+ *
+ * If <b>inherit_result_out</b> is non-NULL, set it to one of
+ * INHERIT_RES_KEEP, INHERIT_RES_DROP, or INHERIT_RES_ZERO, depending on the
+ * properties of the returned memory.
+ *
+ * [Note: OS people use the word "anonymous" here to mean that the memory
+ * isn't associated with any file. This has *nothing* to do with the kind of
+ * anonymity that Tor is trying to provide.]
+ */
+void *
+tor_mmap_anonymous(size_t sz, unsigned flags,
+                   inherit_res_t *inherit_result_out)
+{
+  void *ptr;
+  inherit_res_t itmp=0;
+  if (inherit_result_out == NULL) {
+    inherit_result_out = &itmp;
+  }
+  *inherit_result_out = INHERIT_RES_KEEP;
+
+#if defined(_WIN32)
+  HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE,
+                                     NULL, /*attributes*/
+                                     PAGE_READWRITE,
+                                     HIGH_SIZE_T_BYTES(sz),
+                                     sz & 0xffffffff,
+                                     NULL /* name */);
+  raw_assert(mapping != NULL);
+  ptr = MapViewOfFile(mapping, FILE_MAP_WRITE,
+                      0, 0, /* Offset */
+                      0 /* Extend to end of mapping */);
+  raw_assert(ptr);
+  CloseHandle(mapping); /* mapped view holds a reference */
+#elif defined(HAVE_SYS_MMAN_H)
+  ptr = mmap(NULL, sz,
+             PROT_READ|PROT_WRITE,
+             MAP_ANON|MAP_PRIVATE,
+             -1, 0);
+  raw_assert(ptr != MAP_FAILED);
+  raw_assert(ptr != NULL);
+#else
+  ptr = tor_malloc_zero(sz);
+#endif /* defined(_WIN32) || ... */
+
+  if (flags & ANONMAP_PRIVATE) {
+    int lock_result = lock_mem(ptr, sz);
+    raw_assert(lock_result == 0);
+    int nodump_result = nodump_mem(ptr, sz);
+    raw_assert(nodump_result == 0);
+  }
+
+  if (flags & ANONMAP_NOINHERIT) {
+    int noinherit_result = noinherit_mem(ptr, sz, inherit_result_out);
+    raw_assert(noinherit_result == 0);
+  }
+
+  return ptr;
+}
+
+/**
+ * Release <b>sz</b> bytes of memory that were previously mapped at
+ * <b>mapping</b> by tor_mmap_anonymous().
+ **/
+void
+tor_munmap_anonymous(void *mapping, size_t sz)
+{
+  if (!mapping)
+    return;
+
+#if defined(_WIN32)
+  (void)sz;
+  UnmapViewOfFile(mapping);
+#elif defined(HAVE_SYS_MMAN_H)
+  munmap(mapping, sz);
+#else
+  (void)sz;
+  tor_free(mapping);
+#endif /* defined(_WIN32) || ... */
+}
diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h
new file mode 100644
index 0000000000..4c4690e12f
--- /dev/null
+++ b/src/lib/malloc/map_anon.h
@@ -0,0 +1,71 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.h
+ * \brief Headers for map_anon.c
+ **/
+
+#ifndef TOR_MAP_ANON_H
+#define TOR_MAP_ANON_H
+
+#include "lib/malloc/malloc.h"
+#include <stddef.h>
+
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * swapped or dumped.
+ *
+ * In some operating systems, this flag is not implemented.
+ */
+#define ANONMAP_PRIVATE   (1u<<0)
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * inherited after a fork().  In some operating systems, trying to access it
+ * afterwards will cause its contents to be zero.  In others, trying to access
+ * it afterwards will cause a crash.
+ *
+ * In some operating systems, this flag is not implemented at all.
+ */
+#define ANONMAP_NOINHERIT (1u<<1)
+
+typedef enum {
+  /** Possible value for inherit_result_out: the memory will be kept
+   * by any child process. */
+  INHERIT_RES_KEEP=0,
+  /** Possible value for inherit_result_out: the memory will be dropped in the
+   * child process. Attempting to access it will likely cause a segfault. */
+  INHERIT_RES_DROP,
+  /** Possible value for inherit_result_out: the memory will be cleared in
+   * the child process. */
+  INHERIT_RES_ZERO
+} inherit_res_t;
+
+/* Here we define the NOINHERIT_CAN_FAIL macro if and only if
+ * it's possible that ANONMAP_NOINHERIT might yield inheritable memory.
+ */
+#ifdef _WIN32
+/* Windows can't fork, so NOINHERIT is never needed. */
+#elif defined(HAVE_MINHERIT)
+/* minherit() will always have a working MAP_INHERIT_NONE or MAP_INHERIT_ZERO.
+ * NOINHERIT should always work.
+ */
+#elif defined(HAVE_MADVISE)
+/* madvise() sometimes has neither MADV_DONTFORK and MADV_WIPEONFORK.
+ * We need to be ready for the possibility it failed.
+ *
+ * (Linux added DONTFORK in 2.6.16 and WIPEONFORK in 4.14. If we someday
+ * require 2.6.16 or later, we can assume that DONTFORK will work.)
+ */
+#define NOINHERIT_CAN_FAIL
+#else
+#define NOINHERIT_CAN_FAIL
+#endif /* defined(_WIN32) || ... */
+
+void *tor_mmap_anonymous(size_t sz, unsigned flags,
+                         inherit_res_t *inherit_result_out);
+void tor_munmap_anonymous(void *mapping, size_t sz);
+
+#endif /* !defined(TOR_MAP_ANON_H) */
diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include
index 1fd26864dc..f8bc264a5f 100644
--- a/src/lib/math/.may_include
+++ b/src/lib/math/.may_include
@@ -3,3 +3,5 @@ orconfig.h
 lib/cc/*.h
 lib/log/*.h
 lib/math/*.h
+lib/testsupport/*.h
+lib/crypt_ops/*.h
diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c
index eafad358c3..49a2a6a2ca 100644
--- a/src/lib/math/fp.c
+++ b/src/lib/math/fp.c
@@ -75,7 +75,7 @@ clamp_double_to_int64(double number)
 */
 #define PROBLEMATIC_FLOAT_CONVERSION_WARNING
 DISABLE_GCC_WARNING(float-conversion)
-#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */
+#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */
 
 /*
   With clang 4.0 we apparently run into "double promotion" warnings here,
@@ -121,3 +121,23 @@ ENABLE_GCC_WARNING(double-promotion)
 ENABLE_GCC_WARNING(float-conversion)
 #endif
 }
+
+/* isinf() wrapper for tor */
+int
+tor_isinf(double x)
+{
+  /* Same as above, work around the "double promotion" warnings */
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+DISABLE_GCC_WARNING(float-conversion)
+#endif
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+DISABLE_GCC_WARNING(double-promotion)
+#endif
+  return isinf(x);
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+ENABLE_GCC_WARNING(double-promotion)
+#endif
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+ENABLE_GCC_WARNING(float-conversion)
+#endif
+}
diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h
index 6f07152e92..a73789c945 100644
--- a/src/lib/math/fp.h
+++ b/src/lib/math/fp.h
@@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST;
 long tor_lround(double d) ATTR_CONST;
 int64_t tor_llround(double d) ATTR_CONST;
 int64_t clamp_double_to_int64(double number);
+int tor_isinf(double x);
 
-#endif
+#endif /* !defined(TOR_FP_H) */
diff --git a/src/lib/math/include.am b/src/lib/math/include.am
index b088b3f3cc..b2ca280f47 100644
--- a/src/lib/math/include.am
+++ b/src/lib/math/include.am
@@ -5,16 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-math-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_math_a_SOURCES =	\
 		src/lib/math/fp.c		\
-		src/lib/math/laplace.c
-
+		src/lib/math/laplace.c 	\
+		src/lib/math/prob_distr.c
 
 src_lib_libtor_math_testing_a_SOURCES = \
 	$(src_lib_libtor_math_a_SOURCES)
 src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/math/fp.h		\
-		src/lib/math/laplace.h
+		src/lib/math/laplace.h  \
+		src/lib/math/prob_distr.h
diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h
index e8651e5197..02b0e890f0 100644
--- a/src/lib/math/laplace.h
+++ b/src/lib/math/laplace.h
@@ -19,4 +19,4 @@ int64_t sample_laplace_distribution(double mu, double b, double p);
 int64_t add_laplace_noise(int64_t signal, double random, double delta_f,
                           double epsilon);
 
-#endif
+#endif /* !defined(TOR_LAPLACE_H) */
diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c
new file mode 100644
index 0000000000..d44dc28265
--- /dev/null
+++ b/src/lib/math/prob_distr.c
@@ -0,0 +1,1688 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file prob_distr.c
+ *
+ * \brief
+ *  Implements various probability distributions.
+ *  Almost all code is courtesy of Riastradh.
+ *
+ * \details
+ * Here are some details that might help you understand this file:
+ *
+ * - Throughout this file, `eps' means the largest relative error of a
+ *   correctly rounded floating-point operation, which in binary64
+ *   floating-point arithmetic is 2^-53.  Here the relative error of a
+ *   true value x from a computed value y is |x - y|/|x|.  This
+ *   definition of epsilon is conventional for numerical analysts when
+ *   writing error analyses.  (If your libm doesn't provide correctly
+ *   rounded exp and log, their relative error is usually below 2*2^-53
+ *   and probably closer to 1.1*2^-53 instead.)
+ *
+ *   The C constant DBL_EPSILON is actually twice this, and should
+ *   perhaps rather be named ulp(1) -- that is, it is the distance from
+ *   1 to the next greater floating-point number, which is usually of
+ *   more interest to programmers and hardware engineers.
+ *
+ *   Since this file is concerned mainly with error bounds rather than
+ *   with low-level bit-hacking of floating-point numbers, we adopt the
+ *   numerical analysts' definition in the comments, though we do use
+ *   DBL_EPSILON in a handful of places where it is convenient to use
+ *   some function of eps = DBL_EPSILON/2 in a case analysis.
+ *
+ * - In various functions (e.g. sample_log_logistic()) we jump through hoops so
+ *   that we can use reals closer to 0 than closer to 1, since we achieve much
+ *   greater accuracy for floating point numbers near 0. In particular, we can
+ *   represent differences as small as 10^-300 for numbers near 0, but of no
+ *   less than 10^-16 for numbers near 1.
+ **/
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "lib/math/prob_distr.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/cc/ctassert.h"
+#include "lib/log/util_bug.h"
+
+#include <float.h>
+#include <math.h>
+#include <stddef.h>
+
+/** Declare a function that downcasts from a generic dist struct to the actual
+ *  subtype probablity distribution it represents. */
+#define DECLARE_PROB_DISTR_DOWNCAST_FN(name) \
+  static inline                           \
+  const struct name *                             \
+  dist_to_const_##name(const struct dist *obj) {    \
+    tor_assert(obj->ops == &name##_ops);            \
+    return SUBTYPE_P(obj, struct name, base);   \
+  }
+DECLARE_PROB_DISTR_DOWNCAST_FN(uniform)
+DECLARE_PROB_DISTR_DOWNCAST_FN(geometric)
+DECLARE_PROB_DISTR_DOWNCAST_FN(logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(log_logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(genpareto)
+DECLARE_PROB_DISTR_DOWNCAST_FN(weibull)
+
+/**
+ * Count number of one bits in 32-bit word.
+ */
+static unsigned
+bitcount32(uint32_t x)
+{
+
+  /* Count two-bit groups.  */
+  x -= (x >> 1) & UINT32_C(0x55555555);
+
+  /* Count four-bit groups.  */
+  x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333));
+
+  /* Count eight-bit groups.  */
+  x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f);
+
+  /* Sum all eight-bit groups, and extract the sum.  */
+  return (x * UINT32_C(0x01010101)) >> 24;
+}
+
+/**
+ * Count leading zeros in 32-bit word.
+ */
+static unsigned
+clz32(uint32_t x)
+{
+
+  /* Round up to a power of two.  */
+  x |= x >> 1;
+  x |= x >> 2;
+  x |= x >> 4;
+  x |= x >> 8;
+  x |= x >> 16;
+
+  /* Subtract count of one bits from 32.  */
+  return (32 - bitcount32(x));
+}
+
+/*
+ * Some lemmas that will be used throughout this file to prove various error
+ * bounds:
+ *
+ * Lemma 1.  If |d| <= 1/2, then 1/(1 + d) <= 2.
+ *
+ * Proof.  If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1.
+ * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2.  QED.
+ *
+ * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b,
+ * then b = a*(1 + e) for |e| <= 2|d' - d|.
+ *
+ * Proof.  |a - b|/|a|
+ *             = |a - a*(1 + d)/(1 + d')|/|a|
+ *             = |1 - (1 + d)/(1 + d')|
+ *             = |(1 + d' - 1 - d)/(1 + d')|
+ *             = |(d' - d)/(1 + d')|
+ *            <= 2|d' - d|, by Lemma 1,
+ *
+ * QED.
+ *
+ * Lemma 3.  For |d|, |d'| < 1/4,
+ *
+ *     |log((1 + d)/(1 + d'))| <= 4|d - d'|.
+ *
+ * Proof.  Write
+ *
+ *     log((1 + d)/(1 + d'))
+ *      = log(1 + (1 + d)/(1 + d') - 1)
+ *      = log(1 + (1 + d - 1 - d')/(1 + d')
+ *      = log(1 + (d - d')/(1 + d')).
+ *
+ * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor
+ * series of log(1 + x) converges absolutely for (d - d')/(1 + d'),
+ * and thus we have
+ *
+ *     |log(1 + (d - d')/(1 + d'))|
+ *      = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n|
+ *     <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n
+ *      = 1/(1 - |2(d' - d)|)
+ *     <= 4|d' - d|,
+ *
+ * QED.
+ *
+ * Lemma 4.  If 1/e <= 1 + x <= e, then
+ *
+ *     log(1 + (1 + d) x) = (1 + d') log(1 + x)
+ *
+ * for |d'| < 8|d|.
+ *
+ * Proof.  Write
+ *
+ *     log(1 + (1 + d) x)
+ *     = log(1 + x + x*d)
+ *     = log((1 + x) (1 + x + x*d)/(1 + x))
+ *     = log(1 + x) + log((1 + x + x*d)/(1 + x))
+ *     = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)).
+ *
+ * The relative error is bounded by
+ *
+ *     |log((1 + x + x*d)/(1 + x))/log(1 + x)|
+ *     <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3,
+ *      = 4|x*d|/|log(1 + x)|
+ *      < 8|d|,
+ *
+ * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2.  QED.
+ */
+
+/**
+ * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x).
+ * Maps a log-odds-space probability in [-\infty, +\infty] into a direct-space
+ * probability in [0,1].  Inverse of logit.
+ *
+ * Ill-conditioned for large x; the identity logistic(-x) = 1 -
+ * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may
+ * help to rearrange a computation.
+ *
+ * This implementation gives relative error bounded by 7 eps.
+ */
+STATIC double
+logistic(double x)
+{
+  if (x <= log(DBL_EPSILON/2)) {
+    /*
+     * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case
+     * we will approximate the logistic() function with e^x because the
+     * relative error is less than eps. Here is a calculation of the
+     * relative error between the logistic() function and e^x and a proof
+     * that it's less than eps:
+     *
+     *     |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)|
+     *     <= |1 - 1/(1 + e^x)|*|1 + e^x|
+     *      = |e^x/(1 + e^x)|*|1 + e^x|
+     *      = |e^x|
+     *     <= eps.
+     */
+    return exp(x); /* return e^x */
+  } else if (x <= -log(DBL_EPSILON/2)) {
+    /*
+     * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 +
+     * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we
+     * also have 0 < 1/(1 + e^{-x}) < 1.  Thus, if exp has
+     * relative error d0, + has relative error d1, and /
+     * has relative error d2, then we get
+     *
+     *     (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)]
+     *     = (1 + d0)/[1 + e^{-x} + d0 e^{-x}
+     *                     + d1 + d1 e^{-x} + d0 d1 e^{-x}]
+     *     = (1 + d0)/[(1 + e^{-x})
+     *                 * (1 + d0 e^{-x}/(1 + e^{-x})
+     *                      + d1/(1 + e^{-x})
+     *                      + d0 d1 e^{-x}/(1 + e^{-x}))].
+     *     = (1 + d0)/[(1 + e^{-x})(1 + d')]
+     *     = [1/(1 + e^{-x})] (1 + d0)/(1 + d')
+     *
+     * where
+     *
+     *     d' = d0 e^{-x}/(1 + e^{-x})
+     *          + d1/(1 + e^{-x})
+     *          + d0 d1 e^{-x}/(1 + e^{-x}).
+     *
+     * By Lemma 2 this relative error is bounded by
+     *
+     *     2|d0 - d'|
+     *      = 2|d0 - d0 e^{-x}/(1 + e^{-x})
+     *             - d1/(1 + e^{-x})
+     *             - d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})|
+     *             + 2|d1/(1 + e^{-x})|
+     *             + 2|d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1|
+     *     <= 4|d0| + 2|d1| + 2|d0 d1|
+     *     <= 6 eps + 2 eps^2.
+     */
+    return 1/(1 + exp(-x));
+  } else {
+    /*
+     * e^{-x} <= eps, so the relative error of 1 from 1/(1
+     * + e^{-x}) is
+     *
+     *     |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})|
+     *      = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})|
+     *      = |e^{-x}|
+     *     <= eps.
+     *
+     * This computation avoids an intermediate overflow
+     * exception, although the effect on the result is
+     * harmless.
+     *
+     * XXX Should maybe raise inexact here.
+     */
+    return 1;
+  }
+}
+
+/**
+ * Compute the logit function: log p/(1 - p).  Defined on [0,1].  Maps
+ * a direct-space probability in [0,1] to a log-odds-space probability
+ * in [-\infty, +\infty].  Inverse of logistic.
+ *
+ * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) =
+ * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help
+ * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)].
+ *
+ * This implementation gives relative error bounded by 10 eps.
+ */
+STATIC double
+logit(double p)
+{
+
+  /* logistic(-1) <= p <= logistic(+1) */
+  if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) {
+    /*
+     * For inputs near 1/2, we want to compute log1p(near
+     * 0) rather than log(near 1), so write this as:
+     *
+     * log(p/(1 - p)) = -log((1 - p)/p)
+     * = -log(1 + (1 - p)/p - 1)
+     * = -log(1 + (1 - p - p)/p)
+     * = -log(1 + (1 - 2p)/p).
+     *
+     * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point
+     * evaluation of 1 - 2p is exact; the only error arises
+     * from division and log1p.  First, note that if
+     * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies
+     * in the bounds of Lemma 4.
+     *
+     * If division has relative error d0 and log1p has
+     * relative error d1, the outcome is
+     *
+     *     -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p)
+     *     = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p)
+     *     = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p).
+     *
+     * where |d'| < 8|d0| by Lemma 4.  The relative error
+     * is then bounded by
+     *
+     *     |d1 + d' + d1 d'|
+     *     <= |d1| + 8|d0| + 8|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return -log1p((1 - 2*p)/p);
+  } else {
+    /*
+     * For inputs near 0, although 1 - p may be rounded to
+     * 1, it doesn't matter much because the magnitude of
+     * the result is so much larger.  For inputs near 1, we
+     * can compute 1 - p exactly, although the precision on
+     * the input is limited so we won't ever get more than
+     * about 700 for the output.
+     *
+     * If - has relative error d0, / has relative error d1,
+     * and log has relative error d2, then
+     *
+     *     (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)])
+     *     = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))]
+     *     = log(p/(1 - p)) + d2 log(p/(1 - p))
+     *       + (1 + d2) log((1 + d0)/(1 + d1))
+     *     = log(p/(1 - p))*[1 + d2 +
+     *         + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))]
+     *
+     * Since 0 <= p < logistic(-1) or logistic(+1) < p <=
+     * 1, we have |log(p/(1 - p))| > 1.  Hence this error
+     * is bounded by
+     *
+     *     |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))
+     *                      / log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))|
+     *     <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3,
+     *     <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0|
+     *     <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return log(p/(1 - p));
+  }
+}
+
+/**
+ * Compute the logit function, translated in input by 1/2: logithalf(p)
+ * = logit(1/2 + p).  Defined on [-1/2, 1/2].  Inverse of logistichalf.
+ *
+ * Ill-conditioned near +/-1/2.  If |p0| > 1/2 - 1/(1 + e), it may be
+ * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead.
+ * This implementation gives relative error bounded by 34 eps.
+ */
+STATIC double
+logithalf(double p0)
+{
+
+  if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) {
+    /*
+     * logit(1/2 + p0)
+     * = log((1/2 + p0)/(1 - (1/2 + p0)))
+     * = log((1/2 + p0)/(1/2 - p0))
+     * = log(1 + (1/2 + p0)/(1/2 - p0) - 1)
+     * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0))
+     * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0))
+     * = log(1 + 2 p0/(1/2 - p0))
+     *
+     * If the error of subtraction is d0, the error of
+     * division is d1, and the error of log1p is d2, then
+     * what we compute is
+     *
+     *  (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)])
+     *  = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0))
+     *  = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0))
+     *  = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)),
+     *
+     * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and
+     * |d''| < 8|d'| < 32 eps by Lemma 4 since
+     *
+     *  1/e <= 1 + 2*p0/(1/2 - p0) <= e
+     *
+     * when |p0| <= 1/2 - 1/(1 + e).  Hence the relative
+     * error is bounded by
+     *
+     *  |d2 + d'' + d2 d''|
+     *  <= |d2| + |d''| + |d2 d''|
+     *  <= |d1| + 32 |d0| + 32 |d1 d0|
+     *  <= 33 eps + 32 eps^2.
+     */
+    return log1p(2*p0/(0.5 - p0));
+  } else {
+    /*
+     * We have a choice of computing logit(1/2 + p0) or
+     * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0).  It
+     * doesn't matter which way we do this: either way,
+     * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference
+     * are computed exactly.  So let's do the one that
+     * skips the final negation.
+     *
+     * The result is
+     *
+     *  (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)])
+     *  = (1 + d1) (1 + log((1 + d0)/(1 + d2))
+     *                  / log((1/2 + p0)/(1/2 - p0)))
+     *    * log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') logit(1/2 + p0)
+     *
+     * where
+     *
+     *  d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0)
+     *       + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0).
+     *
+     * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1.
+     * Provided |d0|, |d2| < 1/4, by Lemma 3 we have
+     *
+     *  |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|.
+     *
+     * Hence the relative error is bounded by
+     *
+     *  |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2|
+     *       <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2|
+     *       <= 9 eps + 8 eps^2.
+     */
+    return log((0.5 + p0)/(0.5 - p0));
+  }
+}
+
+/*
+ * The following random_uniform_01 is tailored for IEEE 754 binary64
+ * floating-point or smaller.  It can be adapted to larger
+ * floating-point formats like i387 80-bit or IEEE 754 binary128, but
+ * it may require sampling more bits.
+ */
+CTASSERT(FLT_RADIX == 2);
+CTASSERT(-DBL_MIN_EXP <= 1021);
+CTASSERT(DBL_MANT_DIG <= 53);
+
+/**
+ * Draw a floating-point number in [0, 1] with uniform distribution.
+ *
+ * Note that the probability of returning 0 is less than 2^-1074, so
+ * callers need not check for it.  However, callers that cannot handle
+ * rounding to 1 must deal with that, because it occurs with
+ * probability 2^-54, which is small but nonnegligible.
+ */
+STATIC double
+random_uniform_01(void)
+{
+  uint32_t z, x, hi, lo;
+  double s;
+
+  /*
+   * Draw an exponent, geometrically distributed, but give up if
+   * we get a run of more than 1088 zeros, which really means the
+   * system is broken.
+   */
+  z = 0;
+  while ((x = crypto_fast_rng_get_u32(get_thread_fast_rng())) == 0) {
+    if (z >= 1088)
+      /* Your bit sampler is broken.  Go home.  */
+      return 0;
+    z += 32;
+  }
+  z += clz32(x);
+
+  /*
+   * Pick 32-bit halves of an odd normalized significand.
+   * Picking it odd breaks ties in the subsequent rounding, which
+   * occur only with measure zero in the uniform distribution on
+   * [0, 1].
+   */
+  hi = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x80000000);
+  lo = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x00000001);
+
+  /* Round to nearest scaled significand in [2^63, 2^64].  */
+  s = hi*(double)4294967296 + lo;
+
+  /* Rescale into [1/2, 1] and apply exponent in one swell foop.  */
+  return s * ldexp(1, -(64 + z));
+}
+
+/*******************************************************************/
+
+/* Functions for specific probability distributions start here: */
+
+/*
+ * Logistic(mu, sigma) distribution, supported on (-\infty,+\infty)
+ *
+ * This is the uniform distribution on [0,1] mapped into log-odds
+ * space, scaled by sigma and translated by mu.
+ *
+ * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2
+ * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma)
+ * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma)
+ * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p)
+ * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p)
+ */
+
+/**
+ * Compute the CDF of the Logistic(mu, sigma) distribution: the
+ * logistic function.  Well-conditioned for negative inputs and small
+ * positive inputs; ill-conditioned for large positive inputs.
+ */
+STATIC double
+cdf_logistic(double x, double mu, double sigma)
+{
+  return logistic((x - mu)/sigma);
+}
+
+/**
+ * Compute the SF of the Logistic(mu, sigma) distribution: the logistic
+ * function reflected over the y axis.  Well-conditioned for positive
+ * inputs and small negative inputs; ill-conditioned for large negative
+ * inputs.
+ */
+STATIC double
+sf_logistic(double x, double mu, double sigma)
+{
+  return logistic(-(x - mu)/sigma);
+}
+
+/**
+ * Compute the inverse of the CDF of the Logistic(mu, sigma)
+ * distribution: the logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+icdf_logistic(double p, double mu, double sigma)
+{
+  return mu + sigma*logit(p);
+}
+
+/**
+ * Compute the inverse of the SF of the Logistic(mu, sigma)
+ * distribution: the -logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+isf_logistic(double p, double mu, double sigma)
+{
+  return mu - sigma*logit(p);
+}
+
+/*
+ * LogLogistic(alpha, beta) distribution, supported on (0, +\infty).
+ *
+ * This is the uniform distribution on [0,1] mapped into odds space,
+ * scaled by positive alpha and shaped by positive beta.
+ *
+ * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample.
+ * (Name arises because the pdf has LogLogistic(x; alpha, beta) =
+ * Logistic(log x; log alpha, 1/beta) and mathematicians got their
+ * covariance contravariant.)
+ *
+ * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2
+ *        = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} /
+ *              (1 + (x/e^mu)^{1/sigma})^2
+ * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x}/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x - mu})^{-1/sigma})
+ *        = 1/(1 + e^{-(log x - mu)/sigma})
+ *        = logistic((log x - mu)/sigma)
+ *        = logistic((log x - log alpha)/(1/beta))
+ * sf(x) = 1 - 1/(1 + (x/alpha)^-beta)
+ *       = (x/alpha)^-beta/(1 + (x/alpha)^-beta)
+ *       = 1/((x/alpha)^beta + 1)
+ *       = 1/(1 + (x/alpha)^beta)
+ * icdf(p) = alpha (p/(1 - p))^{1/beta}
+ *         = alpha e^{logit(p)/beta}
+ *         = e^{mu + sigma logit(p)}
+ * isf(p) = alpha ((1 - p)/p)^{1/beta}
+ *        = alpha e^{-logit(p)/beta}
+ *        = e^{mu - sigma logit(p)}
+ */
+
+/**
+ * Compute the CDF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      -beta/[1 + (x/alpha)^{-beta}]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps),
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, around which point it levels off at 1.
+ */
+STATIC double
+cdf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and
+   * d3, of the final quotient.  The exponentiation gives
+   *
+   *    ((1 + d0) x/alpha)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + d0)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1)
+   *    = (x/alpha)^{-beta} (1 + d')
+   *
+   * where d' = (1 + d0)^{-beta} - 1.  If y = (x/alpha)^{-beta},
+   * the denominator is
+   *
+   *    (1 + d2) (1 + (1 + d1) (1 + d') y)
+   *    = (1 + d2) (1 + y + (d1 + d' + d1 d') y)
+   *    = 1 + y + (1 + d2) (d1 + d' + d1 d') y
+   *    = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y))
+   *    = (1 + y) (1 + d''),
+   *
+   * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y).  The
+   * final result is
+   *
+   *    (1 + d3) / [(1 + d2) (1 + d'') (1 + y)]
+   *    = (1 + d''') / (1 + y)
+   *
+   * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2
+   * (which may not be the case for very large beta).  This
+   * relative error is therefore bounded by
+   *
+   *    |d'''|
+   *    <= 2|d3 - d''|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')|
+   *     = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'|
+   *      <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'|
+   *         + 2|d2 d1 d'|
+   *      <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|.
+   *
+   * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps,
+   * until it levels off at 1.
+   */
+  return 1/(1 + pow(x/alpha, -beta));
+}
+
+/**
+ * Compute the SF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      beta/[1 + (x/alpha)^beta]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps)
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, beyond which point it grows unbounded.
+ */
+STATIC double
+sf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * The error analysis here is essentially the same as in
+   * cdf_log_logistic, except that rather than levelling off at
+   * 1, |(1 + d0)^beta - 1| grows unbounded.
+   */
+  return 1/(1 + pow(x/alpha, beta));
+}
+
+/**
+ * Compute the inverse of the CDF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and beta near 0 with
+ * condition number 1/[beta (1 - p)].
+ */
+STATIC double
+icdf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow(p/(1 - p), 1/beta);
+}
+
+/**
+ * Compute the inverse of the SF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and for large beta, with
+ * condition number -1/[beta (1 - p)].
+ */
+STATIC double
+isf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow((1 - p)/p, 1/beta);
+}
+
+/*
+ * Weibull(lambda, k) distribution, supported on (0, +\infty).
+ *
+ * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k}
+ * cdf(x) = 1 - e^{-(x/lambda)^k}
+ * icdf(p) = lambda * (-log (1 - p))^{1/k}
+ * sf(x) = e^{-(x/lambda)^k}
+ * isf(p) = lambda * (-log p)^{1/k}
+ */
+
+/**
+ * Compute the CDF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1]
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+cdf_weibull(double x, double lambda, double k)
+{
+  return -expm1(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the SF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+sf_weibull(double x, double lambda, double k)
+{
+  return exp(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the inverse of the CDF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 1, and for k near 0;
+ * condition number is
+ *
+ *      (p/(1 - p))/(k log(1 - p)).
+ */
+STATIC double
+icdf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log1p(-p), 1/k);
+}
+
+/**
+ * Compute the inverse of the SF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 0, and for k near 0;
+ * condition number is
+ *
+ *      1/(k log(p)).
+ */
+STATIC double
+isf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log(p), 1/k);
+}
+
+/*
+ * GeneralizedPareto(mu, sigma, xi), supported on (mu, +\infty) for
+ * nonnegative xi, or (mu, mu - sigma/xi) for negative xi.
+ *
+ * Samples:
+ * = mu - sigma log U, if xi = 0;
+ * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0,
+ * where U is uniform on (0,1].
+ * = mu + sigma (e^{xi X} - 1)/xi,
+ * where X has standard exponential distribution.
+ *
+ * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)}
+ * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi}
+ *        = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi}
+ *        --> 1 - e^{-(x - mu)/sigma}  as  xi --> 0
+ * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi}
+ *       --> e^{-(x - mu)/sigma}  as  xi --> 0
+ * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi
+ *         = mu + sigma*expm1(-xi log p)/xi
+ *         --> mu + sigma*log p  as  xi --> 0
+ * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi
+ *        = mu + sigma*expm1(-xi log1p(-p))/xi
+ *        --> mu + sigma*log1p(-p)  as  xi --> 0
+ */
+
+/**
+ * Compute the CDF of the GeneralizedPareto(mu, sigma, xi)
+ * distribution.  Well-conditioned everywhere.  For standard
+ * distribution (mu=0, sigma=1), condition number
+ *
+ *      (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1)
+ *
+ * is bounded by 1, attained only at x = 0.
+ */
+STATIC double
+cdf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  /*
+   * log(1 + xi x_0)/xi
+   * = (-1/xi) \sum_{n=1}^\infty (-xi x_0)^n/n
+   * = (-1/xi) (-xi x_0 + \sum_{n=2}^\infty (-xi x_0)^n/n)
+   * = x_0 - (1/xi) \sum_{n=2}^\infty (-xi x_0)^n/n
+   * = x_0 - x_0 \sum_{n=2}^\infty (-xi x_0)^{n-1}/n
+   * = x_0 (1 - d),
+   *
+   * where d = \sum_{n=2}^\infty (-xi x_0)^{n-1}/n.  If |xi| <
+   * eps/4|x_0|, then
+   *
+   * |d| <= \sum_{n=2}^\infty (eps/4)^{n-1}/n
+   *     <= \sum_{n=2}^\infty (eps/4)^{n-1}
+   *      = \sum_{n=1}^\infty (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^\infty (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2
+   *
+   * for any 0 < eps < 2.  Thus, the relative error of x_0 from
+   * log(1 + xi x_0)/xi is bounded by eps.
+   */
+  if (fabs(xi) < 1e-17/x_0)
+    return -expm1(-x_0);
+  else
+    return -expm1(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution.
+ * For standard distribution (mu=0, sigma=1), ill-conditioned for xi
+ * near 0; condition number
+ *
+ *      -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi}
+ *      = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi}
+ *      = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi}
+ *      = -x/(1 + x xi)
+ *
+ * is bounded by 1/xi.
+ */
+STATIC double
+sf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  if (fabs(xi) < 1e-17/x_0)
+    return exp(-x_0);
+  else
+    return exp(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; condition number is
+ *
+ *      xi (p/(1 - p))/(1 - (1 - p)^xi)
+ */
+STATIC double
+icdf_genpareto(double p, double mu, double sigma, double xi)
+{
+  /*
+   * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi
+   * for xi near zero (note f(xi) --> -log U as xi --> 0), write
+   * the absolutely convergent Taylor expansion
+   *
+   * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^\infty (-xi log U)^n/n!
+   *       = -log U + (1/xi)*\sum_{n=2}^\infty (-xi log U)^n/n!
+   *       = -log U + \sum_{n=2}^\infty xi^{n-1} (-log U)^n/n!
+   *       = -log U - log U \sum_{n=2}^\infty (-xi log U)^{n-1}/n!
+   *       = -log U (1 + \sum_{n=2}^\infty (-xi log U)^{n-1}/n!).
+   *
+   * Let d = \sum_{n=2}^\infty (-xi log U)^{n-1}/n!.  What do we
+   * lose if we discard it and use -log U as an approximation to
+   * f(xi)?  If |xi| < eps/-4log U, then
+   *
+   * |d| <= \sum_{n=2}^\infty |xi log U|^{n-1}/n!
+   *     <= \sum_{n=2}^\infty (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^\infty (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^\infty (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/-2log U,
+   * f(xi) = -log U (1 + d) for |d| <= eps/2.  |d| is the
+   * relative error of f(xi) from -log U; from this bound, the
+   * relative error of -log U from f(xi) is at most (eps/2)/(1 -
+   * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 <
+   * eps < 1.  Since -log U < 1000 for all U in (0, 1] in
+   * binary64 floating-point, we can safely cut xi off at 1e-20 <
+   * eps/4000 and attain <1ulp error from series truncation.
+   */
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log1p(-p);
+  else
+    return mu + sigma*expm1(-xi*log1p(-p))/xi;
+}
+
+/**
+ * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; conditon number is
+ *
+ *      -xi/(1 - p^{-xi})
+ */
+STATIC double
+isf_genpareto(double p, double mu, double sigma, double xi)
+{
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log(p);
+  else
+    return mu + sigma*expm1(-xi*log(p))/xi;
+}
+
+/*******************************************************************/
+
+/**
+ * Deterministic samplers, parametrized by uniform integer and (0,1]
+ * samples.  No guarantees are made about _which_ mapping from the
+ * integer and (0,1] samples these use; all that is guaranteed is the
+ * distribution of the outputs conditioned on a uniform distribution on
+ * the inputs.  The automatic tests in test_prob_distr.c double-check
+ * the particular mappings we use.
+ *
+ * Beware: Unlike random_uniform_01(), these are not guaranteed to be
+ * supported on all possible outputs.  See Ilya Mironov, `On the
+ * Significance of the Least Significant Bits for Differential
+ * Privacy', for an example of what can go wrong if you try to use
+ * these to conceal information from an adversary but you expose the
+ * specific full-precision floating-point values.
+ *
+ * Note: None of these samplers use rejection sampling; they are all
+ * essentially inverse-CDF transforms with tweaks.  If you were to add,
+ * say, a Gamma sampler with the Marsaglia-Tsang method, you would have
+ * to parametrize it by a potentially infinite stream of uniform (and
+ * perhaps normal) samples rather than a fixed number, which doesn't
+ * make for quite as nice automatic testing as for these.
+ */
+
+/**
+ * Deterministically sample from the interval [a, b], indexed by a
+ * uniform random floating-point number p0 in (0, 1].
+ *
+ * Note that even if p0 is nonzero, the result may be equal to a, if
+ * ulp(a)/2 is nonnegligible, e.g. if a = 1.  For maximum resolution,
+ * arrange |a| <= |b|.
+ */
+STATIC double
+sample_uniform_interval(double p0, double a, double b)
+{
+  /*
+   * XXX Prove that the distribution is, in fact, uniform on
+   * [a,b], particularly around p0 = 1, or at least has very
+   * small deviation from uniform, quantified appropriately
+   * (e.g., like in Monahan 1984, or by KL divergence).  It
+   * almost certainly does but it would be nice to quantify the
+   * error.
+   */
+  if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) {
+    /*
+     * When ab < 0, (1 - t) a + t b is monotonic, since for
+     * a <= b it is a sum of nondecreasing functions of t,
+     * and for b <= a, of nonincreasing functions of t.
+     * Further, clearly at 0 and 1 it attains a and b,
+     * respectively.  Hence it is bounded within [a, b].
+     */
+    return (1 - p0)*a + p0*b;
+  } else {
+    /*
+     * a + (b - a) t is monotonic -- it is obviously a
+     * nondecreasing function of t for a <= b.  Further, it
+     * attains a at 0, and while it may overshoot b at 1,
+     * we have a
+     *
+     * Theorem.  If 0 <= t < 1, then the floating-point
+     * evaluation of a + (b - a) t is bounded in [a, b].
+     *
+     * Lemma 1.  If 0 <= t < 1 is a floating-point number,
+     * then for any normal floating-point number x except
+     * the smallest in magnitude, |round(x*t)| < |x|.
+     *
+     * Proof.  WLOG, assume x >= 0.  Since the rounding
+     * function and t |---> x*t are nondecreasing, their
+     * composition t |---> round(x*t) is also
+     * nondecreasing, so it suffices to consider the
+     * largest floating-point number below 1, in particular
+     * t = 1 - ulp(1)/2.
+     *
+     * Case I: If x is a power of two, then the next
+     * floating-point number below x is x - ulp(x)/2 = x -
+     * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t
+     * is a floating-point number, multiplication is exact,
+     * and thus round(x*t) = x*t < x.
+     *
+     * Case II: If x is not a power of two, then the
+     * greatest lower bound of real numbers rounded to x is
+     * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2,
+     * where T(X) is the largest power of two below x.
+     * Anything below this bound is rounded to a
+     * floating-point number smaller than x, and x*t = x*(1
+     * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2
+     * since T(x) < x, so round(x*t) < x*t < x.  QED.
+     *
+     * Lemma 2.  If x and y are subnormal, then round(x +
+     * y) = x + y.
+     *
+     * Proof.  It is a matter of adding the significands,
+     * since if we treat subnormals as having an implicit
+     * zero bit before the `binary' point, their exponents
+     * are all the same.  There is at most one carry/borrow
+     * bit, which can always be acommodated either in a
+     * subnormal, or, at largest, in the implicit one bit
+     * of a normal.
+     *
+     * Lemma 3.  Let x and y be floating-point numbers.  If
+     * round(x - y) is subnormal or zero, then it is equal
+     * to x - y.
+     *
+     * Proof.  Case I (equal): round(x - y) = 0 iff x = y;
+     * hence if round(x - y) = 0, then round(x - y) = 0 = x
+     * - y.
+     *
+     * Case II (subnormal/subnormal): If x and y are both
+     * subnormal, this follows directly from Lemma 2.
+     *
+     * Case IIIa (normal/subnormal): If x is normal and y
+     * is subnormal, then x and y must share sign, or else
+     * x - y would be larger than x and thus rounded to
+     * normal.  If s is the smallest normal positive
+     * floating-point number, |x| < 2s since by
+     * construction 2s - |y| is normal for all subnormal y.
+     * This means that x and y must have the same exponent,
+     * so the difference is the difference of significands,
+     * which is exact.
+     *
+     * Case IIIb (subnormal/normal): Same as case IIIa for
+     * -(y - x).
+     *
+     * Case IV (normal/normal): If x and y are both normal,
+     * then they must share sign, or else x - y would be
+     * larger than x and thus rounded to normal.  Note that
+     * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <=
+     * -|x| but -|x| is normal like x.  Also, |x|/2 < |y|:
+     * if |x|/2 is subnormal, it must hold because y is
+     * normal; if |x|/2 is normal, then |x|/2 >= s, so
+     * since |x| - |y| < s,
+     *
+     *  |x|/2 = |x| - |x|/2 <= |x| - s <= |y|;
+     *
+     * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz
+     * lemma, round(x - y) = x - y.  QED.
+     *
+     * Proof of theorem.  WLOG, assume 0 <= a <= b.  Since
+     * round(a + round(round(b - a)*t) is nondecreasing in
+     * t and attains a at 0, the lower end of the bound is
+     * trivial; we must show the upper end of the bound
+     * strictly.  It suffices to show this for the largest
+     * floating-point number below 1, namely 1 - ulp(1)/2.
+     *
+     * Case I: round(b - a) is normal.  Then it is at most
+     * the smallest floating-point number above b - a.  By
+     * Lemma 1, round(round(b - a)*t) < round(b - a).
+     * Since the inequality is strict, and since
+     * round(round(b - a)*t) is a floating-point number
+     * below round(b - a), and since there are no
+     * floating-point numbers between b - a and round(b -
+     * a), we must have round(round(b - a)*t) < b - a.
+     * Then since y |---> round(a + y) is nondecreasing, we
+     * must have
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b) = b.
+     *
+     * Case II: round(b - a) is subnormal.  In this case,
+     * Lemma 1 falls apart -- we are not guaranteed the
+     * strict inequality.  However, by Lemma 3, the
+     * difference is exact: round(b - a) = b - a.  Thus,
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + round((b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b)
+     *   = b,
+     *
+     * QED.
+     */
+
+    /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal.  */
+    if (p0 >= 1)
+      return b;
+    return a + (b - a)*p0;
+  }
+}
+
+/**
+ * Deterministically sample from the standard logistic distribution,
+ * indexed by a uniform random 32-bit integer s and uniform random
+ * floating-point numbers t and p0 in (0, 1].
+ */
+STATIC double
+sample_logistic(uint32_t s, double t, double p0)
+{
+  double sign = (s & 1) ? -1 : +1;
+  double r;
+
+  /*
+   * We carve up the interval (0, 1) into subregions to compute
+   * the inverse CDF precisely:
+   *
+   * A = (0, 1/(1 + e)] ---> (-\infty, -1]
+   * B = [1/(1 + e), 1/2] ---> [-1, 0]
+   * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1]
+   * D = [1 - 1/(1 + e), 1) ---> [1, +\infty)
+   *
+   * Cases D and C are mirror images of cases A and B,
+   * respectively, so we choose between them by the sign chosen
+   * by a fair coin toss.  We choose between cases A and B by a
+   * coin toss weighted by
+   *
+   *    2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2):
+   *
+   * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)]
+   * sample p; if it comes up tails, scale p0 into a uniform (0,
+   * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p =
+   * 1/2 - p0.
+   */
+  if (t <= 2/(1 + exp(1))) {
+    /* p uniform in (0, 1/(1 + e)], represented by p.  */
+    p0 /= 1 + exp(1);
+    r = logit(p0);
+  } else {
+    /*
+     * p uniform in [1/(1 + e), 1/2), actually represented
+     * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so
+     * that p = 1/2 - p.
+     */
+    p0 *= 0.5 - 1/(1 + exp(1));
+    r = logithalf(p0);
+  }
+
+  /*
+   * We have chosen from the negative half of the standard
+   * logistic distribution, which is symmetric with the positive
+   * half.  Now use the sign to choose uniformly between them.
+   */
+  return sign*r;
+}
+
+/**
+ * Deterministically sample from the logistic distribution scaled by
+ * sigma and translated by mu.
+ */
+static double
+sample_logistic_locscale(uint32_t s, double t, double p0, double mu,
+    double sigma)
+{
+
+  return mu + sigma*sample_logistic(s, t, p0);
+}
+
+/**
+ * Deterministically sample from the standard log-logistic
+ * distribution, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_log_logistic(uint32_t s, double p0)
+{
+
+  /*
+   * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the
+   * condition numbers of the icdf and the isf coincide at 1/2.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0) {
+    /* p = p0 in (0, 1/2] */
+    return p0/(1 - p0);
+  } else {
+    /* p = 1 - p0 in [1/2, 1) */
+    return (1 - p0)/p0;
+  }
+}
+
+/**
+ * Deterministically sample from the log-logistic distribution with
+ * scale alpha and shape beta.
+ */
+static double
+sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha,
+    double beta)
+{
+  double x = sample_log_logistic(s, p0);
+
+  return alpha*pow(x, 1/beta);
+}
+
+/**
+ * Deterministically sample from the standard exponential distribution,
+ * indexed by a uniform random 32-bit integer s and a uniform random
+ * floating-point number p0 in (0, 1].
+ */
+static double
+sample_exponential(uint32_t s, double p0)
+{
+  /*
+   * We would like to evaluate log(p) for p near 0, and log1p(-p)
+   * for p near 1.  Simply carve the interval into (0, 1/2] and
+   * [1/2, 1) by a fair coin toss.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0)
+    /* p = p0 in (0, 1/2] */
+    return -log(p0);
+  else
+    /* p = 1 - p0 in [1/2, 1) */
+    return -log1p(-p0);
+}
+
+/**
+ * Deterministically sample from a Weibull distribution with scale
+ * lambda and shape k -- just an exponential with a shape parameter in
+ * addition to a scale parameter.  (Yes, lambda really is the scale,
+ * _not_ the rate.)
+ */
+STATIC double
+sample_weibull(uint32_t s, double p0, double lambda, double k)
+{
+
+  return lambda*pow(sample_exponential(s, p0), 1/k);
+}
+
+/**
+ * Deterministically sample from the generalized Pareto distribution
+ * with shape xi, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_genpareto(uint32_t s, double p0, double xi)
+{
+  double x = sample_exponential(s, p0);
+
+  /*
+   * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the
+   * absolutely convergent Taylor series
+   *
+   * f(x) = (1/xi) (xi x + \sum_{n=2}^\infty (xi x)^n/n!)
+   *      = x + (1/xi) \sum_{n=2}^\inty (xi x)^n/n!
+   *      = x + \sum_{n=2}^\infty xi^{n-1} x^n/n!
+   *      = x + x \sum_{n=2}^\infty (xi x)^{n-1}/n!
+   *      = x (1 + \sum_{n=2}^\infty (xi x)^{n-1}/n!).
+   *
+   * d = \sum_{n=2}^\infty (xi x)^{n-1}/n! is the relative error
+   * of f(x) from x.  If |xi| < eps/4x, then
+   *
+   * |d| <= \sum_{n=2}^\infty |xi x|^{n-1}/n!
+   *     <= \sum_{n=2}^\infty (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^\infty (eps/4)
+   *      = (eps/4) \sum_{n=0}^\infty (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/2x, f(xi)
+   * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'|
+   * <= eps.  What bound should we use for x?
+   *
+   * - If x is exponentially distributed, x > 200 with
+   *   probability below e^{-200} << 2^{-256}, i.e. never.
+   *
+   * - If x is computed by -log(U) for U in (0, 1], x is
+   *   guaranteed to be below 1000 in IEEE 754 binary64
+   *   floating-point.
+   *
+   * We can safely cut xi off at 1e-20 < eps/4000 and attain an
+   * error bounded by 0.5 ulp for this expression.
+   */
+  return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi);
+}
+
+/**
+ * Deterministically sample from a generalized Pareto distribution with
+ * shape xi, scaled by sigma and translated by mu.
+ */
+static double
+sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma,
+    double xi)
+{
+
+  return mu + sigma*sample_genpareto(s, p0, xi);
+}
+
+/**
+ * Deterministically sample from the geometric distribution with
+ * per-trial success probability p.
+ *
+ * XXX Quantify the error (KL divergence?) of this
+ * ceiling-of-exponential sampler from a true geometric distribution,
+ * which we could get by rejection sampling.  Relevant papers:
+ *
+ *      John F. Monahan, `Accuracy in Random Number Generation',
+ *      Mathematics of Computation 45(172), October 1984, pp. 559--568.
+*https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf
+ *
+ *      Karl Bringmann and Tobias Friedrich, `Exact and Efficient
+ *      Generation of Geometric Random Variates and Random Graphs', in
+ *      Proceedings of the 40th International Colloaquium on Automata,
+ *      Languages, and Programming -- ICALP 2013, Springer LNCS 7965,
+ *      pp.267--278.
+ *      https://doi.org/10.1007/978-3-642-39206-1_23
+ *      https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf
+ */
+static double
+sample_geometric(uint32_t s, double p0, double p)
+{
+  double x = sample_exponential(s, p0);
+
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1)
+    return 1;
+
+  return ceil(-x/log1p(-p));
+}
+
+/*******************************************************************/
+
+/** Public API for probability distributions:
+ *
+ *  These are wrapper functions on top of the various probability distribution
+ *  operations using the generic <b>dist</b> structure.
+
+ *  These are the functions that should be used by consumers of this API.
+ */
+
+/** Returns the name of the distribution in <b>dist</b>. */
+const char *
+dist_name(const struct dist *dist)
+{
+  return dist->ops->name;
+}
+
+/* Sample a value from <b>dist</b> and return it. */
+double
+dist_sample(const struct dist *dist)
+{
+  return dist->ops->sample(dist);
+}
+
+/** Compute the CDF of <b>dist</b> at <b>x</b>. */
+double
+dist_cdf(const struct dist *dist, double x)
+{
+  return dist->ops->cdf(dist, x);
+}
+
+/** Compute the SF (Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_sf(const struct dist *dist, double x)
+{
+  return dist->ops->sf(dist, x);
+}
+
+/** Compute the iCDF (Inverse CDF) of <b>dist</b> at <b>x</b>. */
+double
+dist_icdf(const struct dist *dist, double p)
+{
+  return dist->ops->icdf(dist, p);
+}
+
+/** Compute the iSF (Inverse Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_isf(const struct dist *dist, double p)
+{
+  return dist->ops->isf(dist, p);
+}
+
+/** Functions for uniform distribution */
+
+static double
+uniform_sample(const struct dist *dist)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  double p0 = random_uniform_01();
+
+  return sample_uniform_interval(p0, U->a, U->b);
+}
+
+static double
+uniform_cdf(const struct dist *dist, double x)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  if (x < U->a)
+    return 0;
+  else if (x < U->b)
+    return (x - U->a)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_sf(const struct dist *dist, double x)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+
+  if (x > U->b)
+    return 0;
+  else if (x > U->a)
+    return (U->b - x)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_icdf(const struct dist *dist, double p)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p)));
+}
+
+static double
+uniform_isf(const struct dist *dist, double p)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p)));
+}
+
+const struct dist_ops uniform_ops = {
+  .name = "uniform",
+  .sample = uniform_sample,
+  .cdf = uniform_cdf,
+  .sf = uniform_sf,
+  .icdf = uniform_icdf,
+  .isf = uniform_isf,
+};
+
+/*******************************************************************/
+
+/** Private functions for each probability distribution. */
+
+/** Functions for logistic distribution: */
+
+static double
+logistic_sample(const struct dist *dist)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double t = random_uniform_01();
+  double p0 = random_uniform_01();
+
+  return sample_logistic_locscale(s, t, p0, L->mu, L->sigma);
+}
+
+static double
+logistic_cdf(const struct dist *dist, double x)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return cdf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_sf(const struct dist *dist, double x)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return sf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_icdf(const struct dist *dist, double p)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return icdf_logistic(p, L->mu, L->sigma);
+}
+
+static double
+logistic_isf(const struct dist *dist, double p)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return isf_logistic(p, L->mu, L->sigma);
+}
+
+const struct dist_ops logistic_ops = {
+  .name = "logistic",
+  .sample = logistic_sample,
+  .cdf = logistic_cdf,
+  .sf = logistic_sf,
+  .icdf = logistic_icdf,
+  .isf = logistic_isf,
+};
+
+/** Functions for log-logistic distribution: */
+
+static double
+log_logistic_sample(const struct dist *dist)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_cdf(const struct dist *dist, double x)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return cdf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_sf(const struct dist *dist, double x)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return sf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_icdf(const struct dist *dist, double p)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return icdf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_isf(const struct dist *dist, double p)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return isf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+const struct dist_ops log_logistic_ops = {
+  .name = "log logistic",
+  .sample = log_logistic_sample,
+  .cdf = log_logistic_cdf,
+  .sf = log_logistic_sf,
+  .icdf = log_logistic_icdf,
+  .isf = log_logistic_isf,
+};
+
+/** Functions for Weibull distribution */
+
+static double
+weibull_sample(const struct dist *dist)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_weibull(s, p0, W->lambda, W->k);
+}
+
+static double
+weibull_cdf(const struct dist *dist, double x)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return cdf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_sf(const struct dist *dist, double x)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return sf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_icdf(const struct dist *dist, double p)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return icdf_weibull(p, W->lambda, W->k);
+}
+
+static double
+weibull_isf(const struct dist *dist, double p)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return isf_weibull(p, W->lambda, W->k);
+}
+
+const struct dist_ops weibull_ops = {
+  .name = "Weibull",
+  .sample = weibull_sample,
+  .cdf = weibull_cdf,
+  .sf = weibull_sf,
+  .icdf = weibull_icdf,
+  .isf = weibull_isf,
+};
+
+/** Functions for generalized Pareto distributions */
+
+static double
+genpareto_sample(const struct dist *dist)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_cdf(const struct dist *dist, double x)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_sf(const struct dist *dist, double x)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return sf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_icdf(const struct dist *dist, double p)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_isf(const struct dist *dist, double p)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return isf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+const struct dist_ops genpareto_ops = {
+  .name = "generalized Pareto",
+  .sample = genpareto_sample,
+  .cdf = genpareto_cdf,
+  .sf = genpareto_sf,
+  .icdf = genpareto_icdf,
+  .isf = genpareto_isf,
+};
+
+/** Functions for geometric distribution on number of trials before success */
+
+static double
+geometric_sample(const struct dist *dist)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_geometric(s, p0, G->p);
+}
+
+static double
+geometric_cdf(const struct dist *dist, double x)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */
+  return -expm1(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_sf(const struct dist *dist, double x)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */
+  return exp(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_icdf(const struct dist *dist, double p)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  return log1p(-p)/log1p(-G->p);
+}
+
+static double
+geometric_isf(const struct dist *dist, double p)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  return log(p)/log1p(-G->p);
+}
+
+const struct dist_ops geometric_ops = {
+  .name = "geometric (1-based)",
+  .sample = geometric_sample,
+  .cdf = geometric_cdf,
+  .sf = geometric_sf,
+  .icdf = geometric_icdf,
+  .isf = geometric_isf,
+};
diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h
new file mode 100644
index 0000000000..7254dc8623
--- /dev/null
+++ b/src/lib/math/prob_distr.h
@@ -0,0 +1,253 @@
+
+/**
+ * \file prob_distr.h
+ *
+ * \brief Header for prob_distr.c
+ **/
+
+#ifndef TOR_PROB_DISTR_H
+#define TOR_PROB_DISTR_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * Container for distribution parameters for sampling, CDF, &c.
+ */
+struct dist {
+  const struct dist_ops *ops;
+};
+
+/**
+ * Untyped initializer element for struct dist using the specified
+ * struct dist_ops pointer.  Don't actually use this directly -- use
+ * the type-specific macro built out of DIST_BASE_TYPED below -- but if
+ * you did use this directly, it would be something like:
+ *
+ *   struct weibull mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     .lambda = ...,
+ *     .k = ...,
+ *   };
+ *
+ * Note there is NO COMPILER FEEDBACK if you accidentally do something
+ * like
+ *
+ *   struct geometric mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     ...
+ *   };
+ */
+#define DIST_BASE(OPS)  { .ops = (OPS) }
+
+/** A compile-time type-checking macro for use with DIST_BASE_TYPED.
+ *
+ *  This macro works by checking that &OBJ is a pointer type that is the same
+ *  type (except for qualifiers) as (const TYPE *)&OBJ. It's a C constraint
+ *  violation (which requires a diagnostic) if two pointers are different types
+ *  and are subtracted. The sizeof() forces compile-time evaluation, and the
+ *  multiplication by zero is to discard the result of the sizeof() from the
+ *  expression.
+ *
+ *  We define this conditionally to suppress false positives from
+ *  Coverity, which gets confused by the sizeof business.
+ */
+#ifdef __COVERITY__
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0
+#else
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \
+  (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ)))
+#endif /* defined(__COVERITY__) */
+
+/**
+* Typed initializer element for struct dist using the specified struct
+* dist_ops pointer.  Don't actually use this directly -- use a
+* type-specific macro built out of it -- but if you did use this
+* directly, it would be something like:
+*
+*   struct weibull mydist = {
+*     DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull),
+*     .lambda = ...,
+*     .k = ...,
+*   };
+*
+* If you want to define a distribution type, define a canonical set of
+* operations and define a type-specific initializer element like so:
+*
+*   struct foo {
+*     struct dist base;
+*     int omega;
+*     double tau;
+*     double phi;
+*   };
+*
+*   struct dist_ops foo_ops = ...;
+*
+*   #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo)
+*
+* Then users can do:
+*
+*   struct foo mydist = {
+*     FOO(mydist),
+*     .omega = ...,
+*     .tau = ...,
+*     .phi = ...,
+*   };
+*
+* If you accidentally write
+*
+*   struct bar mydist = {
+*     FOO(mydist),
+*     ...
+*   };
+*
+* then the compiler will report a type mismatch in the sizeof
+* expression, which otherwise evaporates at runtime.
+*/
+#define DIST_BASE_TYPED(OPS, OBJ, TYPE)                         \
+  DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE))
+
+/**
+ * Generic operations on distributions.  These simply defer to the
+ * corresponding dist_ops function.  In the parlance of C++, these call
+ * virtual member functions.
+ */
+const char *dist_name(const struct dist *);
+double dist_sample(const struct dist *);
+double dist_cdf(const struct dist *, double x);
+double dist_sf(const struct dist *, double x);
+double dist_icdf(const struct dist *, double p);
+double dist_isf(const struct dist *, double p);
+
+/**
+ * Set of operations on a potentially parametric family of
+ * distributions.  In the parlance of C++, this would be called a
+ * `vtable' and the members are virtual member functions.
+ */
+struct dist_ops {
+  const char *name;
+  double (*sample)(const struct dist *);
+  double (*cdf)(const struct dist *, double x);
+  double (*sf)(const struct dist *, double x);
+  double (*icdf)(const struct dist *, double p);
+  double (*isf)(const struct dist *, double p);
+};
+
+/* Geometric distribution on positive number of trials before first success */
+
+struct geometric {
+  struct dist base;
+  double p; /* success probability */
+};
+
+extern const struct dist_ops geometric_ops;
+
+#define GEOMETRIC(OBJ)                                      \
+  DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric)
+
+/* Pareto distribution */
+
+struct genpareto {
+  struct dist base;
+  double mu;
+  double sigma;
+  double xi;
+};
+
+extern const struct dist_ops genpareto_ops;
+
+#define GENPARETO(OBJ)                                      \
+  DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto)
+
+/* Weibull distribution */
+
+struct weibull {
+  struct dist base;
+  double lambda;
+  double k;
+};
+
+extern const struct dist_ops weibull_ops;
+
+#define WEIBULL(OBJ)                                    \
+  DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull)
+
+/* Log-logistic distribution */
+
+struct log_logistic {
+  struct dist base;
+  double alpha;
+  double beta;
+};
+
+extern const struct dist_ops log_logistic_ops;
+
+#define LOG_LOGISTIC(OBJ)                                       \
+  DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic)
+
+/* Logistic distribution */
+
+struct logistic {
+  struct dist base;
+  double mu;
+  double sigma;
+};
+
+extern const struct dist_ops logistic_ops;
+
+#define LOGISTIC(OBJ)                                   \
+  DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic)
+
+/* Uniform distribution */
+
+struct uniform {
+  struct dist base;
+  double a;
+  double b;
+};
+
+extern const struct dist_ops uniform_ops;
+
+#define UNIFORM(OBJ)                                    \
+  DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform)
+
+/** Only by unittests */
+
+#ifdef PROB_DISTR_PRIVATE
+
+STATIC double logithalf(double p0);
+STATIC double logit(double p);
+
+STATIC double random_uniform_01(void);
+
+STATIC double logistic(double x);
+STATIC double cdf_logistic(double x, double mu, double sigma);
+STATIC double sf_logistic(double x, double mu, double sigma);
+STATIC double icdf_logistic(double p, double mu, double sigma);
+STATIC double isf_logistic(double p, double mu, double sigma);
+STATIC double sample_logistic(uint32_t s, double t, double p0);
+
+STATIC double cdf_log_logistic(double x, double alpha, double beta);
+STATIC double sf_log_logistic(double x, double alpha, double beta);
+STATIC double icdf_log_logistic(double p, double alpha, double beta);
+STATIC double isf_log_logistic(double p, double alpha, double beta);
+STATIC double sample_log_logistic(uint32_t s, double p0);
+
+STATIC double cdf_weibull(double x, double lambda, double k);
+STATIC double sf_weibull(double x, double lambda, double k);
+STATIC double icdf_weibull(double p, double lambda, double k);
+STATIC double isf_weibull(double p, double lambda, double k);
+STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k);
+
+STATIC double sample_uniform_interval(double p0, double a, double b);
+
+STATIC double cdf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double sf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double icdf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double isf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double sample_genpareto(uint32_t s, double p0, double xi);
+
+#endif /* defined(PROB_DISTR_PRIVATE) */
+
+#endif /* !defined(TOR_PROB_DISTR_H) */
diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include
index 814652a93c..a1edaf2231 100644
--- a/src/lib/memarea/.may_include
+++ b/src/lib/memarea/.may_include
@@ -1,7 +1,7 @@
 orconfig.h
 lib/arch/*.h
 lib/cc/*.h
-lib/container/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/memarea/*.h
+lib/smartlist_core/*.h
\ No newline at end of file
diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am
index 94343dcead..83fb99ec73 100644
--- a/src/lib/memarea/include.am
+++ b/src/lib/memarea/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-memarea-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_memarea_a_SOURCES =			\
 	src/lib/memarea/memarea.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_memarea_testing_a_SOURCES = \
 src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/memarea/memarea.h
diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c
index 486673116c..f3bb79a1e2 100644
--- a/src/lib/memarea/memarea.c
+++ b/src/lib/memarea/memarea.c
@@ -16,7 +16,8 @@
 
 #include "lib/arch/bytes.h"
 #include "lib/cc/torint.h"
-#include "lib/container/smartlist.h"
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
@@ -67,7 +68,7 @@
   uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]);     \
   tor_assert(sent_val == SENTINEL_VAL);                                 \
   STMT_END
-#else /* !(defined(USE_SENTINELS)) */
+#else /* !defined(USE_SENTINELS) */
 #define SENTINEL_LEN 0
 #define SET_SENTINEL(chunk) STMT_NIL
 #define CHECK_SENTINEL(chunk) STMT_NIL
@@ -314,7 +315,7 @@ memarea_assert_ok(memarea_t *area)
   }
 }
 
-#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */
+#else /* defined(DISABLE_MEMORY_SENTINELS) */
 
 struct memarea_t {
   smartlist_t *pieces;
diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am
index d1fdde6313..12c1bff72d 100644
--- a/src/lib/meminfo/include.am
+++ b/src/lib/meminfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_meminfo_a_SOURCES =			\
 	src/lib/meminfo/meminfo.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_meminfo_testing_a_SOURCES = \
 src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/meminfo/meminfo.h
diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c
index f4fa45167e..bff71c2f05 100644
--- a/src/lib/meminfo/meminfo.c
+++ b/src/lib/meminfo/meminfo.c
@@ -54,7 +54,7 @@ tor_log_mallinfo(int severity)
       mi.arena, mi.ordblks, mi.smblks, mi.hblks,
       mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks,
       mi.keepcost);
-#else /* !(defined(HAVE_MALLINFO)) */
+#else /* !defined(HAVE_MALLINFO) */
   (void)severity;
 #endif /* defined(HAVE_MALLINFO) */
 }
diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h
index 2d64e1ab06..9580640f4d 100644
--- a/src/lib/meminfo/meminfo.h
+++ b/src/lib/meminfo/meminfo.h
@@ -18,4 +18,4 @@
 void tor_log_mallinfo(int severity);
 MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
 
-#endif
+#endif /* !defined(TOR_MEMINFO_H) */
diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include
index 13b209bbed..e4368f799b 100644
--- a/src/lib/net/.may_include
+++ b/src/lib/net/.may_include
@@ -1,8 +1,9 @@
 orconfig.h
-siphash.h
-ht.h
+ext/siphash.h
+ext/ht.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
@@ -11,5 +12,6 @@ lib/lock/*.h
 lib/log/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/malloc/*.h
\ No newline at end of file
diff --git a/src/lib/net/address.c b/src/lib/net/address.c
index 076ca3eb34..41017935e1 100644
--- a/src/lib/net/address.c
+++ b/src/lib/net/address.c
@@ -40,6 +40,7 @@
 
 #include "lib/net/address.h"
 #include "lib/net/socket.h"
+#include "lib/cc/ctassert.h"
 #include "lib/container/smartlist.h"
 #include "lib/ctime/di_ops.h"
 #include "lib/log/log.h"
@@ -52,7 +53,7 @@
 #include "lib/string/printf.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
@@ -98,6 +99,7 @@
 #if AF_UNSPEC != 0
 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please!
 #endif
+CTASSERT(AF_UNSPEC == 0);
 
 /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a
  * sockaddr object in *<b>sa_out</b> of object size <b>len</b>.  If not enough
@@ -337,7 +339,7 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
       break;
     case AF_INET6:
       /* Shortest addr [ :: ] + \0 */
-      if (len < (3 + (decorate ? 2 : 0)))
+      if (len < (3u + (decorate ? 2 : 0)))
         return NULL;
 
       if (decorate)
@@ -371,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
  *
  * If <b>accept_regular</b> is set and the address is in neither recognized
  * reverse lookup hostname format, try parsing the address as a regular
- * IPv4 or IPv6 address too.
+ * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or
+ * without square brackets.
  */
 int
 tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
@@ -1186,37 +1189,75 @@ fmt_addr32(uint32_t addr)
 }
 
 /** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
- * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
- * square brackets.
+ * may be an IPv4 address, or an IPv6 address surrounded by square brackets.
  *
- *  Return an address family on success, or -1 if an invalid address string is
- *  provided. */
-int
-tor_addr_parse(tor_addr_t *addr, const char *src)
+ * If <b>allow_ipv6_without_brackets</b> is true, also allow IPv6 addresses
+ * without brackets.
+ *
+ * Always rejects IPv4 addresses with brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+static int
+tor_addr_parse_impl(tor_addr_t *addr, const char *src,
+                    bool allow_ipv6_without_brackets)
 {
   /* Holds substring of IPv6 address after removing square brackets */
   char *tmp = NULL;
-  int result;
+  int result = -1;
   struct in_addr in_tmp;
   struct in6_addr in6_tmp;
+  int brackets_detected = 0;
+
   tor_assert(addr && src);
-  if (src[0] == '[' && src[1])
+
+  size_t len = strlen(src);
+
+  if (len && src[0] == '[' && src[len - 1] == ']') {
+    brackets_detected = 1;
     src = tmp = tor_strndup(src+1, strlen(src)-2);
+  }
 
-  if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
-    result = AF_INET6;
-    tor_addr_from_in6(addr, &in6_tmp);
-  } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
-    result = AF_INET;
-    tor_addr_from_in(addr, &in_tmp);
-  } else {
-    result = -1;
+  /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses
+   * without brackets are allowed */
+  if (brackets_detected || allow_ipv6_without_brackets) {
+    if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
+      result = AF_INET6;
+      tor_addr_from_in6(addr, &in6_tmp);
+    }
+  }
+
+  /* Try to parse an IPv4 address without brackets */
+  if (!brackets_detected) {
+    if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
+      result = AF_INET;
+      tor_addr_from_in(addr, &in_tmp);
+    }
+  }
+
+  /* Clear the address on error, to avoid returning uninitialised or partly
+   * parsed data.
+   */
+  if (result == -1) {
+    memset(addr, 0, sizeof(tor_addr_t));
   }
 
   tor_free(tmp);
   return result;
 }
 
+/** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
+ * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
+ * square brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+int
+tor_addr_parse(tor_addr_t *addr, const char *src)
+{
+  return tor_addr_parse_impl(addr, src, 1);
+}
+
 #ifdef HAVE_IFADDRS_TO_SMARTLIST
 /*
  * Convert a linked list consisting of <b>ifaddrs</b> structures
@@ -1709,6 +1750,11 @@ get_interface_address6_list,(int severity,
  * form "ip" or "ip:0".  Otherwise, accept those forms, and set
  * *<b>port_out</b> to <b>default_port</b>.
  *
+ * This function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
  * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_parse(int severity, const char *addrport,
@@ -1718,6 +1764,7 @@ tor_addr_port_parse(int severity, const char *addrport,
   int retval = -1;
   int r;
   char *addr_tmp = NULL;
+  bool has_port;
 
   tor_assert(addrport);
   tor_assert(address_out);
@@ -1727,28 +1774,47 @@ tor_addr_port_parse(int severity, const char *addrport,
   if (r < 0)
     goto done;
 
-  if (!*port_out) {
+  has_port = !! *port_out;
+  /* If there's no port, use the default port, or fail if there is no default
+   */
+  if (!has_port) {
     if (default_port >= 0)
       *port_out = default_port;
     else
       goto done;
   }
 
-  /* make sure that address_out is an IP address */
-  if (tor_addr_parse(address_out, addr_tmp) < 0)
+  /* Make sure that address_out is an IP address.
+   * If there is no port in addrport, allow IPv6 addresses without brackets. */
+  if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0)
     goto done;
 
   retval = 0;
 
  done:
+  /* Clear the address and port on error, to avoid returning uninitialised or
+   * partly parsed data.
+   */
+  if (retval == -1) {
+    memset(address_out, 0, sizeof(tor_addr_t));
+    *port_out = 0;
+  }
   tor_free(addr_tmp);
   return retval;
 }
 
 /** Given an address of the form "host[:port]", try to divide it into its host
- * and port portions, setting *<b>address_out</b> to a newly allocated string
- * holding the address portion and *<b>port_out</b> to the port (or 0 if no
- * port is given).  Return 0 on success, -1 on failure. */
+ * and port portions.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Sets *<b>address_out</b> to a newly allocated string holding the address
+ * portion, and *<b>port_out</b> to the port (or 0 if no port is given).
+ *
+ * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_split(int severity, const char *addrport,
                     char **address_out, uint16_t *port_out)
@@ -1757,8 +1823,11 @@ tor_addr_port_split(int severity, const char *addrport,
   tor_assert(addrport);
   tor_assert(address_out);
   tor_assert(port_out);
+
   /* We need to check for IPv6 manually because the logic below doesn't
-   * do a good job on IPv6 addresses that lack a port. */
+   * do a good job on IPv6 addresses that lack a port.
+   * If an IPv6 address without square brackets is ambiguous, it gets parsed
+   * here as an address, rather than address:port. */
   if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) {
     *port_out = 0;
     *address_out = tor_strdup(addrport);
@@ -1798,8 +1867,7 @@ tor_addr_port_split(int severity, const char *addrport,
     tor_free(address_);
   }
 
-  if (port_out)
-    *port_out = ok ? ((uint16_t) port_) : 0;
+  *port_out = ok ? ((uint16_t) port_) : 0;
 
   return ok ? 0 : -1;
 }
@@ -2018,8 +2086,12 @@ string_is_valid_nonrfc_hostname(const char *string)
 
   smartlist_split_string(components,string,".",0,0);
 
-  if (BUG(smartlist_len(components) == 0))
-    return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks.
+  if (BUG(smartlist_len(components) == 0)) {
+    // LCOV_EXCL_START should be impossible given the earlier checks.
+    smartlist_free(components);
+    return 0;
+    // LCOV_EXCL_STOP
+  }
 
   /* Allow a single terminating '.' used rarely to indicate domains
    * are FQDNs rather than relative. */
diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h
index c45f42be81..4d0d0dd57c 100644
--- a/src/lib/net/alertsock.h
+++ b/src/lib/net/alertsock.h
@@ -42,4 +42,4 @@ typedef struct alert_sockets_t {
 int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags);
 void alert_sockets_close(alert_sockets_t *socks);
 
-#endif
+#endif /* !defined(TOR_ALERTSOCK_H) */
diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c
index 3eb0a033d5..cfe1a7dc26 100644
--- a/src/lib/net/buffers_net.c
+++ b/src/lib/net/buffers_net.c
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/net/buffers_net.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/net/nettypes.h"
@@ -22,6 +22,10 @@
 
 #include <stdlib.h>
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 #ifdef PARANOIA
 /** Helper: If PARANOIA is defined, assert that the buffer in local variable
  * <b>buf</b> is well-formed. */
@@ -30,27 +34,36 @@
 #define check() STMT_NIL
 #endif /* defined(PARANOIA) */
 
-/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into
  * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
- * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
- * and the number of bytes read otherwise. */
+ * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b>
+ * is true, otherwise it uses <b>read()</b>.  Return -1 on error (and sets
+ * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read
+ * otherwise. */
 static inline int
 read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
-              int *reached_eof, int *socket_error)
+              int *reached_eof, int *error, bool is_socket)
 {
   ssize_t read_result;
   if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
     at_most = CHUNK_REMAINING_CAPACITY(chunk);
-  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (is_socket)
+    read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+  else
+    read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most);
 
   if (read_result < 0) {
-    int e = tor_socket_errno(fd);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
-        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+        log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?",
+                 is_socket ? "recv" : "read");
 #endif
-      *socket_error = e;
+      if (error)
+        *error = e;
       return -1;
     }
     return 0; /* would block. */
@@ -68,16 +81,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
   }
 }
 
-/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
- * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
- * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
- * error; else return the number of bytes read.
+/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>.  Read
+ * at most <b>at_most</b> bytes, growing the buffer as necessary.  If recv()
+ * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0.
+ * Return -1 on error; else return the number of bytes read.
  */
 /* XXXX indicate "read blocked" somehow? */
-int
-buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
-                     int *reached_eof,
-                     int *socket_error)
+static int
+buf_read_from_fd(buf_t *buf, int fd, size_t at_most,
+                 int *reached_eof,
+                 int *socket_error,
+                 bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes read" are not mutually exclusive.
@@ -87,7 +101,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 
   check();
   tor_assert(reached_eof);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
 
   if (BUG(buf->datalen >= INT_MAX))
     return -1;
@@ -108,7 +122,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
         readlen = cap;
     }
 
-    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    r = read_to_chunk(buf, chunk, fd, readlen,
+                      reached_eof, socket_error, is_socket);
     check();
     if (r < 0)
       return r; /* Error */
@@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 }
 
 /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
- * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, 0 on blocking, -1 on failure.
+ * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>.  On
+ * success, deduct the bytes written from *<b>buf_flushlen</b>.  Return the
+ * number of bytes written on success, 0 on blocking, -1 on failure.
  */
 static inline int
-flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
-            size_t *buf_flushlen)
+flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen, bool is_socket)
 {
   ssize_t write_result;
 
   if (sz > chunk->datalen)
     sz = chunk->datalen;
-  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (is_socket)
+    write_result = tor_socket_send(fd, chunk->data, sz, 0);
+  else
+    write_result = write(fd, chunk->data, sz);
 
   if (write_result < 0) {
-    int e = tor_socket_errno(s);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
@@ -155,15 +175,15 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
   }
 }
 
-/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+/** Write data from <b>buf</b> to the file descriptor <b>fd</b>.  Write at most
  * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
  * the number of bytes actually written, and remove the written bytes
  * from the buffer.  Return the number of bytes written on success,
  * -1 on failure.  Return 0 if write() would block.
  */
-int
-buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
-                    size_t *buf_flushlen)
+static int
+buf_flush_to_fd(buf_t *buf, int fd, size_t sz,
+                size_t *buf_flushlen, bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes flushed" are not mutually exclusive.
@@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
   int r;
   size_t flushed = 0;
   tor_assert(buf_flushlen);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
   if (BUG(*buf_flushlen > buf->datalen)) {
     *buf_flushlen = buf->datalen;
   }
@@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
     else
       flushlen0 = buf->head->datalen;
 
-    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket);
     check();
     if (r < 0)
       return r;
@@ -200,3 +220,55 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
   tor_assert(flushed < INT_MAX);
   return (int)flushed;
 }
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, s, sz, buf_flushlen, true);
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true);
+}
+
+/** Write data from <b>buf</b> to the pipe <b>fd</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_pipe(buf_t *buf, int fd, size_t sz,
+                  size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false);
+}
+
+/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If read() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_pipe(buf_t *buf, int fd, size_t at_most,
+                   int *reached_eof,
+                   int *socket_error)
+{
+  return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false);
+}
diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h
index 5f69bebedf..5058dd0a26 100644
--- a/src/lib/net/buffers_net.h
+++ b/src/lib/net/buffers_net.h
@@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most,
 int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz,
                         size_t *buf_flushlen);
 
-#endif /* !defined(TOR_BUFFERS_H) */
+int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most,
+                       int *reached_eof,
+                       int *socket_error);
+
+int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz,
+                      size_t *buf_flushlen);
+
+#endif /* !defined(TOR_BUFFERS_NET_H) */
diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h
index 69b0528bc0..b3b77b0589 100644
--- a/src/lib/net/gethostname.h
+++ b/src/lib/net/gethostname.h
@@ -16,4 +16,4 @@
 
 MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
 
-#endif
+#endif /* !defined(TOR_GETHOSTNAME_H) */
diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c
index 1a2406ce5f..d9ae7cd562 100644
--- a/src/lib/net/inaddr.c
+++ b/src/lib/net/inaddr.c
@@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst)
   if (af == AF_INET) {
     return tor_inet_aton(src, dst);
   } else if (af == AF_INET6) {
+    ssize_t len = strlen(src);
+
+    /* Reject if src has needless trailing ':'. */
+    if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') {
+      return 0;
+    }
+
     struct in6_addr *out = dst;
     uint16_t words[8];
     int gapPos = -1, i, setWords=0;
@@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst)
         return 0;
       if (TOR_ISXDIGIT(*src)) {
         char *next;
-        ssize_t len;
         long r = strtol(src, &next, 16);
         if (next == NULL || next == src) {
           /* The 'next == src' error case can happen on versions of openbsd
diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h
index 36352b65ea..602573944c 100644
--- a/src/lib/net/inaddr.h
+++ b/src/lib/net/inaddr.h
@@ -24,4 +24,4 @@ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
 const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len);
 int tor_inet_pton(int af, const char *src, void *dst);
 
-#endif
+#endif /* !defined(TOR_INADDR_H) */
diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h
index 806f2c096a..230f29a63a 100644
--- a/src/lib/net/inaddr_st.h
+++ b/src/lib/net/inaddr_st.h
@@ -104,4 +104,4 @@ struct sockaddr_in6 {
 };
 #endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */
 
-#endif /* TOR_INADDR_ST_H */
+#endif /* !defined(TOR_INADDR_ST_H) */
diff --git a/src/lib/net/include.am b/src/lib/net/include.am
index ff0967e786..485019f4b7 100644
--- a/src/lib/net/include.am
+++ b/src/lib/net/include.am
@@ -5,12 +5,14 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-net-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_net_a_SOURCES =			\
 	src/lib/net/address.c			\
 	src/lib/net/alertsock.c                 \
 	src/lib/net/buffers_net.c		\
 	src/lib/net/gethostname.c		\
 	src/lib/net/inaddr.c			\
+	src/lib/net/network_sys.c		\
 	src/lib/net/resolve.c			\
 	src/lib/net/socket.c			\
 	src/lib/net/socketpair.c
@@ -20,6 +22,7 @@ src_lib_libtor_net_testing_a_SOURCES = \
 src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/net/address.h			\
 	src/lib/net/alertsock.h                 \
@@ -28,6 +31,7 @@ noinst_HEADERS +=				\
 	src/lib/net/inaddr.h			\
 	src/lib/net/inaddr_st.h			\
 	src/lib/net/nettypes.h			\
+	src/lib/net/network_sys.h		\
 	src/lib/net/resolve.h			\
 	src/lib/net/socket.h			\
 	src/lib/net/socketpair.h		\
diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h
index 6209bbe18a..60039bac09 100644
--- a/src/lib/net/nettypes.h
+++ b/src/lib/net/nettypes.h
@@ -31,7 +31,7 @@ typedef int socklen_t;
 #define TOR_SOCKET_T_FORMAT "%"PRIuPTR
 #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
 #define TOR_INVALID_SOCKET INVALID_SOCKET
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 /** Type used for a network socket. */
 #define tor_socket_t int
 #define TOR_SOCKET_T_FORMAT "%d"
@@ -41,4 +41,4 @@ typedef int socklen_t;
 #define TOR_INVALID_SOCKET (-1)
 #endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_NET_TYPES_H) */
diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c
new file mode 100644
index 0000000000..e0a2625d73
--- /dev/null
+++ b/src/lib/net/network_sys.c
@@ -0,0 +1,46 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file network_sys.c
+ * \brief Subsystem object for networking setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/net/network_sys.h"
+#include "lib/net/resolve.h"
+#include "lib/net/socket.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#endif
+
+static int
+subsys_network_initialize(void)
+{
+  if (network_init() < 0)
+    return -1;
+
+  return 0;
+}
+
+static void
+subsys_network_shutdown(void)
+{
+#ifdef _WIN32
+  WSACleanup();
+#endif
+  tor_free_getaddrinfo_cache();
+}
+
+const subsys_fns_t sys_network = {
+  .name = "network",
+  /* Network depends on logging, and a lot of other modules depend on network.
+   */
+  .level = -80,
+  .supported = true,
+  .initialize = subsys_network_initialize,
+  .shutdown = subsys_network_shutdown,
+};
diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h
new file mode 100644
index 0000000000..43e62592ca
--- /dev/null
+++ b/src/lib/net/network_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_network.h
+ * \brief Declare subsystem object for the network module.
+ **/
+
+#ifndef TOR_NETWORK_SYS_H
+#define TOR_NETWORK_SYS_H
+
+extern const struct subsys_fns_t sys_network;
+
+#endif /* !defined(TOR_NETWORK_SYS_H) */
diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c
index 8cee29df37..442bc4a6b3 100644
--- a/src/lib/net/resolve.c
+++ b/src/lib/net/resolve.c
@@ -8,6 +8,7 @@
  * \brief Use the libc DNS resolver to convert hostnames into addresses.
  **/
 
+#define RESOLVE_PRIVATE
 #include "lib/net/resolve.h"
 
 #include "lib/net/address.h"
@@ -16,8 +17,8 @@
 #include "lib/string/parse_int.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
-#include "ht.h"
+#include "ext/siphash.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
@@ -35,6 +36,8 @@
  * *<b>addr</b> to the proper IP address, in host byte order.  Returns 0
  * on success, -1 on failure; 1 on transient failure.
  *
+ * This function only accepts IPv4 addresses.
+ *
  * (This function exists because standard windows gethostbyname
  * doesn't treat raw IP addresses properly.)
  */
@@ -45,6 +48,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   tor_addr_t myaddr;
   int ret;
 
+  if (BUG(!addr))
+    return -1;
+
+  *addr = 0;
+
   if ((ret = tor_addr_lookup(name, AF_INET, &myaddr)))
     return ret;
 
@@ -56,183 +64,257 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   return -1;
 }
 
-/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
- * *<b>addr</b> to the proper IP address and family. The <b>family</b>
- * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
- * <i>preferred</i> family, though another one may be returned if only one
- * family is implemented for this address.
+#ifdef HAVE_GETADDRINFO
+
+/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is
+ * available on this system.
  *
- * Return 0 on success, -1 on failure; 1 on transient failure.
+ * See tor_addr_lookup() for details.
  */
-MOCK_IMPL(int,
-tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                          tor_addr_t *addr))
 {
-  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
-   * something.
-   */
-  struct in_addr iaddr;
-  struct in6_addr iaddr6;
-  tor_assert(name);
-  tor_assert(addr);
-  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
-  if (!*name) {
-    /* Empty address is an error. */
-    return -1;
-  } else if (tor_inet_pton(AF_INET, name, &iaddr)) {
-    /* It's an IPv4 IP. */
-    if (family == AF_INET6)
-      return -1;
-    tor_addr_from_in(addr, &iaddr);
-    return 0;
-  } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) {
-    if (family == AF_INET)
-      return -1;
-    tor_addr_from_in6(addr, &iaddr6);
-    return 0;
-  } else {
-#ifdef HAVE_GETADDRINFO
-    int err;
-    struct addrinfo *res=NULL, *res_p;
-    struct addrinfo *best=NULL;
-    struct addrinfo hints;
-    int result = -1;
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = family;
-    hints.ai_socktype = SOCK_STREAM;
-    err = tor_getaddrinfo(name, NULL, &hints, &res);
-    /* The check for 'res' here shouldn't be necessary, but it makes static
-     * analysis tools happy. */
-    if (!err && res) {
-      best = NULL;
-      for (res_p = res; res_p; res_p = res_p->ai_next) {
-        if (family == AF_UNSPEC) {
-          if (res_p->ai_family == AF_INET) {
-            best = res_p;
-            break;
-          } else if (res_p->ai_family == AF_INET6 && !best) {
-            best = res_p;
-          }
-        } else if (family == res_p->ai_family) {
+  int err;
+  struct addrinfo *res=NULL, *res_p;
+  struct addrinfo *best=NULL;
+  struct addrinfo hints;
+  int result = -1;
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = family;
+  hints.ai_socktype = SOCK_STREAM;
+  err = tor_getaddrinfo(name, NULL, &hints, &res);
+  /* The check for 'res' here shouldn't be necessary, but it makes static
+   * analysis tools happy. */
+  if (!err && res) {
+    best = NULL;
+    for (res_p = res; res_p; res_p = res_p->ai_next) {
+      if (family == AF_UNSPEC) {
+        if (res_p->ai_family == AF_INET) {
           best = res_p;
           break;
+        } else if (res_p->ai_family == AF_INET6 && !best) {
+          best = res_p;
         }
+      } else if (family == res_p->ai_family) {
+        best = res_p;
+        break;
       }
-      if (!best)
-        best = res;
-      if (best->ai_family == AF_INET) {
-        tor_addr_from_in(addr,
-                         &((struct sockaddr_in*)best->ai_addr)->sin_addr);
-        result = 0;
-      } else if (best->ai_family == AF_INET6) {
-        tor_addr_from_in6(addr,
-                          &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
-        result = 0;
-      }
-      tor_freeaddrinfo(res);
-      return result;
     }
-    return (err == EAI_AGAIN) ? 1 : -1;
-#else /* !(defined(HAVE_GETADDRINFO)) */
-    struct hostent *ent;
-    int err;
+    if (!best)
+      best = res;
+    if (best->ai_family == AF_INET) {
+      tor_addr_from_in(addr,
+                       &((struct sockaddr_in*)best->ai_addr)->sin_addr);
+      result = 0;
+    } else if (best->ai_family == AF_INET6) {
+      tor_addr_from_in6(addr,
+                        &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
+      result = 0;
+    }
+    tor_freeaddrinfo(res);
+    return result;
+  }
+  return (err == EAI_AGAIN) ? 1 : -1;
+}
+
+#else /* !defined(HAVE_GETADDRINFO) */
+
+/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname().
+ * Used when getaddrinfo() is not available on this system.
+ *
+ * See tor_addr_lookup() for details.
+ */
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                           tor_addr_t *addr))
+{
+  (void) family;
+  struct hostent *ent;
+  int err;
 #ifdef HAVE_GETHOSTBYNAME_R_6_ARG
-    char buf[2048];
-    struct hostent hostent;
-    int r;
-    r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
+  char buf[2048];
+  struct hostent hostent;
+  int r;
+  r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_5_ARG)
-    char buf[2048];
-    struct hostent hostent;
-    ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
+  char buf[2048];
+  struct hostent hostent;
+  ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_3_ARG)
-    struct hostent_data data;
-    struct hostent hent;
-    memset(&data, 0, sizeof(data));
-    err = gethostbyname_r(name, &hent, &data);
-    ent = err ? NULL : &hent;
+  struct hostent_data data;
+  struct hostent hent;
+  memset(&data, 0, sizeof(data));
+  err = gethostbyname_r(name, &hent, &data);
+  ent = err ? NULL : &hent;
 #else
-    ent = gethostbyname(name);
+  ent = gethostbyname(name);
 #ifdef _WIN32
-    err = WSAGetLastError();
+  err = WSAGetLastError();
 #else
-    err = h_errno;
-#endif
+  err = h_errno;
+#endif /* defined(_WIN32) */
 #endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */
-    if (ent) {
-      if (ent->h_addrtype == AF_INET) {
-        tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
-      } else if (ent->h_addrtype == AF_INET6) {
-        tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
-      } else {
-        tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
-      }
-      return 0;
+  if (ent) {
+    if (ent->h_addrtype == AF_INET) {
+      tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
+    } else if (ent->h_addrtype == AF_INET6) {
+      tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
+    } else {
+      tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
     }
+    return 0;
+  }
 #ifdef _WIN32
-    return (err == WSATRY_AGAIN) ? 1 : -1;
+  return (err == WSATRY_AGAIN) ? 1 : -1;
 #else
-    return (err == TRY_AGAIN) ? 1 : -1;
+  return (err == TRY_AGAIN) ? 1 : -1;
 #endif
+}
 #endif /* defined(HAVE_GETADDRINFO) */
+
+/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
+ * *<b>addr</b> to the proper IP address and family. The <b>family</b>
+ * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
+ * <i>preferred</i> family, though another one may be returned if only one
+ * family is implemented for this address.
+ *
+ * Like tor_addr_parse(), this function accepts IPv6 addresses with or without
+ * square brackets.
+ *
+ * Return 0 on success, -1 on failure; 1 on transient failure.
+ */
+MOCK_IMPL(int,
+tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+{
+  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
+   * something.
+   */
+  int parsed_family = 0;
+  int result = -1;
+
+  tor_assert(name);
+  tor_assert(addr);
+  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
+
+  if (!*name) {
+    /* Empty address is an error. */
+    goto permfail;
+  }
+
+  /* Is it an IP address? */
+  parsed_family = tor_addr_parse(addr, name);
+
+  if (parsed_family >= 0) {
+    /* If the IP address family matches, or was unspecified */
+    if (parsed_family == family || family == AF_UNSPEC) {
+      goto success;
+    } else {
+      goto permfail;
+    }
+  } else {
+    /* Clear the address after a failed tor_addr_parse(). */
+    memset(addr, 0, sizeof(tor_addr_t));
+    result = tor_addr_lookup_host_impl(name, family, addr);
+    goto done;
   }
+
+ /* If we weren't successful, and haven't already set the result,
+  * assume it's a permanent failure */
+ permfail:
+  result = -1;
+  goto done;
+ success:
+  result = 0;
+
+ /* We have set the result, now it's time to clean up */
+ done:
+  if (result) {
+    /* Clear the address on error */
+    memset(addr, 0, sizeof(tor_addr_t));
+  }
+  return result;
 }
 
 /** Parse an address or address-port combination from <b>s</b>, resolve the
  * address as needed, and put the result in <b>addr_out</b> and (optionally)
- * <b>port_out</b>.  Return 0 on success, negative on failure. */
+ * <b>port_out</b>.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Return 0 on success, negative on failure. */
 int
 tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out)
 {
-  const char *port;
   tor_addr_t addr;
-  uint16_t portval;
+  uint16_t portval = 0;
   char *tmp = NULL;
+  int rv = 0;
+  int result;
 
   tor_assert(s);
   tor_assert(addr_out);
 
   s = eat_whitespace(s);
 
-  if (*s == '[') {
-    port = strstr(s, "]");
-    if (!port)
-      goto err;
-    tmp = tor_strndup(s+1, port-(s+1));
-    port = port+1;
-    if (*port == ':')
-      port++;
-    else
-      port = NULL;
-  } else {
-    port = strchr(s, ':');
-    if (port)
-      tmp = tor_strndup(s, port-s);
-    else
-      tmp = tor_strdup(s);
-    if (port)
-      ++port;
+  /* Try parsing s as an address:port first, so we don't have to duplicate
+   * the logic that rejects IPv6:Port with no square brackets. */
+  rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0);
+  /* That was easy, no DNS required. */
+  if (rv == 0)
+    goto success;
+
+  /* Now let's check for malformed IPv6 addresses and ports:
+   * tor_addr_port_parse() requires squared brackes if there is a port,
+   * and we want tor_addr_port_lookup() to have the same requirement.
+   * But we strip the port using tor_addr_port_split(), so tor_addr_lookup()
+   * only sees the address, and will accept it without square brackets. */
+  int family = tor_addr_parse(&addr, s);
+  /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need
+   * to reject this address as malformed. */
+  if (family >= 0) {
+    /* Double-check it's an IPv6 address. If not, we have a parsing bug.
+     */
+    tor_assertf_nonfatal(family == AF_INET6,
+                         "Wrong family: %d (should be IPv6: %d) which "
+                         "failed IP:port parsing, but passed IP parsing. "
+                         "input string: '%s'; parsed address: '%s'.",
+                         family, AF_INET6, s, fmt_addr(&addr));
+    goto err;
   }
 
-  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+  /* Now we have a hostname. Let's split off the port, if any. */
+  rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval);
+  if (rv < 0)
     goto err;
-  tor_free(tmp);
 
-  if (port) {
-    portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL);
-    if (!portval)
-      goto err;
-  } else {
-    portval = 0;
-  }
+  /* And feed the hostname to the lookup function. */
+  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+    goto err;
 
+ success:
   if (port_out)
     *port_out = portval;
   tor_addr_copy(addr_out, &addr);
+  result = 0;
+  goto done;
 
-  return 0;
  err:
+  /* Clear the address and port on error */
+  memset(addr_out, 0, sizeof(tor_addr_t));
+  if (port_out)
+    *port_out = 0;
+  result = -1;
+
+ /* We have set the result, now it's time to clean up */
+ done:
   tor_free(tmp);
-  return -1;
+  return result;
 }
 
 #ifdef USE_SANDBOX_GETADDRINFO
@@ -421,4 +503,13 @@ tor_make_getaddrinfo_cache_active(void)
 {
   sandbox_getaddrinfo_is_active = 1;
 }
-#endif
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
+void
+sandbox_disable_getaddrinfo_cache(void)
+{
+}
+void
+tor_make_getaddrinfo_cache_active(void)
+{
+}
+#endif /* defined(USE_SANDBOX_GETADDRINFO) */
diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h
index 47a283c81c..b979b2fb41 100644
--- a/src/lib/net/resolve.h
+++ b/src/lib/net/resolve.h
@@ -24,12 +24,18 @@
 
 struct tor_addr_t;
 
+/*
+ * Primary lookup functions.
+ */
 MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr));
 MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
                                 struct tor_addr_t *addr_out));
 int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out,
                          uint16_t *port_out);
 
+/*
+ * Sandbox helpers
+ */
 struct addrinfo;
 #ifdef USE_SANDBOX_GETADDRINFO
 /** Pre-calls getaddrinfo in order to pre-record result. */
@@ -42,8 +48,7 @@ int tor_getaddrinfo(const char *name, const char *servname,
                         struct addrinfo **res);
 void tor_freeaddrinfo(struct addrinfo *addrinfo);
 void tor_free_getaddrinfo_cache(void);
-void tor_make_getaddrinfo_cache_active(void);
-#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
 #define tor_getaddrinfo(name, servname, hints, res)  \
   getaddrinfo((name),(servname), (hints),(res))
 #define tor_add_addrinfo(name) \
@@ -54,5 +59,15 @@ void tor_make_getaddrinfo_cache_active(void);
 #endif /* defined(USE_SANDBOX_GETADDRINFO) */
 
 void sandbox_disable_getaddrinfo_cache(void);
+void tor_make_getaddrinfo_cache_active(void);
 
+/*
+ * Internal resolver wrapper; exposed for mocking.
+ */
+#ifdef RESOLVE_PRIVATE
+MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name,
+                                                  uint16_t family,
+                                                  struct tor_addr_t *addr));
 #endif
+
+#endif /* !defined(TOR_RESOLVE_H) */
diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c
index fba90b7506..e1b82251ed 100644
--- a/src/lib/net/socket.c
+++ b/src/lib/net/socket.c
@@ -31,6 +31,9 @@
 #endif
 #include <stddef.h>
 #include <string.h>
+#ifdef __FreeBSD__
+#include <sys/sysctl.h>
+#endif
 
 /** Called before we make any calls to network-related functions.
  * (Some operating systems require their network libraries to be
@@ -60,6 +63,32 @@ network_init(void)
   return 0;
 }
 
+/**
+ * Warn the user if any system network parameters should be changed.
+ */
+void
+check_network_configuration(bool server_mode)
+{
+#ifdef __FreeBSD__
+  if (server_mode) {
+    int random_id_state;
+    size_t state_size = sizeof(random_id_state);
+
+    if (sysctlbyname("net.inet.ip.random_id", &random_id_state,
+                     &state_size, NULL, 0)) {
+      log_warn(LD_CONFIG,
+               "Failed to figure out if IP ids are randomized.");
+    } else if (random_id_state == 0) {
+      log_warn(LD_CONFIG, "Looks like IP ids are not randomized. "
+               "Please consider setting the net.inet.ip.random_id sysctl, "
+               "so your relay makes it harder to figure out how busy it is.");
+    }
+  }
+#else /* !defined(__FreeBSD__) */
+  (void) server_mode;
+#endif /* defined(__FreeBSD__) */
+}
+
 /* When set_max_file_sockets() is called, update this with the max file
  * descriptor value so we can use it to check the limit when opening a new
  * socket. Default value is what Debian sets as the default hard limit. */
@@ -177,7 +206,7 @@ mark_socket_closed(tor_socket_t s)
     bitarray_clear(open_sockets, s);
   }
 }
-#else /* !(defined(DEBUG_SOCKET_COUNTING)) */
+#else /* !defined(DEBUG_SOCKET_COUNTING) */
 #define mark_socket_open(s) ((void) (s))
 #define mark_socket_closed(s) ((void) (s))
 #endif /* defined(DEBUG_SOCKET_COUNTING) */
@@ -279,7 +308,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -389,7 +418,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -429,7 +458,9 @@ get_n_open_sockets(void)
  * localhost is inaccessible (for example, if the networking
  * stack is down). And even if it succeeds, the socket pair will not
  * be able to read while localhost is down later (the socket pair may
- * even close, depending on OS-specific timeouts).
+ * even close, depending on OS-specific timeouts). The socket pair
+ * should work on IPv4-only, IPv6-only, and dual-stack systems, as long
+ * as they have the standard localhost addresses.
  *
  * Returns 0 on success and -errno on failure; do not rely on the value
  * of errno or WSAGetLastError().
@@ -456,11 +487,11 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
   r = socketpair(family, type, protocol, fd);
   if (r < 0)
     return -errno;
-#else
+#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */
   r = tor_ersatz_socketpair(family, type, protocol, fd);
   if (r < 0)
     return -r;
-#endif
+#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */
 
 #if defined(FD_CLOEXEC)
   if (SOCKET_OK(fd[0])) {
diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h
index 0909619510..53a9f1bb92 100644
--- a/src/lib/net/socket.h
+++ b/src/lib/net/socket.h
@@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
 int set_socket_nonblocking(tor_socket_t socket);
 int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]);
 int network_init(void);
+void check_network_configuration(bool server_mode);
 
 int get_max_sockets(void);
 void set_max_sockets(int);
@@ -91,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count);
 #define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
 int tor_socket_errno(tor_socket_t sock);
 const char *tor_socket_strerror(int e);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 #define SOCK_ERRNO(e) e
 #if EAGAIN == EWOULDBLOCK
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
@@ -115,4 +116,4 @@ const char *tor_socket_strerror(int e);
 #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKET_H) */
diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c
index 10eb749735..f3a0c3770a 100644
--- a/src/lib/net/socketpair.c
+++ b/src/lib/net/socketpair.c
@@ -22,11 +22,11 @@
 #include <windows.h>
 #define socket_errno() (WSAGetLastError())
 #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT
-#else
+#else /* !defined(_WIN32) */
 #define closesocket(x) close(x)
 #define socket_errno() (errno)
 #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef NEED_ERSATZ_SOCKETPAIR
 
@@ -105,7 +105,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2)
 /**
  * Helper used to implement socketpair on systems that lack it, by
  * making a direct connection to localhost.
- */
+ *
+ * See tor_socketpair() for details.
+ *
+ * The direct connection defaults to IPv4, but falls back to IPv6 if
+ * IPv4 is not supported.
+ **/
 int
 tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
 {
diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h
index 6be0803881..5820606973 100644
--- a/src/lib/net/socketpair.h
+++ b/src/lib/net/socketpair.h
@@ -16,4 +16,4 @@ int tor_ersatz_socketpair(int family, int type, int protocol,
                           tor_socket_t fd[2]);
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKETPAIR_H) */
diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h
index e55242ce66..e55119e0b0 100644
--- a/src/lib/net/socks5_status.h
+++ b/src/lib/net/socks5_status.h
@@ -29,4 +29,4 @@ typedef enum {
   SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08,
 } socks5_reply_status_t;
 
-#endif
+#endif /* !defined(TOR_SOCKS5_STATUS_H) */
diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am
index 16c5812604..84bd7feb00 100644
--- a/src/lib/osinfo/include.am
+++ b/src/lib/osinfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_osinfo_a_SOURCES =			\
 	src/lib/osinfo/uname.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_osinfo_testing_a_SOURCES = \
 src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/osinfo/uname.h
diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c
index 2b37ff136c..34860c407a 100644
--- a/src/lib/osinfo/uname.c
+++ b/src/lib/osinfo/uname.c
@@ -137,7 +137,7 @@ get_uname,(void))
         if (!is_server && !is_client) {
           strlcat(uname_result, " [client or server]", sizeof(uname_result));
         }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
         /* LCOV_EXCL_START -- can't provoke uname failure */
         strlcpy(uname_result, "Unknown platform", sizeof(uname_result));
         /* LCOV_EXCL_STOP */
diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h
index fcce629074..443a30d358 100644
--- a/src/lib/osinfo/uname.h
+++ b/src/lib/osinfo/uname.h
@@ -15,4 +15,4 @@
 
 MOCK_DECL(const char *, get_uname,(void));
 
-#endif
+#endif /* !defined(HAVE_TOR_UNAME_H) */
diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include
index 05414d2a96..ce1b6ecf59 100644
--- a/src/lib/process/.may_include
+++ b/src/lib/process/.may_include
@@ -1,17 +1,20 @@
 orconfig.h
 
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
 lib/err/*.h
-lib/intmath/*.h
+lib/evloop/*.h
 lib/fs/*.h
+lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/process/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 
-ht.h
\ No newline at end of file
+ext/ht.h
diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c
index 3b90bef671..ae34b5bcb8 100644
--- a/src/lib/process/daemon.c
+++ b/src/lib/process/daemon.c
@@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd)
 
   return 0;
 }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
 /* defined(_WIN32) */
 int
 start_daemon(void)
diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h
index 20920e0aae..423c498837 100644
--- a/src/lib/process/daemon.h
+++ b/src/lib/process/daemon.h
@@ -18,4 +18,4 @@ int finish_daemon(const char *desired_cwd);
 
 bool start_daemon_has_been_called(void);
 
-#endif
+#endif /* !defined(TOR_DAEMON_H) */
diff --git a/src/lib/process/env.c b/src/lib/process/env.c
index 0060200ba1..3912ade197 100644
--- a/src/lib/process/env.c
+++ b/src/lib/process/env.c
@@ -47,7 +47,7 @@ get_environment(void)
    * when we do a mostly-static build on OSX 10.7, the resulting binary won't
    * work on OSX 10.6. */
   return *_NSGetEnviron();
-#else /* !(defined(HAVE__NSGETENVIRON)) */
+#else /* !defined(HAVE__NSGETENVIRON) */
   return environ;
 #endif /* defined(HAVE__NSGETENVIRON) */
 }
diff --git a/src/lib/process/env.h b/src/lib/process/env.h
index 15d59351e0..19c2235970 100644
--- a/src/lib/process/env.h
+++ b/src/lib/process/env.h
@@ -38,4 +38,4 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
                                            const char *new_var,
                                            void (*free_old)(void*),
                                            int free_p);
-#endif
+#endif /* !defined(TOR_ENV_H) */
diff --git a/src/lib/process/include.am b/src/lib/process/include.am
index c6cc3a6699..af5f99617b 100644
--- a/src/lib/process/include.am
+++ b/src/lib/process/include.am
@@ -5,25 +5,35 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-process-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_process_a_SOURCES =		\
 	src/lib/process/daemon.c		\
 	src/lib/process/env.c			\
 	src/lib/process/pidfile.c		\
+	src/lib/process/process.c		\
+	src/lib/process/process_sys.c		\
+	src/lib/process/process_unix.c		\
+	src/lib/process/process_win32.c		\
 	src/lib/process/restrict.c		\
 	src/lib/process/setuid.c		\
-	src/lib/process/subprocess.c		\
-	src/lib/process/waitpid.c
+	src/lib/process/waitpid.c		\
+	src/lib/process/winprocess_sys.c
 
 src_lib_libtor_process_testing_a_SOURCES = \
 	$(src_lib_libtor_process_a_SOURCES)
 src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/process/daemon.h		\
 	src/lib/process/env.h			\
 	src/lib/process/pidfile.h		\
+	src/lib/process/process.h		\
+	src/lib/process/process_sys.h		\
+	src/lib/process/process_unix.h		\
+	src/lib/process/process_win32.h		\
 	src/lib/process/restrict.h		\
 	src/lib/process/setuid.h		\
-	src/lib/process/subprocess.h		\
-	src/lib/process/waitpid.h
+	src/lib/process/waitpid.h		\
+	src/lib/process/winprocess_sys.h
diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h
index dfeb39e046..af59041f80 100644
--- a/src/lib/process/pidfile.h
+++ b/src/lib/process/pidfile.h
@@ -13,4 +13,4 @@
 
 int write_pidfile(const char *filename);
 
-#endif
+#endif /* !defined(TOR_PIDFILE_H) */
diff --git a/src/lib/process/process.c b/src/lib/process/process.c
new file mode 100644
index 0000000000..2194a603ff
--- /dev/null
+++ b/src/lib/process/process.c
@@ -0,0 +1,797 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.c
+ * \brief Module for working with other processes.
+ **/
+
+#define PROCESS_PRIVATE
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_STDDEF_H
+#include <stddef.h>
+#endif
+
+/** A list of all <b>process_t</b> instances currently allocated. */
+static smartlist_t *processes;
+
+/**
+ * Boolean.  If true, then Tor may call execve or CreateProcess via
+ * tor_spawn_background.
+ **/
+static int may_spawn_background_process = 1;
+
+/** Structure to represent a child process. */
+struct process_t {
+  /** Process status. */
+  process_status_t status;
+
+  /** Which protocol is the process using? */
+  process_protocol_t protocol;
+
+  /** Which function to call when we have data ready from stdout? */
+  process_read_callback_t stdout_read_callback;
+
+  /** Which function to call when we have data ready from stderr? */
+  process_read_callback_t stderr_read_callback;
+
+  /** Which function call when our process terminated? */
+  process_exit_callback_t exit_callback;
+
+  /** Our exit code when the process have terminated. */
+  process_exit_code_t exit_code;
+
+  /** Name of the command we want to execute (for example: /bin/ls). */
+  char *command;
+
+  /** The arguments used for the new process. The format here is one argument
+   * per element of the smartlist_t. On Windows these arguments are combined
+   * together using the <b>tor_join_win_cmdline</b> function. On Unix the
+   * process name (argv[0]) and the trailing NULL is added automatically before
+   * the process is executed. */
+  smartlist_t *arguments;
+
+  /** The environment used for the new process. */
+  smartlist_t *environment;
+
+  /** Buffer to store data from stdout when it is read. */
+  buf_t *stdout_buffer;
+
+  /** Buffer to store data from stderr when it is read. */
+  buf_t *stderr_buffer;
+
+  /** Buffer to store data to stdin before it is written. */
+  buf_t *stdin_buffer;
+
+  /** Do we need to store some custom data with the process? */
+  void *data;
+
+#ifndef _WIN32
+  /** Our Unix process handle. */
+  process_unix_t *unix_process;
+#else
+  /** Our Win32 process handle. */
+  process_win32_t *win32_process;
+#endif /* !defined(_WIN32) */
+};
+
+/** Convert a given process status in <b>status</b> to its string
+ * representation. */
+const char *
+process_status_to_string(process_status_t status)
+{
+  switch (status) {
+  case PROCESS_STATUS_NOT_RUNNING:
+    return "not running";
+  case PROCESS_STATUS_RUNNING:
+    return "running";
+  case PROCESS_STATUS_ERROR:
+    return "error";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/** Convert a given process protocol in <b>protocol</b> to its string
+ * representation. */
+const char *
+process_protocol_to_string(process_protocol_t protocol)
+{
+  switch (protocol) {
+  case PROCESS_PROTOCOL_LINE:
+    return "Line";
+  case PROCESS_PROTOCOL_RAW:
+    return "Raw";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * Turn off may_spawn_background_process, so that all future calls to
+ * tor_spawn_background are guaranteed to fail.
+ **/
+void
+tor_disable_spawning_background_processes(void)
+{
+  may_spawn_background_process = 0;
+}
+
+/** Initialize the Process subsystem.  This function initializes the Process
+ * subsystem's global state. For cleaning up, <b>process_free_all()</b> should
+ * be called. */
+void
+process_init(void)
+{
+  processes = smartlist_new();
+
+#ifdef _WIN32
+  process_win32_init();
+#endif
+}
+
+/** Free up all resources that is handled by the Process subsystem. Note that
+ * this call does not terminate already running processes. */
+void
+process_free_all(void)
+{
+#ifdef _WIN32
+  process_win32_deinit();
+#endif
+
+  SMARTLIST_FOREACH(processes, process_t *, x, process_free(x));
+  smartlist_free(processes);
+}
+
+/** Get a list of all processes.  This function returns a smartlist of
+ * <b>process_t</b> containing all the currently allocated processes. */
+const smartlist_t *
+process_get_all_processes(void)
+{
+  return processes;
+}
+
+/** Allocate and initialize a new process.  This function returns a newly
+ * allocated and initialized process data, which can be used to configure and
+ * later run a subprocess of Tor. Use the various <b>process_set_*()</b>
+ * methods to configure it and run the process using <b>process_exec()</b>. Use
+ * <b>command</b> to specify the path to the command to run. You can either
+ * specify an absolute path to the command or relative where Tor will use the
+ * underlying operating system's functionality for finding the command to run.
+ * */
+process_t *
+process_new(const char *command)
+{
+  tor_assert(command);
+
+  process_t *process;
+  process = tor_malloc_zero(sizeof(process_t));
+
+  /* Set our command. */
+  process->command = tor_strdup(command);
+
+  /* By default we are not running. */
+  process->status = PROCESS_STATUS_NOT_RUNNING;
+
+  /* Prepare process environment. */
+  process->arguments = smartlist_new();
+  process->environment = smartlist_new();
+
+  /* Prepare the buffers. */
+  process->stdout_buffer = buf_new();
+  process->stderr_buffer = buf_new();
+  process->stdin_buffer = buf_new();
+
+#ifndef _WIN32
+  /* Prepare our Unix process handle. */
+  process->unix_process = process_unix_new();
+#else
+  /* Prepare our Win32 process handle. */
+  process->win32_process = process_win32_new();
+#endif /* !defined(_WIN32) */
+
+  smartlist_add(processes, process);
+
+  return process;
+}
+
+/** Deallocate the given process in <b>process</b>. */
+void
+process_free_(process_t *process)
+{
+  if (! process)
+    return;
+
+  /* Cleanup parameters. */
+  tor_free(process->command);
+
+  /* Cleanup arguments and environment. */
+  SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x));
+  smartlist_free(process->arguments);
+
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+
+  /* Cleanup the buffers. */
+  buf_free(process->stdout_buffer);
+  buf_free(process->stderr_buffer);
+  buf_free(process->stdin_buffer);
+
+#ifndef _WIN32
+  /* Cleanup our Unix process handle. */
+  process_unix_free(process->unix_process);
+#else
+  /* Cleanup our Win32 process handle. */
+  process_win32_free(process->win32_process);
+#endif /* !defined(_WIN32) */
+
+  smartlist_remove(processes, process);
+
+  tor_free(process);
+}
+
+/** Execute the given process.  This function executes the given process as a
+ * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_exec(process_t *process)
+{
+  tor_assert(process);
+
+  if (BUG(may_spawn_background_process == 0))
+    return PROCESS_STATUS_ERROR;
+
+  process_status_t status = PROCESS_STATUS_NOT_RUNNING;
+
+  log_info(LD_PROCESS, "Starting new process: %s", process->command);
+
+#ifndef _WIN32
+  status = process_unix_exec(process);
+#else
+  status = process_win32_exec(process);
+#endif
+
+  /* Update our state. */
+  process_set_status(process, status);
+
+  if (status != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_PROCESS, "Failed to start process: %s",
+             process_get_command(process));
+  }
+
+  return status;
+}
+
+/** Terminate the given process. Returns true on success,
+ * otherwise false. */
+bool
+process_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  /* Terminating a non-running process isn't going to work. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  log_debug(LD_PROCESS, "Terminating process");
+
+#ifndef _WIN32
+  return process_unix_terminate(process);
+#else
+  return process_win32_terminate(process);
+#endif
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+#ifndef _WIN32
+  return process_unix_get_pid(process);
+#else
+  return process_win32_get_pid(process);
+#endif
+}
+
+/** Set the callback function for output from the child process's standard out
+ * handle.  This function sets the callback function which is called every time
+ * the child process have written output to its standard out file handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stdout_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stdout_read_callback = callback;
+}
+
+/** Set the callback function for output from the child process's standard
+ * error handle.  This function sets the callback function which is called
+ * every time the child process have written output to its standard error file
+ * handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stderr_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stderr_read_callback = callback;
+}
+
+/** Set the callback function for process exit notification.  The
+ * <b>callback</b> function will be called every time your child process have
+ * terminated. */
+void
+process_set_exit_callback(process_t *process,
+                          process_exit_callback_t callback)
+{
+  tor_assert(process);
+  process->exit_callback = callback;
+}
+
+/** Get the current command of the given process. */
+const char *
+process_get_command(const process_t *process)
+{
+  tor_assert(process);
+  return process->command;
+}
+
+void
+process_set_protocol(process_t *process, process_protocol_t protocol)
+{
+  tor_assert(process);
+  process->protocol = protocol;
+}
+
+/** Get the currently used protocol of the given process. */
+process_protocol_t
+process_get_protocol(const process_t *process)
+{
+  tor_assert(process);
+  return process->protocol;
+}
+
+/** Set opague pointer to data.  This function allows you to store a pointer to
+ * your own data in the given process. Use <b>process_get_data()</b> in the
+ * various callback functions to retrieve the data again.
+ *
+ * Note that the given process does NOT take ownership of the data and you are
+ * responsible for freeing up any resources allocated by the given data.
+ * */
+void
+process_set_data(process_t *process, void *data)
+{
+  tor_assert(process);
+  process->data = data;
+}
+
+/** Get the opaque pointer to callback data from the given process.  This
+ * function allows you get the data you stored with <b>process_set_data()</b>
+ * in the different callback functions. */
+void *
+process_get_data(const process_t *process)
+{
+  tor_assert(process);
+  return process->data;
+}
+
+/** Set the status of a given process. */
+void
+process_set_status(process_t *process, process_status_t status)
+{
+  tor_assert(process);
+  process->status = status;
+}
+
+/** Get the status of the given process. */
+process_status_t
+process_get_status(const process_t *process)
+{
+  tor_assert(process);
+  return process->status;
+}
+
+/** Append an argument to the list of arguments in the given process. */
+void
+process_append_argument(process_t *process, const char *argument)
+{
+  tor_assert(process);
+  tor_assert(argument);
+
+  smartlist_add(process->arguments, tor_strdup(argument));
+}
+
+/** Returns a list of arguments (excluding the command itself) from the
+ * given process. */
+const smartlist_t *
+process_get_arguments(const process_t *process)
+{
+  tor_assert(process);
+  return process->arguments;
+}
+
+/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b>
+ * to deallocate it after use. */
+char **
+process_get_argv(const process_t *process)
+{
+  tor_assert(process);
+
+  /** Generate a Unix style process argument vector from our process's
+   * arguments smartlist_t. */
+  char **argv = NULL;
+
+  char *filename = process->command;
+  const smartlist_t *arguments = process->arguments;
+  const size_t size = smartlist_len(arguments);
+
+  /* Make space for the process filename as argv[0] and a trailing NULL. */
+  argv = tor_malloc_zero(sizeof(char *) * (size + 2));
+
+  /* Set our filename as first argument. */
+  argv[0] = filename;
+
+  /* Put in the rest of the values from arguments. */
+  SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) {
+    tor_assert(arg_val != NULL);
+
+    argv[arg_val_sl_idx + 1] = arg_val;
+  } SMARTLIST_FOREACH_END(arg_val);
+
+  return argv;
+}
+
+/** This function clears the internal environment and copies over every string
+ * from <b>env</b> as the new environment. */
+void
+process_reset_environment(process_t *process, const smartlist_t *env)
+{
+  tor_assert(process);
+  tor_assert(env);
+
+  /* Cleanup old environment. */
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+  process->environment = smartlist_new();
+
+  SMARTLIST_FOREACH(env, char *, x,
+                    smartlist_add(process->environment, tor_strdup(x)));
+}
+
+/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the
+ * given process. */
+void
+process_set_environment(process_t *process,
+                        const char *key,
+                        const char *value)
+{
+  tor_assert(process);
+  tor_assert(key);
+  tor_assert(value);
+
+  smartlist_add_asprintf(process->environment, "%s=%s", key, value);
+}
+
+/** Returns a newly allocated <b>process_environment_t</b> containing the
+ * environment variables for the given process. */
+process_environment_t *
+process_get_environment(const process_t *process)
+{
+  tor_assert(process);
+  return process_environment_make(process->environment);
+}
+
+#ifndef _WIN32
+/** Get the internal handle for the Unix backend. */
+process_unix_t *
+process_get_unix_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->unix_process);
+  return process->unix_process;
+}
+#else /* defined(_WIN32) */
+/** Get the internal handle for Windows backend. */
+process_win32_t *
+process_get_win32_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->win32_process);
+  return process->win32_process;
+}
+#endif /* !defined(_WIN32) */
+
+/** Write <b>size</b> bytes of <b>data</b> to the given process's standard
+ * input. */
+void
+process_write(process_t *process,
+              const uint8_t *data, size_t size)
+{
+  tor_assert(process);
+  tor_assert(data);
+
+  buf_add(process->stdin_buffer, (char *)data, size);
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** As tor_vsnprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_vprintf(process_t *process,
+                const char *format, va_list args)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  int size;
+  char *data;
+
+  size = tor_vasprintf(&data, format, args);
+  process_write(process, (uint8_t *)data, size);
+  tor_free(data);
+}
+
+/** As tor_snprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_printf(process_t *process,
+               const char *format, ...)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  va_list ap;
+  va_start(ap, format);
+  process_vprintf(process, format, ap);
+  va_end(ap);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard output
+ * handle. */
+void
+process_notify_event_stdout(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stdout(process, process->stdout_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stdout_buffer,
+                      process->stdout_read_callback);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard error
+ * handle. */
+void
+process_notify_event_stderr(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stderr(process, process->stderr_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stderr_buffer,
+                      process->stderr_read_callback);
+}
+
+/** This function is called by the Process backend when a given process is
+ * allowed to begin writing data to the standard input of the child process. */
+void
+process_notify_event_stdin(process_t *process)
+{
+  tor_assert(process);
+
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** This function is called by the Process backend when a given process have
+ * terminated. The exit status code is passed in <b>exit_code</b>. We mark the
+ * process as no longer running and calls the <b>exit_callback</b> with
+ * information about the process termination. The given <b>process</b> is
+ * free'd iff the exit_callback returns true. */
+void
+process_notify_event_exit(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_debug(LD_PROCESS,
+            "Process terminated with exit code: %"PRIu64, exit_code);
+
+  /* Update our state. */
+  process_set_status(process, PROCESS_STATUS_NOT_RUNNING);
+  process->exit_code = exit_code;
+
+  /* Call our exit callback, if it exists. */
+  bool free_process_handle = false;
+
+  /* The exit callback will tell us if we should process_free() our handle. */
+  if (process->exit_callback)
+    free_process_handle = process->exit_callback(process, exit_code);
+
+  if (free_process_handle)
+    process_free(process);
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard out handle. Returns the number of
+ * bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stdout(process, buffer);
+#else
+  return process_win32_read_stdout(process, buffer);
+#endif
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard error handle. Returns the number
+ * of bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stderr(process, buffer);
+#else
+  return process_win32_read_stderr(process, buffer);
+#endif
+}
+
+/** This function calls the backend function for the given process whenever
+ * there is data to be written to the backends' file handles. */
+MOCK_IMPL(STATIC void, process_write_stdin,
+          (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  process_unix_write(process, buffer);
+#else
+  process_win32_write(process, buffer);
+#endif
+}
+
+/** This function calls the protocol handlers based on the value of
+ * <b>process_get_protocol(process)</b>. Currently we call
+ * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and
+ * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */
+STATIC void
+process_read_data(process_t *process,
+                  buf_t *buffer,
+                  process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  switch (process_get_protocol(process)) {
+  case PROCESS_PROTOCOL_RAW:
+    process_read_buffer(process, buffer, callback);
+    break;
+  case PROCESS_PROTOCOL_LINE:
+    process_read_lines(process, buffer, callback);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_unreached();
+    return;
+    /* LCOV_EXCL_STOP */
+  }
+}
+
+/** This function takes the content of the given <b>buffer</b> and passes it to
+ * the given <b>callback</b> function, but ensures that an additional zero byte
+ * is added to the end of the data such that the given callback implementation
+ * can threat the content as a ASCIIZ string. */
+STATIC void
+process_read_buffer(process_t *process,
+                    buf_t *buffer,
+                    process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer);
+
+  /* We allocate an extra byte for the zero byte in the end. */
+  char *data = tor_malloc_zero(size + 1);
+
+  buf_get_bytes(buffer, data, size);
+  log_debug(LD_PROCESS, "Read data from process");
+
+  if (callback)
+    callback(process, data, size);
+
+  tor_free(data);
+}
+
+/** This function tries to extract complete lines from the given <b>buffer</b>
+ * and calls the given <b>callback</b> function whenever it has a complete
+ * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n"
+ * from the line. If we are unable to extract a complete line we leave the data
+ * in the buffer for next call. */
+STATIC void
+process_read_lines(process_t *process,
+                   buf_t *buffer,
+                   process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer) + 1;
+  size_t line_size = 0;
+  char *data = tor_malloc_zero(size);
+  int ret;
+
+  while (true) {
+    line_size = size;
+    ret = buf_get_line(buffer, data, &line_size);
+
+    /* A complete line should always be smaller than the size of our
+     * buffer. */
+    tor_assert(ret != -1);
+
+    /* Remove \n from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\n') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    /* Remove \r from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\r') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    if (ret == 1) {
+      log_debug(LD_PROCESS, "Read line from process: \"%s\"", data);
+
+      if (callback)
+        callback(process, data, line_size);
+
+      /* We have read a whole line, let's see if there is more lines to read.
+       * */
+      continue;
+    }
+
+    /* No complete line for us to read. We are done for now. */
+    tor_assert_nonfatal(ret == 0);
+    break;
+  }
+
+  tor_free(data);
+}
diff --git a/src/lib/process/process.h b/src/lib/process/process.h
new file mode 100644
index 0000000000..05c091a5bf
--- /dev/null
+++ b/src/lib/process/process.h
@@ -0,0 +1,145 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.h
+ * \brief Header for process.c
+ **/
+
+#ifndef TOR_PROCESS_H
+#define TOR_PROCESS_H
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+/** Maximum number of bytes to write to a process' stdin. */
+#define PROCESS_MAX_WRITE (1024)
+
+/** Maximum number of bytes to read from a process' stdout/stderr. */
+#define PROCESS_MAX_READ  (1024)
+
+typedef enum {
+  /** The process is not running. */
+  PROCESS_STATUS_NOT_RUNNING,
+
+  /** The process is running. */
+  PROCESS_STATUS_RUNNING,
+
+  /** The process is in an erroneous state. */
+  PROCESS_STATUS_ERROR
+} process_status_t;
+
+const char *process_status_to_string(process_status_t status);
+
+typedef enum {
+  /** Pass complete \n-terminated lines to the
+   *  callback (with the \n or \r\n removed). */
+  PROCESS_PROTOCOL_LINE,
+
+  /** Pass the raw response from read() to the callback. */
+  PROCESS_PROTOCOL_RAW
+} process_protocol_t;
+
+const char *process_protocol_to_string(process_protocol_t protocol);
+
+void tor_disable_spawning_background_processes(void);
+
+struct smartlist_t;
+
+struct process_t;
+typedef struct process_t process_t;
+
+typedef uint64_t process_exit_code_t;
+typedef uint64_t process_pid_t;
+
+typedef void (*process_read_callback_t)(process_t *,
+                                        const char *,
+                                        size_t);
+typedef bool
+(*process_exit_callback_t)(process_t *, process_exit_code_t);
+
+void process_init(void);
+void process_free_all(void);
+const struct smartlist_t *process_get_all_processes(void);
+
+process_t *process_new(const char *command);
+void process_free_(process_t *process);
+#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s))
+
+process_status_t process_exec(process_t *process);
+bool process_terminate(process_t *process);
+
+process_pid_t process_get_pid(process_t *process);
+
+void process_set_stdout_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_stderr_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_exit_callback(process_t *,
+                               process_exit_callback_t);
+
+const char *process_get_command(const process_t *process);
+
+void process_append_argument(process_t *process, const char *argument);
+const struct smartlist_t *process_get_arguments(const process_t *process);
+char **process_get_argv(const process_t *process);
+
+void process_reset_environment(process_t *process,
+                               const struct smartlist_t *env);
+void process_set_environment(process_t *process,
+                             const char *key,
+                             const char *value);
+
+struct process_environment_t;
+struct process_environment_t *process_get_environment(const process_t *);
+
+void process_set_protocol(process_t *process, process_protocol_t protocol);
+process_protocol_t process_get_protocol(const process_t *process);
+
+void process_set_data(process_t *process, void *data);
+void *process_get_data(const process_t *process);
+
+void process_set_status(process_t *process, process_status_t status);
+process_status_t process_get_status(const process_t *process);
+
+#ifndef _WIN32
+struct process_unix_t;
+struct process_unix_t *process_get_unix_process(const process_t *process);
+#else
+struct process_win32_t;
+struct process_win32_t *process_get_win32_process(const process_t *process);
+#endif /* !defined(_WIN32) */
+
+void process_write(process_t *process,
+                   const uint8_t *data, size_t size);
+void process_vprintf(process_t *process,
+                     const char *format, va_list args) CHECK_PRINTF(2, 0);
+void process_printf(process_t *process,
+                    const char *format, ...) CHECK_PRINTF(2, 3);
+
+void process_notify_event_stdout(process_t *process);
+void process_notify_event_stderr(process_t *process);
+void process_notify_event_stdin(process_t *process);
+void process_notify_event_exit(process_t *process,
+                               process_exit_code_t);
+
+#ifdef PROCESS_PRIVATE
+MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *));
+MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *));
+MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *));
+
+STATIC void process_read_data(process_t *process,
+                              buf_t *buffer,
+                              process_read_callback_t callback);
+STATIC void process_read_buffer(process_t *process,
+                                buf_t *buffer,
+                                process_read_callback_t callback);
+STATIC void process_read_lines(process_t *process,
+                               buf_t *buffer,
+                               process_read_callback_t callback);
+#endif /* defined(PROCESS_PRIVATE) */
+
+#endif /* !defined(TOR_PROCESS_H) */
diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c
new file mode 100644
index 0000000000..3c809a00e8
--- /dev/null
+++ b/src/lib/process/process_sys.c
@@ -0,0 +1,33 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.c
+ * \brief Subsystem object for process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/process/process_sys.h"
+#include "lib/process/process.h"
+
+static int
+subsys_process_initialize(void)
+{
+  process_init();
+  return 0;
+}
+
+static void
+subsys_process_shutdown(void)
+{
+  process_free_all();
+}
+
+const subsys_fns_t sys_process = {
+  .name = "process",
+  .level = -35,
+  .supported = true,
+  .initialize = subsys_process_initialize,
+  .shutdown = subsys_process_shutdown
+};
diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h
new file mode 100644
index 0000000000..b7a116d838
--- /dev/null
+++ b/src/lib/process/process_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.h
+ * \brief Declare subsystem object for the process module.
+ **/
+
+#ifndef TOR_PROCESS_SYS_H
+#define TOR_PROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_process;
+
+#endif /* !defined(TOR_PROCESS_SYS_H) */
diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c
new file mode 100644
index 0000000000..8191bdc1f0
--- /dev/null
+++ b/src/lib/process/process_unix.c
@@ -0,0 +1,698 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.c
+ * \brief Module for working with Unix processes.
+ **/
+
+#define PROCESS_UNIX_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/waitpid.h"
+#include "lib/process/env.h"
+
+#include <stdio.h>
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+#include <sys/prctl.h>
+#endif
+
+#if HAVE_SIGNAL_H
+#include <signal.h>
+#endif
+
+#ifndef _WIN32
+
+/** Maximum number of file descriptors, if we cannot get it via sysconf() */
+#define DEFAULT_MAX_FD 256
+
+/** Internal state for Unix handles. */
+struct process_unix_handle_t {
+  /** Unix File Descriptor. */
+  int fd;
+
+  /** Have we reached end of file? */
+  bool reached_eof;
+
+  /** Event structure for libevent. */
+  struct event *event;
+
+  /** Are we writing? */
+  bool is_writing;
+};
+
+/** Internal state for our Unix process. */
+struct process_unix_t {
+  /** Standard in handle. */
+  process_unix_handle_t stdin_handle;
+
+  /** Standard out handle. */
+  process_unix_handle_t stdout_handle;
+
+  /** Standard error handle. */
+  process_unix_handle_t stderr_handle;
+
+  /** The process identifier of our process. */
+  pid_t pid;
+
+  /** Waitpid Callback structure. */
+  waitpid_callback_t *waitpid;
+};
+
+/** Returns a newly allocated <b>process_unix_t</b>. */
+process_unix_t *
+process_unix_new(void)
+{
+  process_unix_t *unix_process;
+  unix_process = tor_malloc_zero(sizeof(process_unix_t));
+
+  unix_process->stdin_handle.fd = -1;
+  unix_process->stderr_handle.fd = -1;
+  unix_process->stdout_handle.fd = -1;
+
+  return unix_process;
+}
+
+/** Deallocates the given <b>unix_process</b>. */
+void
+process_unix_free_(process_unix_t *unix_process)
+{
+  if (! unix_process)
+    return;
+
+  /* Clean up our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+
+  /* FIXME(ahf): Refactor waitpid code? */
+  unix_process->waitpid = NULL;
+
+  /* Close all our file descriptors. */
+  process_unix_close_file_descriptors(unix_process);
+
+  tor_event_free(unix_process->stdout_handle.event);
+  tor_event_free(unix_process->stderr_handle.event);
+  tor_event_free(unix_process->stdin_handle.event);
+
+  tor_free(unix_process);
+}
+
+/** Executes the given process as a child process of Tor.  This function is
+ * responsible for setting up the child process and run it. This includes
+ * setting up pipes for interprocess communication, initialize the waitpid
+ * callbacks, and finally run fork() followed by execve(). Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_unix_exec(process_t *process)
+{
+  static int max_fd = -1;
+
+  process_unix_t *unix_process;
+  pid_t pid;
+  int stdin_pipe[2];
+  int stdout_pipe[2];
+  int stderr_pipe[2];
+  int retval, fd;
+
+  unix_process = process_get_unix_process(process);
+
+  /* Create standard in pipe. */
+  retval = pipe(stdin_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdin "
+             "communication with process: %s",
+             strerror(errno));
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard out pipe. */
+  retval = pipe(stdout_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdout "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard error pipe. */
+  retval = pipe(stderr_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stderr "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+#ifdef _SC_OPEN_MAX
+  if (-1 == max_fd) {
+    max_fd = (int)sysconf(_SC_OPEN_MAX);
+
+    if (max_fd == -1) {
+      max_fd = DEFAULT_MAX_FD;
+      log_warn(LD_PROCESS,
+               "Cannot find maximum file descriptor, assuming: %d", max_fd);
+    }
+  }
+#else /* !defined(_SC_OPEN_MAX) */
+  max_fd = DEFAULT_MAX_FD;
+#endif /* defined(_SC_OPEN_MAX) */
+
+  pid = fork();
+
+  if (0 == pid) {
+    /* This code is running in the child process context. */
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+    /* Attempt to have the kernel issue a SIGTERM if the parent
+     * goes away. Certain attributes of the binary being execve()ed
+     * will clear this during the execve() call, but it's better
+     * than nothing.
+     */
+    prctl(PR_SET_PDEATHSIG, SIGTERM);
+#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
+
+    /* Link process stdout to the write end of the pipe. */
+    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stderr to the write end of the pipe. */
+    retval = dup2(stderr_pipe[1], STDERR_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stdin to the read end of the pipe */
+    retval = dup2(stdin_pipe[0], STDIN_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Close our pipes now after they have been dup2()'ed. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /* Close all other fds, including the read end of the pipe.  XXX: We should
+     * now be doing enough FD_CLOEXEC setting to make this needless.
+     */
+    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++)
+      close(fd);
+
+    /* Create the argv value for our new process. */
+    char **argv = process_get_argv(process);
+
+    /* Create the env value for our new process. */
+    process_environment_t *env = process_get_environment(process);
+
+    /* Call the requested program. */
+    execve(argv[0], argv, env->unixoid_environment_block);
+
+    /* If we made it here it is because execve failed :-( */
+    tor_free(argv);
+    process_environment_free(env);
+
+ error:
+    fprintf(stderr, "Error from child process: %s", strerror(errno));
+    _exit(1);
+  }
+
+  /* We are in the parent process. */
+  if (-1 == pid) {
+    log_warn(LD_PROCESS,
+             "Failed to create child process: %s", strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    /** Cleanup standard error pipe. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Register our PID. */
+  unix_process->pid = pid;
+
+  /* Setup waitpid callbacks. */
+  unix_process->waitpid = set_waitpid_callback(pid,
+                                               process_unix_waitpid_callback,
+                                               process);
+
+  /* Handle standard out. */
+  unix_process->stdout_handle.fd = stdout_pipe[0];
+  retval = close(stdout_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard error. */
+  unix_process->stderr_handle.fd = stderr_pipe[0];
+  retval = close(stderr_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Failed to close write end of standard error pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard in. */
+  unix_process->stdin_handle.fd = stdin_pipe[1];
+  retval = close(stdin_pipe[0]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s",
+             strerror(errno));
+  }
+
+  /* Setup our handles. */
+  process_unix_setup_handle(process,
+                            &unix_process->stdout_handle,
+                            EV_READ|EV_PERSIST,
+                            stdout_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stderr_handle,
+                            EV_READ|EV_PERSIST,
+                            stderr_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stdin_handle,
+                            EV_WRITE|EV_PERSIST,
+                            stdin_write_callback);
+
+  /* Start reading from standard out and standard error. */
+  process_unix_start_reading(&unix_process->stdout_handle);
+  process_unix_start_reading(&unix_process->stderr_handle);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_unix_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* All running processes should have a waitpid. */
+  if (BUG(unix_process->waitpid == NULL))
+    return false;
+
+  bool success = true;
+
+  /* Send a SIGTERM to our child process. */
+  int ret;
+
+  ret = kill(unix_process->pid, SIGTERM);
+
+  if (ret == -1) {
+    log_warn(LD_PROCESS, "Unable to terminate process: %s",
+             strerror(errno));
+    success = false;
+  }
+
+  /* Close all our FD's. */
+  if (! process_unix_close_file_descriptors(unix_process))
+    success = false;
+
+  return success;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_unix_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+  return (process_pid_t)unix_process->pid;
+}
+
+/** Write the given <b>buffer</b> as input to the given <b>process</b>'s
+ * standard input. Returns the number of bytes written. */
+int
+process_unix_write(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  size_t buffer_flush_len = buf_datalen(buffer);
+  const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len);
+
+  /* If we have data to write (when buffer_flush_len > 0) and we are not
+   * currently getting file descriptor events from the kernel, we tell the
+   * kernel to start notifying us about when we can write to our file
+   * descriptor and return. */
+  if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) {
+    process_unix_start_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We don't have any data to write, but the kernel is currently notifying us
+   * about whether we are able to write or not. Tell the kernel to stop
+   * notifying us until we have data to write. */
+  if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) {
+    process_unix_stop_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We have data to write and the kernel have told us to write it. */
+  return buf_flush_to_pipe(buffer,
+                           process_get_unix_process(process)->stdin_handle.fd,
+                           max_to_write, &buffer_flush_len);
+}
+
+/** Read data from the given process's standard output and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stdout(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stdout_handle,
+                                  buffer);
+}
+
+/** Read data from the given process's standard error and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stderr(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stderr_handle,
+                                  buffer);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard output. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stdout_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdout(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard error. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stderr_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stderr(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * written the child process's standard input. We notify the Process subsystem,
+ * which is then responsible for calling back to us for doing the actual write
+ * of the data. */
+STATIC void
+stdin_write_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdin(process);
+}
+
+/** This function tells libevent that we are interested in receiving read
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * read events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+}
+
+/** This function tells libevent that we are interested in receiving write
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+
+  handle->is_writing = true;
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * write events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+
+  handle->is_writing = false;
+}
+
+/** This function is called when the waitpid system have detected that our
+ * process have terminated. We disable the waitpid system and notify the
+ * Process subsystem that we have terminated. */
+STATIC void
+process_unix_waitpid_callback(int status, void *data)
+{
+  tor_assert(data);
+
+  process_t *process = data;
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* Remove our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+  unix_process->waitpid = NULL;
+
+  /* Notify our process. */
+  process_notify_event_exit(process, status);
+
+  /* Make sure you don't modify the process after we have called
+   * process_notify_event_exit() on it, to allow users to process_free() it in
+   * the exit callback. */
+}
+
+/** This function sets the file descriptor in the <b>handle</b> as non-blocking
+ * and configures the libevent event structure based on the given <b>flags</b>
+ * to ensure that <b>callback</b> is called whenever we have events on the
+ * given <b>handle</b>. */
+STATIC void
+process_unix_setup_handle(process_t *process,
+                          process_unix_handle_t *handle,
+                          short flags,
+                          event_callback_fn callback)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(callback);
+
+  /* Put our file descriptor into non-blocking mode. */
+  if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) {
+    log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s",
+             strerror(errno));
+  }
+
+  /* Setup libevent event. */
+  handle->event = tor_event_new(tor_libevent_get_base(),
+                                handle->fd,
+                                flags,
+                                callback,
+                                process);
+}
+
+/** This function reads data from the given <b>handle</b> and puts it into
+ * <b>buffer</b>. Returns the number of bytes read this way. */
+STATIC int
+process_unix_read_handle(process_t *process,
+                         process_unix_handle_t *handle,
+                         buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(buffer);
+
+  int ret = 0;
+  int eof = 0;
+  int error = 0;
+
+  ret = buf_read_from_pipe(buffer,
+                           handle->fd,
+                           PROCESS_MAX_READ,
+                           &eof,
+                           &error);
+
+  if (error)
+    log_warn(LD_PROCESS,
+             "Unable to read data: %s", strerror(error));
+
+  if (eof) {
+    handle->reached_eof = true;
+    process_unix_stop_reading(handle);
+  }
+
+  return ret;
+}
+
+/** Close the standard in, out, and error handles of the given
+ * <b>unix_process</b>. */
+STATIC bool
+process_unix_close_file_descriptors(process_unix_t *unix_process)
+{
+  tor_assert(unix_process);
+
+  int ret;
+  bool success = true;
+
+  /* Stop reading and writing before we close() our
+   * file descriptors. */
+  if (! unix_process->stdout_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stdout_handle);
+
+  if (! unix_process->stderr_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stderr_handle);
+
+  if (unix_process->stdin_handle.is_writing)
+    process_unix_stop_writing(&unix_process->stdin_handle);
+
+  if (unix_process->stdin_handle.fd != -1) {
+    ret = close(unix_process->stdin_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard in");
+      success = false;
+    }
+
+    unix_process->stdin_handle.fd = -1;
+  }
+
+  if (unix_process->stdout_handle.fd != -1) {
+    ret = close(unix_process->stdout_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard out");
+      success = false;
+    }
+
+    unix_process->stdout_handle.fd = -1;
+  }
+
+  if (unix_process->stderr_handle.fd != -1) {
+    ret = close(unix_process->stderr_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard error");
+      success = false;
+    }
+
+    unix_process->stderr_handle.fd = -1;
+  }
+
+  return success;
+}
+
+#endif /* !defined(_WIN32) */
diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h
new file mode 100644
index 0000000000..da40b3e567
--- /dev/null
+++ b/src/lib/process/process_unix.h
@@ -0,0 +1,68 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.h
+ * \brief Header for process_unix.c
+ **/
+
+#ifndef TOR_PROCESS_UNIX_H
+#define TOR_PROCESS_UNIX_H
+
+#ifndef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+
+#include <event2/event.h>
+
+struct process_t;
+
+struct process_unix_t;
+typedef struct process_unix_t process_unix_t;
+
+process_unix_t *process_unix_new(void);
+void process_unix_free_(process_unix_t *unix_process);
+#define process_unix_free(s) \
+  FREE_AND_NULL(process_unix_t, process_unix_free_, (s))
+
+process_status_t process_unix_exec(struct process_t *process);
+bool process_unix_terminate(struct process_t *process);
+
+process_pid_t process_unix_get_pid(struct process_t *process);
+
+int process_unix_write(struct process_t *process, buf_t *buffer);
+int process_unix_read_stdout(struct process_t *process, buf_t *buffer);
+int process_unix_read_stderr(struct process_t *process, buf_t *buffer);
+
+#ifdef PROCESS_UNIX_PRIVATE
+struct process_unix_handle_t;
+typedef struct process_unix_handle_t process_unix_handle_t;
+
+STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data);
+
+STATIC void process_unix_start_reading(process_unix_handle_t *);
+STATIC void process_unix_stop_reading(process_unix_handle_t *);
+
+STATIC void process_unix_start_writing(process_unix_handle_t *);
+STATIC void process_unix_stop_writing(process_unix_handle_t *);
+
+STATIC void process_unix_waitpid_callback(int status, void *data);
+
+STATIC void process_unix_setup_handle(process_t *process,
+                                      process_unix_handle_t *handle,
+                                      short flags,
+                                      event_callback_fn callback);
+STATIC int process_unix_read_handle(process_t *,
+                                    process_unix_handle_t *,
+                                    buf_t *);
+STATIC bool process_unix_close_file_descriptors(process_unix_t *);
+#endif /* defined(PROCESS_UNIX_PRIVATE) */
+
+#endif /* !defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_UNIX_H) */
diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c
new file mode 100644
index 0000000000..7e4082ad13
--- /dev/null
+++ b/src/lib/process/process_win32.c
@@ -0,0 +1,1105 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.c
+ * \brief Module for working with Windows processes.
+ **/
+
+#define PROCESS_WIN32_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/process/process.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef _WIN32
+
+/** The size of our intermediate buffers. */
+#define BUFFER_SIZE (1024)
+
+/** Timer that ticks once a second and calls the process_win32_timer_callback()
+ * function. */
+static periodic_timer_t *periodic_timer;
+
+/** Structure to represent the state around the pipe HANDLE.
+ *
+ * This structure is used to store state about a given HANDLE, including
+ * whether we have reached end of file, its intermediate buffers, and how much
+ * data that is available in the intermediate buffer. */
+struct process_win32_handle_t {
+  /** Standard out pipe handle. */
+  HANDLE pipe;
+
+  /** True iff we have reached EOF from the pipe. */
+  bool reached_eof;
+
+  /** How much data is available in buffer. */
+  size_t data_available;
+
+  /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */
+  char buffer[BUFFER_SIZE];
+
+  /** Overlapped structure for ReadFileEx() and WriteFileEx(). */
+  OVERLAPPED overlapped;
+
+  /** Are we waiting for another I/O operation to complete? */
+  bool busy;
+};
+
+/** Structure to represent the Windows specific implementation details of this
+ * Process backend.
+ *
+ * This structure is attached to <b>process_t</b> (see process.h) and is
+ * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b>
+ * method. */
+struct process_win32_t {
+  /** Standard in state. */
+  process_win32_handle_t stdin_handle;
+
+  /** Standard out state. */
+  process_win32_handle_t stdout_handle;
+
+  /** Standard error state. */
+  process_win32_handle_t stderr_handle;
+
+  /** Process Information. */
+  PROCESS_INFORMATION process_information;
+};
+
+/** Create a new <b>process_win32_t</b>.
+ *
+ * This function constructs a new <b>process_win32_t</b> and initializes the
+ * default values. */
+process_win32_t *
+process_win32_new(void)
+{
+  process_win32_t *win32_process;
+  win32_process = tor_malloc_zero(sizeof(process_win32_t));
+
+  win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE;
+
+  return win32_process;
+}
+
+/** Free a given <b>process_win32_t</b>.
+ *
+ * This function deinitializes and frees up the resources allocated for the
+ * given <b>process_win32_t</b>. */
+void
+process_win32_free_(process_win32_t *win32_process)
+{
+  if (! win32_process)
+    return;
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  tor_free(win32_process);
+}
+
+/** Initialize the Windows backend of the Process subsystem. */
+void
+process_win32_init(void)
+{
+  /* We don't start the periodic timer here because it makes no sense to have
+   * the timer running until we have some processes that benefits from the
+   * timer timer ticks. */
+}
+
+/** Deinitialize the Windows backend of the Process subsystem. */
+void
+process_win32_deinit(void)
+{
+  /* Stop our timer, but only if it's running. */
+  if (process_win32_timer_running())
+    process_win32_timer_stop();
+}
+
+/** Execute the given process. This function is responsible for setting up
+ * named pipes for I/O between the child process and the Tor process. Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_win32_exec(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  HANDLE stdout_pipe_read = NULL;
+  HANDLE stdout_pipe_write = NULL;
+  HANDLE stderr_pipe_read = NULL;
+  HANDLE stderr_pipe_write = NULL;
+  HANDLE stdin_pipe_read = NULL;
+  HANDLE stdin_pipe_write = NULL;
+  BOOL ret = FALSE;
+
+  /* Setup our security attributes. */
+  SECURITY_ATTRIBUTES security_attributes;
+  security_attributes.nLength = sizeof(security_attributes);
+  security_attributes.bInheritHandle = TRUE;
+  /* FIXME: should we set explicit security attributes?
+   * (See Ticket #2046, comment 5) */
+  security_attributes.lpSecurityDescriptor = NULL;
+
+  /* Create our standard out pipe. */
+  if (! process_win32_create_pipe(&stdout_pipe_read,
+                                  &stdout_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create our standard error pipe. */
+  if (! process_win32_create_pipe(&stderr_pipe_read,
+                                  &stderr_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create out standard in pipe. */
+  if (! process_win32_create_pipe(&stdin_pipe_read,
+                                  &stdin_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_WRITER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Configure startup info for our child process. */
+  STARTUPINFOA startup_info;
+
+  memset(&startup_info, 0, sizeof(startup_info));
+  startup_info.cb = sizeof(startup_info);
+  startup_info.hStdError = stderr_pipe_write;
+  startup_info.hStdOutput = stdout_pipe_write;
+  startup_info.hStdInput = stdin_pipe_read;
+  startup_info.dwFlags |= STARTF_USESTDHANDLES;
+
+  /* Create the env value for our new process. */
+  process_environment_t *env = process_get_environment(process);
+
+  /* Create the argv value for our new process. */
+  char **argv = process_get_argv(process);
+
+  /* Windows expects argv to be a whitespace delimited string, so join argv up
+   */
+  char *joined_argv = tor_join_win_cmdline((const char **)argv);
+
+  /* Create the child process */
+  ret = CreateProcessA(NULL,
+                       joined_argv,
+                       NULL,
+                       NULL,
+                       TRUE,
+                       CREATE_NO_WINDOW,
+                       env->windows_environment_block[0] == '\0' ?
+                         NULL : env->windows_environment_block,
+                       NULL,
+                       &startup_info,
+                       &win32_process->process_information);
+
+  tor_free(argv);
+  tor_free(joined_argv);
+  process_environment_free(env);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "CreateProcessA() failed: %s",
+      format_win32_error(GetLastError()));
+
+    /* Cleanup our handles. */
+    CloseHandle(stdout_pipe_read);
+    CloseHandle(stdout_pipe_write);
+    CloseHandle(stderr_pipe_read);
+    CloseHandle(stderr_pipe_write);
+    CloseHandle(stdin_pipe_read);
+    CloseHandle(stdin_pipe_write);
+
+    /* In the Unix backend, we do not get an error in the Tor process when a
+     * child process fails to spawn its target executable since we need to
+     * first do the fork() call in the Tor process and then the child process
+     * is responsible for doing the call to execve().
+     *
+     * This means that the user of the process_exec() API must check for
+     * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on
+     * Unix, but will happen for error cases on Windows where it does not
+     * happen on Unix. For example: when the target executable does not exist
+     * on the file system.
+     *
+     * To have somewhat feature compatibility between the Unix and the Windows
+     * backend, we here notify the process_t owner that the process have exited
+     * (even though it never managed to run) to ensure that the exit callback
+     * is executed.
+     */
+    process_notify_event_exit(process, 0);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* TODO: Should we close hProcess and hThread in
+   * process_handle->process_information? */
+  win32_process->stdout_handle.pipe = stdout_pipe_read;
+  win32_process->stderr_handle.pipe = stderr_pipe_read;
+  win32_process->stdin_handle.pipe = stdin_pipe_write;
+
+  /* Close our ends of the pipes that is now owned by the child process. */
+  CloseHandle(stdout_pipe_write);
+  CloseHandle(stderr_pipe_write);
+  CloseHandle(stdin_pipe_read);
+
+  /* Used by the callback functions from ReadFileEx() and WriteFileEx() such
+   * that we can figure out which process_t that was responsible for the event.
+   *
+   * Warning, here be dragons:
+   *
+   *   MSDN says that the hEvent member of the overlapped structure is unused
+   *   for ReadFileEx() and WriteFileEx, which allows us to store a pointer to
+   *   our process state there.
+   */
+  win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process;
+
+  /* Start our timer if it is not already running. */
+  if (! process_win32_timer_running())
+    process_win32_timer_start();
+
+  /* We use Windows Extended I/O functions, so our completion callbacks are
+   * called automatically for us when there is data to read. Because of this
+   * we start the read of standard out and error right away. */
+  process_notify_event_stdout(process);
+  process_notify_event_stderr(process);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_win32_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Terminate our process. */
+  BOOL ret;
+
+  ret = TerminateProcess(win32_process->process_information.hProcess, 0);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "TerminateProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  return true;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_win32_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  return (process_pid_t)win32_process->process_information.dwProcessId;
+}
+
+/** Schedule an async write of the data found in <b>buffer</b> for the given
+ * process.  This function runs an async write operation of the content of
+ * buffer, if we are not already waiting for a pending I/O request. Returns the
+ * number of bytes that Windows will hopefully write for us in the background.
+ * */
+int
+process_win32_write(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD error_code = 0;
+  const size_t buffer_size = buf_datalen(buffer);
+
+  /* Windows is still writing our buffer. */
+  if (win32_process->stdin_handle.busy)
+    return 0;
+
+  /* Nothing for us to do right now. */
+  if (buffer_size == 0)
+    return 0;
+
+  /* We have reached end of file already? */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return 0;
+
+  /* Figure out how much data we should read. */
+  const size_t write_size = MIN(buffer_size,
+                                sizeof(win32_process->stdin_handle.buffer));
+
+  /* Read data from the process_t buffer into our intermediate buffer. */
+  buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size);
+
+  /* Because of the slightly weird API for WriteFileEx() we must set this to 0
+   * before we call WriteFileEx() because WriteFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Schedule our write. */
+  ret = WriteFileEx(win32_process->stdin_handle.pipe,
+                    win32_process->stdin_handle.buffer,
+                    write_size,
+                    &win32_process->stdin_handle.overlapped,
+                    process_win32_stdin_write_done);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "WriteFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for WriteFileEx() we
+   * should check GetLastError() after a call to WriteFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* This cast should be safe since our buffer can maximum be BUFFER_SIZE
+   * large. */
+  return (int)write_size;
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard output of the child process
+ * and asks Windows to call process_win32_stdout_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stdout(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stdout_handle,
+                                        buffer,
+                                        process_win32_stdout_read_done);
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard error of the child process
+ * and asks Windows to call process_win32_stderr_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stderr(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stderr_handle,
+                                        buffer,
+                                        process_win32_stderr_read_done);
+}
+
+/** This function is responsible for moving the Tor process into what Microsoft
+ * calls an "alertable" state. Once the process is in an alertable state the
+ * Windows kernel will notify us when our background I/O requests have finished
+ * and the callbacks will be executed. */
+void
+process_win32_trigger_completion_callbacks(void)
+{
+  DWORD ret;
+
+  /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep
+   * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause
+   * the process to enter alertable state, where the Windows kernel will notify
+   * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which
+   * will cause our completion callbacks to be executed.
+   *
+   * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION
+   * if one or more I/O callbacks were executed. */
+  ret = SleepEx(0, TRUE);
+
+  /* Warn us if the function returned something we did not anticipate. */
+  if (ret != 0 && ret != WAIT_IO_COMPLETION) {
+    log_warn(LD_PROCESS, "SleepEx() returned %lu", ret);
+  }
+}
+
+/** Start the periodic timer which is reponsible for checking whether processes
+ * are still alive and to make sure that the Tor process is periodically being
+ * moved into an alertable state. */
+void
+process_win32_timer_start(void)
+{
+  /* Make sure we never start our timer if it's already running. */
+  if (BUG(process_win32_timer_running()))
+    return;
+
+  /* Wake up once a second. */
+  static const struct timeval interval = {1, 0};
+
+  log_info(LD_PROCESS, "Starting Windows Process I/O timer");
+  periodic_timer = periodic_timer_new(tor_libevent_get_base(),
+                                      &interval,
+                                      process_win32_timer_callback,
+                                      NULL);
+}
+
+/** Stops the periodic timer. */
+void
+process_win32_timer_stop(void)
+{
+  if (BUG(periodic_timer == NULL))
+    return;
+
+  log_info(LD_PROCESS, "Stopping Windows Process I/O timer");
+  periodic_timer_free(periodic_timer);
+}
+
+/** Returns true iff the periodic timer is running. */
+bool
+process_win32_timer_running(void)
+{
+  return periodic_timer != NULL;
+}
+
+/** This function is called whenever the periodic_timer ticks. The function is
+ * responsible for moving the Tor process into an alertable state once a second
+ * and checking for whether our child processes have terminated since the last
+ * tick. */
+STATIC void
+process_win32_timer_callback(periodic_timer_t *timer, void *data)
+{
+  tor_assert(timer == periodic_timer);
+  tor_assert(data == NULL);
+
+  /* Move the process into an alertable state. */
+  process_win32_trigger_completion_callbacks();
+
+  /* Check if our processes are still alive. */
+
+  /* Since the call to process_win32_timer_test_process() might call
+   * process_notify_event_exit() which again might call process_free() which
+   * updates the list of processes returned by process_get_all_processes() it
+   * is important here that we make sure to not touch the list of processes if
+   * the call to process_win32_timer_test_process() returns true. */
+  bool done;
+
+  do {
+    const smartlist_t *processes = process_get_all_processes();
+    done = true;
+
+    SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) {
+      /* If process_win32_timer_test_process() returns true, it means that
+       * smartlist_remove() might have been called on the list returned by
+       * process_get_all_processes(). We start the loop over again until we
+       * have a succesful run over the entire list where the list was not
+       * modified. */
+      if (process_win32_timer_test_process(process)) {
+        done = false;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(process);
+  } while (! done);
+}
+
+/** Test whether a given process is still alive. Notify the Process subsystem
+ * if our process have died. Returns true iff the given process have
+ * terminated. */
+STATIC bool
+process_win32_timer_test_process(process_t *process)
+{
+  tor_assert(process);
+
+  /* No need to look at processes that don't claim they are running. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD exit_code = 0;
+
+  /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error
+   * message until some time after the process have actually terminated. We
+   * make sure that our ReadFileEx() calls for the process have *all* returned
+   * and both standard out and error have been marked as EOF before we try to
+   * see if the process terminated.
+   *
+   * This ensures that we *never* call the exit callback of the `process_t`,
+   * which potentially ends up calling `process_free()` on our `process_t`,
+   * before all data have been received from the process.
+   *
+   * We do NOT have a check here for whether standard in reached EOF since
+   * standard in's WriteFileEx() function is only called on-demand when we have
+   * something to write and is thus usually not awaiting to finish any
+   * operations. If we WriteFileEx() to a file that has terminated we'll simply
+   * get an error from ReadFileEx() or its completion routine and move on with
+   * life.  */
+  if (! win32_process->stdout_handle.reached_eof)
+    return false;
+
+  if (! win32_process->stderr_handle.reached_eof)
+    return false;
+
+  /* We start by testing whether our process is still running. */
+  ret = GetExitCodeProcess(win32_process->process_information.hProcess,
+                           &exit_code);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Notify our process_t that our process have terminated. Since our
+   * exit_callback might decide to process_free() our process handle it is very
+   * important that we do not touch the process_t after the call to
+   * process_notify_event_exit(). */
+  if (exit_code != STILL_ACTIVE) {
+    process_notify_event_exit(process, exit_code);
+    return true;
+  }
+
+  return false;
+}
+
+/** Create a new overlapped named pipe. This function creates a new connected,
+ * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is
+ * succesful. Returns true on sucess, false on failure. */
+STATIC bool
+process_win32_create_pipe(HANDLE *read_pipe,
+                          HANDLE *write_pipe,
+                          SECURITY_ATTRIBUTES *attributes,
+                          process_win32_pipe_type_t pipe_type)
+{
+  tor_assert(read_pipe);
+  tor_assert(write_pipe);
+  tor_assert(attributes);
+
+  BOOL ret = FALSE;
+
+  /* Buffer size. */
+  const size_t size = 4096;
+
+  /* Our additional read/write modes that depends on which pipe type we are
+   * creating. */
+  DWORD read_mode = 0;
+  DWORD write_mode = 0;
+
+  /* Generate the unique pipe name. */
+  char pipe_name[MAX_PATH];
+  static DWORD process_id = 0;
+  static DWORD counter = 0;
+
+  if (process_id == 0)
+    process_id = GetCurrentProcessId();
+
+  tor_snprintf(pipe_name, sizeof(pipe_name),
+               "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu",
+               process_id, counter++);
+
+  /* Only one of our handles can be overlapped. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    read_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    write_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Setup our read and write handles. */
+  HANDLE read_handle;
+  HANDLE write_handle;
+
+  /* Create our named pipe. */
+  read_handle = CreateNamedPipeA(pipe_name,
+                                 (PIPE_ACCESS_INBOUND|read_mode),
+                                 (PIPE_TYPE_BYTE|PIPE_WAIT),
+                                 1,
+                                 size,
+                                 size,
+                                 1000,
+                                 attributes);
+
+  if (read_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Create our file in the pipe namespace. */
+  write_handle = CreateFileA(pipe_name,
+                             GENERIC_WRITE,
+                             0,
+                             attributes,
+                             OPEN_EXISTING,
+                             (FILE_ATTRIBUTE_NORMAL|write_mode),
+                             NULL);
+
+  if (write_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateFileA() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+
+    return false;
+  }
+
+  /* Set the inherit flag for our pipe. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "SetHandleInformation() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+    CloseHandle(write_handle);
+
+    return false;
+  }
+
+  /* Everything is good. */
+  *read_pipe = read_handle;
+  *write_pipe = write_handle;
+
+  return true;
+}
+
+/** Cleanup a given <b>handle</b>. */
+STATIC void
+process_win32_cleanup_handle(process_win32_handle_t *handle)
+{
+  tor_assert(handle);
+
+#if 0
+  BOOL ret;
+  DWORD error_code;
+
+  /* Cancel any pending I/O requests: This means that instead of getting
+   * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem
+   * like this is needed. */
+  ret = CancelIo(handle->pipe);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* There was no pending I/O requests for our handle. */
+    if (error_code != ERROR_NOT_FOUND) {
+      log_warn(LD_PROCESS, "CancelIo() failed: %s",
+               format_win32_error(error_code));
+    }
+  }
+#endif /* 0 */
+
+  /* Close our handle. */
+  if (handle->pipe != INVALID_HANDLE_VALUE) {
+    CloseHandle(handle->pipe);
+    handle->pipe = INVALID_HANDLE_VALUE;
+    handle->reached_eof = true;
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard output. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stdout_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stdout_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stdout(process);
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard error. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stderr_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stderr_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stderr(process);
+  }
+}
+
+/** This function is called when WriteFileEx() completes its background write
+ * to the child process's standard input. We notify the Process subsystem that
+ * it can write data to us again. */
+STATIC VOID WINAPI
+process_win32_stdin_write_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  (void)byte_count;
+
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  win32_process->stdin_handle.busy = false;
+
+  /* Check if we have been asked to write to the handle that have been marked
+   * as having reached EOF. */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return;
+
+  if (error_code == 0) {
+    /** Our data have been succesfully written. Clear our state and schedule
+     * the next write. */
+    win32_process->stdin_handle.data_available = 0;
+    memset(win32_process->stdin_handle.buffer, 0,
+           sizeof(win32_process->stdin_handle.buffer));
+
+    /* Schedule the next write. */
+    process_notify_event_stdin(process);
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our WriteFileEx() call was succesful, but we reached the end of our
+     * file.  We mark our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    win32_process->stdin_handle.reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from WriteFileEx(): %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+  }
+}
+
+/** This function reads data from the given <b>handle</b>'s internal buffer and
+ * moves it into the given <b>buffer</b>. Additionally, we start the next
+ * ReadFileEx() background operation with the given <b>callback</b> as
+ * completion callback. Returns the number of bytes written to the buffer. */
+STATIC int
+process_win32_read_from_handle(process_win32_handle_t *handle,
+                               buf_t *buffer,
+                               LPOVERLAPPED_COMPLETION_ROUTINE callback)
+{
+  tor_assert(handle);
+  tor_assert(buffer);
+  tor_assert(callback);
+
+  BOOL ret = FALSE;
+  int bytes_available = 0;
+  DWORD error_code = 0;
+
+  /* We already have a request to read data that isn't complete yet. */
+  if (BUG(handle->busy))
+    return 0;
+
+  /* Check if we have been asked to read from a handle that have already told
+   * us that we have reached the end of the file. */
+  if (BUG(handle->reached_eof))
+    return 0;
+
+  /* This cast should be safe since our buffer can be at maximum up to
+   * BUFFER_SIZE in size. */
+  bytes_available = (int)handle->data_available;
+
+  if (handle->data_available > 0) {
+    /* Read data from our intermediate buffer into the process_t buffer. */
+    buf_add(buffer, handle->buffer, handle->data_available);
+
+    /* Reset our read state. */
+    handle->data_available = 0;
+    memset(handle->buffer, 0, sizeof(handle->buffer));
+  }
+
+  /* Because of the slightly weird API for ReadFileEx() we must set this to 0
+   * before we call ReadFileEx() because ReadFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Ask the Windows kernel to read data from our pipe into our buffer and call
+   * the callback function when it is done. */
+  ret = ReadFileEx(handle->pipe,
+                   handle->buffer,
+                   sizeof(handle->buffer),
+                   &handle->overlapped,
+                   callback);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "ReadFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    handle->reached_eof = true;
+    return bytes_available;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for ReadFileEx() we
+   * should check GetLastError() after a call to ReadFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    handle->reached_eof = true;
+    return bytes_available;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* We mark our handle as having a pending I/O request. */
+  handle->busy = true;
+
+  return bytes_available;
+}
+
+/** This function checks the callback values from ReadFileEx() in
+ * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true
+ * iff our caller should request more data from ReadFileEx(). */
+STATIC bool
+process_win32_handle_read_completion(process_win32_handle_t *handle,
+                                     DWORD error_code,
+                                     DWORD byte_count)
+{
+  tor_assert(handle);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  handle->busy = false;
+
+  if (error_code == 0) {
+    /* Our ReadFileEx() call was succesful and there is data for us. */
+
+    /* This cast should be safe since byte_count should never be larger than
+     * BUFFER_SIZE. */
+    tor_assert(byte_count <= BUFFER_SIZE);
+    handle->data_available = (size_t)byte_count;
+
+    /* Tell our caller to schedule the next read. */
+    return true;
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our ReadFileEx() finished, but we reached the end of our file.  We mark
+     * our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    handle->reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from ReadFileEx(): %s",
+             format_win32_error(error_code));
+
+    handle->reached_eof = true;
+  }
+
+  /* Our caller should NOT schedule the next read. */
+  return false;
+}
+
+/** Format a single argument for being put on a Windows command line.
+ * Returns a newly allocated string */
+STATIC char *
+format_win_cmdline_argument(const char *arg)
+{
+  char *formatted_arg;
+  char need_quotes;
+  const char *c;
+  int i;
+  int bs_counter = 0;
+  /* Backslash we can point to when one is inserted into the string */
+  const char backslash = '\\';
+
+  /* Smartlist of *char */
+  smartlist_t *arg_chars;
+  arg_chars = smartlist_new();
+
+  /* Quote string if it contains whitespace or is empty */
+  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
+
+  /* Build up smartlist of *chars */
+  for (c=arg; *c != '\0'; c++) {
+    if ('"' == *c) {
+      /* Double up backslashes preceding a quote */
+      for (i=0; i<(bs_counter*2); i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      /* Escape the quote */
+      smartlist_add(arg_chars, (void*)&backslash);
+      smartlist_add(arg_chars, (void*)c);
+    } else if ('\\' == *c) {
+      /* Count backslashes until we know whether to double up */
+      bs_counter++;
+    } else {
+      /* Don't double up slashes preceding a non-quote */
+      for (i=0; i<bs_counter; i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      smartlist_add(arg_chars, (void*)c);
+    }
+  }
+  /* Don't double up trailing backslashes */
+  for (i=0; i<bs_counter; i++)
+    smartlist_add(arg_chars, (void*)&backslash);
+
+  /* Allocate space for argument, quotes (if needed), and terminator */
+  const size_t formatted_arg_len = smartlist_len(arg_chars) +
+    (need_quotes ? 2 : 0) + 1;
+  formatted_arg = tor_malloc_zero(formatted_arg_len);
+
+  /* Add leading quote */
+  i=0;
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+
+  /* Add characters */
+  SMARTLIST_FOREACH(arg_chars, char*, ch,
+  {
+    formatted_arg[i++] = *ch;
+  });
+
+  /* Add trailing quote */
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+  formatted_arg[i] = '\0';
+
+  smartlist_free(arg_chars);
+  return formatted_arg;
+}
+
+/** Format a command line for use on Windows, which takes the command as a
+ * string rather than string array. Follows the rules from "Parsing C++
+ * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
+ * Python subprocess module. Returns a newly allocated string */
+STATIC char *
+tor_join_win_cmdline(const char *argv[])
+{
+  smartlist_t *argv_list;
+  char *joined_argv;
+  int i;
+
+  /* Format each argument and put the result in a smartlist */
+  argv_list = smartlist_new();
+  for (i=0; argv[i] != NULL; i++) {
+    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
+  }
+
+  /* Join the arguments with whitespace */
+  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
+
+  /* Free the newly allocated arguments, and the smartlist */
+  SMARTLIST_FOREACH(argv_list, char *, arg,
+  {
+    tor_free(arg);
+  });
+  smartlist_free(argv_list);
+
+  return joined_argv;
+}
+
+#endif /* defined(_WIN32) */
diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h
new file mode 100644
index 0000000000..a50d86df5b
--- /dev/null
+++ b/src/lib/process/process_win32.h
@@ -0,0 +1,97 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.h
+ * \brief Header for process_win32.c
+ **/
+
+#ifndef TOR_PROCESS_WIN32_H
+#define TOR_PROCESS_WIN32_H
+
+#ifdef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include <windows.h>
+
+struct process_t;
+
+struct process_win32_t;
+typedef struct process_win32_t process_win32_t;
+
+process_win32_t *process_win32_new(void);
+void process_win32_free_(process_win32_t *win32_process);
+#define process_win32_free(s) \
+  FREE_AND_NULL(process_win32_t, process_win32_free_, (s))
+
+void process_win32_init(void);
+void process_win32_deinit(void);
+
+process_status_t process_win32_exec(struct process_t *process);
+bool process_win32_terminate(struct process_t *process);
+
+process_pid_t process_win32_get_pid(struct process_t *process);
+
+int process_win32_write(struct process_t *process, buf_t *buffer);
+int process_win32_read_stdout(struct process_t *process, buf_t *buffer);
+int process_win32_read_stderr(struct process_t *process, buf_t *buffer);
+
+void process_win32_trigger_completion_callbacks(void);
+
+/* Timer handling. */
+void process_win32_timer_start(void);
+void process_win32_timer_stop(void);
+bool process_win32_timer_running(void);
+
+#ifdef PROCESS_WIN32_PRIVATE
+STATIC void process_win32_timer_callback(periodic_timer_t *, void *);
+STATIC bool process_win32_timer_test_process(process_t *);
+
+/* I/O pipe handling. */
+struct process_win32_handle_t;
+typedef struct process_win32_handle_t process_win32_handle_t;
+
+typedef enum process_win32_pipe_type_t {
+  /** This pipe is used for reading. */
+  PROCESS_WIN32_PIPE_TYPE_READER,
+
+  /** This pipe is used for writing. */
+  PROCESS_WIN32_PIPE_TYPE_WRITER
+} process_win32_pipe_type_t;
+
+STATIC bool process_win32_create_pipe(HANDLE *,
+                                      HANDLE *,
+                                      SECURITY_ATTRIBUTES *,
+                                      process_win32_pipe_type_t);
+
+STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle);
+
+STATIC VOID WINAPI process_win32_stdout_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stderr_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stdin_write_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+
+STATIC int process_win32_read_from_handle(process_win32_handle_t *,
+                                          buf_t *,
+                                          LPOVERLAPPED_COMPLETION_ROUTINE);
+STATIC bool process_win32_handle_read_completion(process_win32_handle_t *,
+                                                 DWORD,
+                                                 DWORD);
+
+STATIC char *format_win_cmdline_argument(const char *arg);
+STATIC char *tor_join_win_cmdline(const char *argv[]);
+#endif /* defined(PROCESS_WIN32_PRIVATE) */
+
+#endif /* defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_WIN32_H) */
diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c
index 534b39d101..fda284f3d9 100644
--- a/src/lib/process/restrict.c
+++ b/src/lib/process/restrict.c
@@ -152,7 +152,7 @@ tor_mlockall(void)
                            "pages: %s", strerror(errno));
     return -1;
   }
-#else /* !(defined(HAVE_UNIX_MLOCKALL)) */
+#else /* !defined(HAVE_UNIX_MLOCKALL) */
   log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?");
   return -1;
 #endif /* defined(HAVE_UNIX_MLOCKALL) */
@@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
     return -1;
   }
   limit = MAX_CONNECTIONS;
-#else /* !(!defined(HAVE_GETRLIMIT)) */
+#else /* defined(HAVE_GETRLIMIT) */
   struct rlimit rlim;
 
   if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c
index 6e8258f279..3c94ce4bac 100644
--- a/src/lib/process/setuid.c
+++ b/src/lib/process/setuid.c
@@ -72,7 +72,7 @@ log_credential_status(void)
            "UID is %u (real), %u (effective), %u (saved)",
            (unsigned)ruid, (unsigned)euid, (unsigned)suid);
   }
-#else /* !(defined(HAVE_GETRESUID)) */
+#else /* !defined(HAVE_GETRESUID) */
   /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */
   ruid = getuid();
   euid = geteuid();
@@ -93,7 +93,7 @@ log_credential_status(void)
            "GID is %u (real), %u (effective), %u (saved)",
            (unsigned)rgid, (unsigned)egid, (unsigned)sgid);
   }
-#else /* !(defined(HAVE_GETRESGID)) */
+#else /* !defined(HAVE_GETRESGID) */
   /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */
   rgid = getgid();
   egid = getegid();
@@ -154,7 +154,7 @@ have_capability_support(void)
     return 0;
   cap_free(caps);
   return 1;
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   return 0;
 #endif /* defined(HAVE_LINUX_CAPABILITIES) */
 }
@@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags)
     if (drop_capabilities(1))
       return -1;
   }
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   (void) keep_bindlow;
   if (warn_if_no_caps) {
     log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support "
@@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags)
 #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */
   return 0;
 
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void)user;
   (void)flags;
 
diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h
index 7d03e1f025..a2125d2d06 100644
--- a/src/lib/process/setuid.h
+++ b/src/lib/process/setuid.h
@@ -19,4 +19,4 @@ int have_capability_support(void);
 #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1)
 int switch_id(const char *user, unsigned flags);
 
-#endif
+#endif /* !defined(TOR_SETUID_H) */
diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c
deleted file mode 100644
index f4429d7f76..0000000000
--- a/src/lib/process/subprocess.c
+++ /dev/null
@@ -1,1236 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.c
- * \brief Launch and monitor other processes.
- **/
-
-#define SUBPROCESS_PRIVATE
-#include "lib/process/subprocess.h"
-
-#include "lib/container/smartlist.h"
-#include "lib/err/torerr.h"
-#include "lib/log/log.h"
-#include "lib/log/util_bug.h"
-#include "lib/log/win32err.h"
-#include "lib/malloc/malloc.h"
-#include "lib/process/env.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/compat_ctype.h"
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_PRCTL_H
-#include <sys/prctl.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-#include <errno.h>
-#include <string.h>
-
-/** Format a single argument for being put on a Windows command line.
- * Returns a newly allocated string */
-static char *
-format_win_cmdline_argument(const char *arg)
-{
-  char *formatted_arg;
-  char need_quotes;
-  const char *c;
-  int i;
-  int bs_counter = 0;
-  /* Backslash we can point to when one is inserted into the string */
-  const char backslash = '\\';
-
-  /* Smartlist of *char */
-  smartlist_t *arg_chars;
-  arg_chars = smartlist_new();
-
-  /* Quote string if it contains whitespace or is empty */
-  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
-
-  /* Build up smartlist of *chars */
-  for (c=arg; *c != '\0'; c++) {
-    if ('"' == *c) {
-      /* Double up backslashes preceding a quote */
-      for (i=0; i<(bs_counter*2); i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      /* Escape the quote */
-      smartlist_add(arg_chars, (void*)&backslash);
-      smartlist_add(arg_chars, (void*)c);
-    } else if ('\\' == *c) {
-      /* Count backslashes until we know whether to double up */
-      bs_counter++;
-    } else {
-      /* Don't double up slashes preceding a non-quote */
-      for (i=0; i<bs_counter; i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      smartlist_add(arg_chars, (void*)c);
-    }
-  }
-  /* Don't double up trailing backslashes */
-  for (i=0; i<bs_counter; i++)
-    smartlist_add(arg_chars, (void*)&backslash);
-
-  /* Allocate space for argument, quotes (if needed), and terminator */
-  const size_t formatted_arg_len = smartlist_len(arg_chars) +
-    (need_quotes ? 2 : 0) + 1;
-  formatted_arg = tor_malloc_zero(formatted_arg_len);
-
-  /* Add leading quote */
-  i=0;
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-
-  /* Add characters */
-  SMARTLIST_FOREACH(arg_chars, char*, ch,
-  {
-    formatted_arg[i++] = *ch;
-  });
-
-  /* Add trailing quote */
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-  formatted_arg[i] = '\0';
-
-  smartlist_free(arg_chars);
-  return formatted_arg;
-}
-
-/** Format a command line for use on Windows, which takes the command as a
- * string rather than string array. Follows the rules from "Parsing C++
- * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
- * Python subprocess module. Returns a newly allocated string */
-char *
-tor_join_win_cmdline(const char *argv[])
-{
-  smartlist_t *argv_list;
-  char *joined_argv;
-  int i;
-
-  /* Format each argument and put the result in a smartlist */
-  argv_list = smartlist_new();
-  for (i=0; argv[i] != NULL; i++) {
-    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
-  }
-
-  /* Join the arguments with whitespace */
-  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
-
-  /* Free the newly allocated arguments, and the smartlist */
-  SMARTLIST_FOREACH(argv_list, char *, arg,
-  {
-    tor_free(arg);
-  });
-  smartlist_free(argv_list);
-
-  return joined_argv;
-}
-
-#ifndef _WIN32
-/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in
- * <b>hex_errno</b>.  Called between fork and _exit, so must be signal-handler
- * safe.
- *
- * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available.
- *
- * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded
- * with spaces. CHILD_STATE indicates where
- * in the process of starting the child process did the failure occur (see
- * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of
- * errno when the failure occurred.
- *
- * On success return the number of characters added to hex_errno, not counting
- * the terminating NUL; return -1 on error.
- */
-STATIC int
-format_helper_exit_status(unsigned char child_state, int saved_errno,
-                          char *hex_errno)
-{
-  unsigned int unsigned_errno;
-  int written, left;
-  char *cur;
-  size_t i;
-  int res = -1;
-
-  /* Fill hex_errno with spaces, and a trailing newline (memset may
-     not be signal handler safe, so we can't use it) */
-  for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++)
-    hex_errno[i] = ' ';
-  hex_errno[HEX_ERRNO_SIZE - 1] = '\n';
-
-  /* Convert errno to be unsigned for hex conversion */
-  if (saved_errno < 0) {
-    // Avoid overflow on the cast to unsigned int when result is INT_MIN
-    // by adding 1 to the signed int negative value,
-    // then, after it has been negated and cast to unsigned,
-    // adding the original 1 back (the double-addition is intentional).
-    // Otherwise, the cast to signed could cause a temporary int
-    // to equal INT_MAX + 1, which is undefined.
-    unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1;
-  } else {
-    unsigned_errno = (unsigned int) saved_errno;
-  }
-
-  /*
-   * Count how many chars of space we have left, and keep a pointer into the
-   * current point in the buffer.
-   */
-  left = HEX_ERRNO_SIZE+1;
-  cur = hex_errno;
-
-  /* Emit child_state */
-  written = format_hex_number_sigsafe(child_state, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-  if (left <= 0)
-    goto err;
-
-  /* Now the '/' */
-  *cur = '/';
-
-  /* Adjust left and cur */
-  ++cur;
-  --left;
-  if (left <= 0)
-    goto err;
-
-  /* Need minus? */
-  if (saved_errno < 0) {
-    *cur = '-';
-    ++cur;
-    --left;
-    if (left <= 0)
-      goto err;
-  }
-
-  /* Emit unsigned_errno */
-  written = format_hex_number_sigsafe(unsigned_errno, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-
-  /* Check that we have enough space left for a newline and a NUL */
-  if (left <= 1)
-    goto err;
-
-  /* Emit the newline and NUL */
-  *cur++ = '\n';
-  *cur++ = '\0';
-
-  res = (int)(cur - hex_errno - 1);
-
-  goto done;
-
- err:
-  /*
-   * In error exit, just write a '\0' in the first char so whatever called
-   * this at least won't fall off the end.
-   */
-  *hex_errno = '\0';
-
- done:
-  return res;
-}
-#endif /* !defined(_WIN32) */
-
-/* Maximum number of file descriptors, if we cannot get it via sysconf() */
-#define DEFAULT_MAX_FD 256
-
-/** Terminate the process of <b>process_handle</b>, if that process has not
- * already exited.
- *
- * Return 0 if we succeeded in terminating the process (or if the process
- * already exited), and -1 if we tried to kill the process but failed.
- *
- * Based on code originally borrowed from Python's os.kill. */
-int
-tor_terminate_process(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) {
-    HANDLE handle = process_handle->pid.hProcess;
-
-    if (!TerminateProcess(handle, 0))
-      return -1;
-    else
-      return 0;
-  }
-#else /* !(defined(_WIN32)) */
-  if (process_handle->waitpid_cb) {
-    /* We haven't got a waitpid yet, so we can just kill off the process. */
-    return kill(process_handle->pid, SIGTERM);
-  }
-#endif /* defined(_WIN32) */
-
-  return 0; /* We didn't need to kill the process, so report success */
-}
-
-/** Return the Process ID of <b>process_handle</b>. */
-int
-tor_process_get_pid(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  return (int) process_handle->pid.dwProcessId;
-#else
-  return (int) process_handle->pid;
-#endif
-}
-
-#ifdef _WIN32
-HANDLE
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#else /* !(defined(_WIN32)) */
-/* DOCDOC tor_process_get_stdout_pipe */
-int
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#endif /* defined(_WIN32) */
-
-/* DOCDOC process_handle_new */
-static process_handle_t *
-process_handle_new(void)
-{
-  process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t));
-
-#ifdef _WIN32
-  out->stdin_pipe = INVALID_HANDLE_VALUE;
-  out->stdout_pipe = INVALID_HANDLE_VALUE;
-  out->stderr_pipe = INVALID_HANDLE_VALUE;
-#else
-  out->stdin_pipe = -1;
-  out->stdout_pipe = -1;
-  out->stderr_pipe = -1;
-#endif /* defined(_WIN32) */
-
-  return out;
-}
-
-#ifndef _WIN32
-/** Invoked when a process that we've launched via tor_spawn_background() has
- * been found to have terminated.
- */
-static void
-process_handle_waitpid_cb(int status, void *arg)
-{
-  process_handle_t *process_handle = arg;
-
-  process_handle->waitpid_exit_status = status;
-  clear_waitpid_callback(process_handle->waitpid_cb);
-  if (process_handle->status == PROCESS_STATUS_RUNNING)
-    process_handle->status = PROCESS_STATUS_NOTRUNNING;
-  process_handle->waitpid_cb = 0;
-}
-#endif /* !defined(_WIN32) */
-
-/**
- * @name child-process states
- *
- * Each of these values represents a possible state that a child process can
- * be in.  They're used to determine what to say when telling the parent how
- * far along we were before failure.
- *
- * @{
- */
-#define CHILD_STATE_INIT 0
-#define CHILD_STATE_PIPE 1
-#define CHILD_STATE_MAXFD 2
-#define CHILD_STATE_FORK 3
-#define CHILD_STATE_DUPOUT 4
-#define CHILD_STATE_DUPERR 5
-#define CHILD_STATE_DUPIN 6
-#define CHILD_STATE_CLOSEFD 7
-#define CHILD_STATE_EXEC 8
-#define CHILD_STATE_FAILEXEC 9
-/** @} */
-/**
- * Boolean.  If true, then Tor may call execve or CreateProcess via
- * tor_spawn_background.
- **/
-static int may_spawn_background_process = 1;
-/**
- * Turn off may_spawn_background_process, so that all future calls to
- * tor_spawn_background are guaranteed to fail.
- **/
-void
-tor_disable_spawning_background_processes(void)
-{
-  may_spawn_background_process = 0;
-}
-/** Start a program in the background. If <b>filename</b> contains a '/', then
- * it will be treated as an absolute or relative path.  Otherwise, on
- * non-Windows systems, the system path will be searched for <b>filename</b>.
- * On Windows, only the current directory will be searched. Here, to search the
- * system path (as well as the application directory, current working
- * directory, and system directories), set filename to NULL.
- *
- * The strings in <b>argv</b> will be passed as the command line arguments of
- * the child program (following convention, argv[0] should normally be the
- * filename of the executable, and this must be the case if <b>filename</b> is
- * NULL). The last element of argv must be NULL. A handle to the child process
- * will be returned in process_handle (which must be non-NULL). Read
- * process_handle.status to find out if the process was successfully launched.
- * For convenience, process_handle.status is returned by this function.
- *
- * Some parts of this code are based on the POSIX subprocess module from
- * Python, and example code from
- * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx.
- */
-int
-tor_spawn_background(const char *const filename, const char **argv,
-                     process_environment_t *env,
-                     process_handle_t **process_handle_out)
-{
-  if (BUG(may_spawn_background_process == 0)) {
-    /* We should never reach this point if we're forbidden to spawn
-     * processes. Instead we should have caught the attempt earlier. */
-    return PROCESS_STATUS_ERROR;
-  }
-
-#ifdef _WIN32
-  HANDLE stdout_pipe_read = NULL;
-  HANDLE stdout_pipe_write = NULL;
-  HANDLE stderr_pipe_read = NULL;
-  HANDLE stderr_pipe_write = NULL;
-  HANDLE stdin_pipe_read = NULL;
-  HANDLE stdin_pipe_write = NULL;
-  process_handle_t *process_handle;
-  int status;
-
-  STARTUPINFOA siStartInfo;
-  BOOL retval = FALSE;
-
-  SECURITY_ATTRIBUTES saAttr;
-  char *joined_argv;
-
-  saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
-  saAttr.bInheritHandle = TRUE;
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-  saAttr.lpSecurityDescriptor = NULL;
-
-  /* Assume failure to start process */
-  status = PROCESS_STATUS_ERROR;
-
-  /* Set up pipe for stdout */
-  if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdout communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdout communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stderr */
-  if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stderr communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stderr communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stdin */
-  if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdin communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdin communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Create the child process */
-
-  /* Windows expects argv to be a whitespace delimited string, so join argv up
-   */
-  joined_argv = tor_join_win_cmdline(argv);
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-
-  ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION));
-  ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
-  siStartInfo.cb = sizeof(STARTUPINFO);
-  siStartInfo.hStdError = stderr_pipe_write;
-  siStartInfo.hStdOutput = stdout_pipe_write;
-  siStartInfo.hStdInput = stdin_pipe_read;
-  siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
-
-  /* Create the child process */
-
-  retval = CreateProcessA(filename,      // module name
-                 joined_argv,   // command line
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-                 NULL,          // process security attributes
-                 NULL,          // primary thread security attributes
-                 TRUE,          // handles are inherited
-  /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess()
-   * work?) */
-                 CREATE_NO_WINDOW,             // creation flags
-                 (env==NULL) ? NULL : env->windows_environment_block,
-                 NULL,          // use parent's current directory
-                 &siStartInfo,  // STARTUPINFO pointer
-                 &(process_handle->pid));  // receives PROCESS_INFORMATION
-
-  tor_free(joined_argv);
-
-  if (!retval) {
-    log_warn(LD_GENERAL,
-      "Failed to create child process %s: %s", filename?filename:argv[0],
-      format_win32_error(GetLastError()));
-    tor_free(process_handle);
-  } else  {
-    /* TODO: Close hProcess and hThread in process_handle->pid? */
-    process_handle->stdout_pipe = stdout_pipe_read;
-    process_handle->stderr_pipe = stderr_pipe_read;
-    process_handle->stdin_pipe = stdin_pipe_write;
-    status = process_handle->status = PROCESS_STATUS_RUNNING;
-  }
-
-  /* TODO: Close pipes on exit */
-  *process_handle_out = process_handle;
-  return status;
-#else /* !(defined(_WIN32)) */
-  pid_t pid;
-  int stdout_pipe[2];
-  int stderr_pipe[2];
-  int stdin_pipe[2];
-  int fd, retval;
-  process_handle_t *process_handle;
-  int status;
-
-  const char *error_message = SPAWN_ERROR_MESSAGE;
-  size_t error_message_length;
-
-  /* Represents where in the process of spawning the program is;
-     this is used for printing out the error message */
-  unsigned char child_state = CHILD_STATE_INIT;
-
-  char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */
-
-  static int max_fd = -1;
-
-  status = PROCESS_STATUS_ERROR;
-
-  /* We do the strlen here because strlen() is not signal handler safe,
-     and we are not allowed to use unsafe functions between fork and exec */
-  error_message_length = strlen(error_message);
-
-  // child_state = CHILD_STATE_PIPE;
-
-  /* Set up pipe for redirecting stdout, stderr, and stdin of child */
-  retval = pipe(stdout_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdout communication with child process: %s",
-       strerror(errno));
-    return status;
-  }
-
-  retval = pipe(stderr_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stderr communication with child process: %s",
-      strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-
-    return status;
-  }
-
-  retval = pipe(stdin_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdin communication with child process: %s",
-       strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-
-    return status;
-  }
-
-  // child_state = CHILD_STATE_MAXFD;
-
-#ifdef _SC_OPEN_MAX
-  if (-1 == max_fd) {
-    max_fd = (int) sysconf(_SC_OPEN_MAX);
-    if (max_fd == -1) {
-      max_fd = DEFAULT_MAX_FD;
-      log_warn(LD_GENERAL,
-               "Cannot find maximum file descriptor, assuming %d", max_fd);
-    }
-  }
-#else /* !(defined(_SC_OPEN_MAX)) */
-  max_fd = DEFAULT_MAX_FD;
-#endif /* defined(_SC_OPEN_MAX) */
-
-  // child_state = CHILD_STATE_FORK;
-
-  pid = fork();
-  if (0 == pid) {
-    /* In child */
-
-#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
-    /* Attempt to have the kernel issue a SIGTERM if the parent
-     * goes away. Certain attributes of the binary being execve()ed
-     * will clear this during the execve() call, but it's better
-     * than nothing.
-     */
-    prctl(PR_SET_PDEATHSIG, SIGTERM);
-#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
-
-    child_state = CHILD_STATE_DUPOUT;
-
-    /* Link child stdout to the write end of the pipe */
-    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPERR;
-
-    /* Link child stderr to the write end of the pipe */
-    retval = dup2(stderr_pipe[1], STDERR_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPIN;
-
-    /* Link child stdin to the read end of the pipe */
-    retval = dup2(stdin_pipe[0], STDIN_FILENO);
-    if (-1 == retval)
-      goto error;
-
-    // child_state = CHILD_STATE_CLOSEFD;
-
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-
-    /* Close all other fds, including the read end of the pipe */
-    /* XXX: We should now be doing enough FD_CLOEXEC setting to make
-     * this needless. */
-    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) {
-      close(fd);
-    }
-
-    // child_state = CHILD_STATE_EXEC;
-
-    /* Call the requested program. We need the cast because
-       execvp doesn't define argv as const, even though it
-       does not modify the arguments */
-    if (env)
-      execve(filename, (char *const *) argv, env->unixoid_environment_block);
-    else {
-      static char *new_env[] = { NULL };
-      execve(filename, (char *const *) argv, new_env);
-    }
-
-    /* If we got here, the exec or open(/dev/null) failed */
-
-    child_state = CHILD_STATE_FAILEXEC;
-
-  error:
-    {
-      /* XXX: are we leaking fds from the pipe? */
-      int n, err=0;
-      ssize_t nbytes;
-
-      n = format_helper_exit_status(child_state, errno, hex_errno);
-
-      if (n >= 0) {
-        /* Write the error message. GCC requires that we check the return
-           value, but there is nothing we can do if it fails */
-        /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */
-        nbytes = write(STDOUT_FILENO, error_message, error_message_length);
-        err = (nbytes < 0);
-        nbytes = write(STDOUT_FILENO, hex_errno, n);
-        err += (nbytes < 0);
-      }
-
-      _exit(err?254:255); // exit ok: in child.
-    }
-
-    /* Never reached, but avoids compiler warning */
-    return status; // LCOV_EXCL_LINE
-  }
-
-  /* In parent */
-
-  if (-1 == pid) {
-    log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno));
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    return status;
-  }
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-  process_handle->pid = pid;
-
-  /* TODO: If the child process forked but failed to exec, waitpid it */
-
-  /* Return read end of the pipes to caller, and close write end */
-  process_handle->stdout_pipe = stdout_pipe[0];
-  retval = close(stdout_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stdout pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  process_handle->waitpid_cb = set_waitpid_callback(pid,
-                                                    process_handle_waitpid_cb,
-                                                    process_handle);
-
-  process_handle->stderr_pipe = stderr_pipe[0];
-  retval = close(stderr_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stderr pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  /* Return write end of the stdin pipe to caller, and close the read end */
-  process_handle->stdin_pipe = stdin_pipe[1];
-  retval = close(stdin_pipe[0]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close read end of stdin pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  status = process_handle->status = PROCESS_STATUS_RUNNING;
-  /* Set stdin/stdout/stderr pipes to be non-blocking */
-  if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) {
-    log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
-             "nonblocking in parent process: %s", strerror(errno));
-  }
-
-  *process_handle_out = process_handle;
-  return status;
-#endif /* defined(_WIN32) */
-}
-
-/** Destroy all resources allocated by the process handle in
- *  <b>process_handle</b>.
- *  If <b>also_terminate_process</b> is true, also terminate the
- *  process of the process handle. */
-MOCK_IMPL(void,
-tor_process_handle_destroy,(process_handle_t *process_handle,
-                            int also_terminate_process))
-{
-  if (!process_handle)
-    return;
-
-  if (also_terminate_process) {
-    if (tor_terminate_process(process_handle) < 0) {
-      const char *errstr =
-#ifdef _WIN32
-        format_win32_error(GetLastError());
-#else
-        strerror(errno);
-#endif
-      log_notice(LD_GENERAL, "Failed to terminate process with "
-                 "PID '%d' ('%s').", tor_process_get_pid(process_handle),
-                 errstr);
-    } else {
-      log_info(LD_GENERAL, "Terminated process with PID '%d'.",
-               tor_process_get_pid(process_handle));
-    }
-  }
-
-  process_handle->status = PROCESS_STATUS_NOTRUNNING;
-
-#ifdef _WIN32
-  if (process_handle->stdout_pipe)
-    CloseHandle(process_handle->stdout_pipe);
-
-  if (process_handle->stderr_pipe)
-    CloseHandle(process_handle->stderr_pipe);
-
-  if (process_handle->stdin_pipe)
-    CloseHandle(process_handle->stdin_pipe);
-#else /* !(defined(_WIN32)) */
-  close(process_handle->stdout_pipe);
-  close(process_handle->stderr_pipe);
-  close(process_handle->stdin_pipe);
-
-  clear_waitpid_callback(process_handle->waitpid_cb);
-#endif /* defined(_WIN32) */
-
-  memset(process_handle, 0x0f, sizeof(process_handle_t));
-  tor_free(process_handle);
-}
-
-/** Get the exit code of a process specified by <b>process_handle</b> and store
- * it in <b>exit_code</b>, if set to a non-NULL value.  If <b>block</b> is set
- * to true, the call will block until the process has exited.  Otherwise if
- * the process is still running, the function will return
- * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns
- * PROCESS_EXIT_EXITED if the process did exit. If there is a failure,
- * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if
- * non-NULL) will be undefined. N.B. Under *nix operating systems, this will
- * probably not work in Tor, because waitpid() is called in main.c to reap any
- * terminated child processes.*/
-int
-tor_get_exit_code(process_handle_t *process_handle,
-                  int block, int *exit_code)
-{
-#ifdef _WIN32
-  DWORD retval;
-  BOOL success;
-
-  if (block) {
-    /* Wait for the process to exit */
-    retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE);
-    if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-              (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  } else {
-    retval = WaitForSingleObject(process_handle->pid.hProcess, 0);
-    if (WAIT_TIMEOUT == retval) {
-      /* Process has not exited */
-      return PROCESS_EXIT_RUNNING;
-    } else if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-               (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-
-  if (exit_code != NULL) {
-    success = GetExitCodeProcess(process_handle->pid.hProcess,
-                                 (PDWORD)exit_code);
-    if (!success) {
-      log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s",
-               format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-#else /* !(defined(_WIN32)) */
-  int stat_loc;
-  int retval;
-
-  if (process_handle->waitpid_cb) {
-    /* We haven't processed a SIGCHLD yet. */
-    retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG);
-    if (retval == process_handle->pid) {
-      clear_waitpid_callback(process_handle->waitpid_cb);
-      process_handle->waitpid_cb = NULL;
-      process_handle->waitpid_exit_status = stat_loc;
-    }
-  } else {
-    /* We already got a SIGCHLD for this process, and handled it. */
-    retval = process_handle->pid;
-    stat_loc = process_handle->waitpid_exit_status;
-  }
-
-  if (!block && 0 == retval) {
-    /* Process has not exited */
-    return PROCESS_EXIT_RUNNING;
-  } else if (retval != process_handle->pid) {
-    log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s",
-             (int)process_handle->pid, strerror(errno));
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (!WIFEXITED(stat_loc)) {
-    log_warn(LD_GENERAL, "Process %d did not exit normally",
-             (int)process_handle->pid);
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (exit_code != NULL)
-    *exit_code = WEXITSTATUS(stat_loc);
-#endif /* defined(_WIN32) */
-
-  return PROCESS_EXIT_EXITED;
-}
-
-#ifdef _WIN32
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>hProcess</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle
- * to the process owning the <b>h</b>. In this case, the function will exit
- * only once the process has exited, or <b>count</b> bytes are read. Returns
- * the number of bytes read, or -1 on error. */
-ssize_t
-tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                    const process_handle_t *process)
-{
-  size_t numread = 0;
-  BOOL retval;
-  DWORD byte_count;
-  BOOL process_exited = FALSE;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    /* Check if there is anything to read */
-    retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
-    if (!retval) {
-      log_warn(LD_GENERAL,
-        "Failed to peek from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* Nothing available: process exited or it is busy */
-
-      /* Exit if we don't know whether the process is running */
-      if (NULL == process)
-        break;
-
-      /* The process exited and there's nothing left to read from it */
-      if (process_exited)
-        break;
-
-      /* If process is not running, check for output one more time in case
-         it wrote something after the peek was performed. Otherwise keep on
-         waiting for output */
-      tor_assert(process != NULL);
-      byte_count = WaitForSingleObject(process->pid.hProcess, 0);
-      if (WAIT_TIMEOUT != byte_count)
-        process_exited = TRUE;
-
-      continue;
-    }
-
-    /* There is data to read; read it */
-    retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL);
-    tor_assert(byte_count + numread <= count);
-    if (!retval) {
-      log_warn(LD_GENERAL, "Failed to read from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* End of file */
-      break;
-    }
-    numread += byte_count;
-  }
-  return (ssize_t)numread;
-}
-#else /* !(defined(_WIN32)) */
-/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>process</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise data will be read until end of file, or
- * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
- * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
- * file has been reached. */
-ssize_t
-tor_read_all_handle(int fd, char *buf, size_t count,
-                    const process_handle_t *process,
-                    int *eof)
-{
-  size_t numread = 0;
-  ssize_t result;
-
-  if (eof)
-    *eof = 0;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    result = read(fd, buf+numread, count-numread);
-
-    if (result == 0) {
-      log_debug(LD_GENERAL, "read() reached end of file");
-      if (eof)
-        *eof = 1;
-      break;
-    } else if (result < 0 && errno == EAGAIN) {
-      if (process)
-        continue;
-      else
-        break;
-    } else if (result < 0) {
-      log_warn(LD_GENERAL, "read() failed: %s", strerror(errno));
-      return -1;
-    }
-
-    numread += result;
-  }
-
-  log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread);
-  return (ssize_t)numread;
-}
-#endif /* defined(_WIN32) */
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stdout(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stderr(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Return a string corresponding to <b>stream_status</b>. */
-const char *
-stream_status_to_string(enum stream_status stream_status)
-{
-  switch (stream_status) {
-    case IO_STREAM_OKAY:
-      return "okay";
-    case IO_STREAM_EAGAIN:
-      return "temporarily unavailable";
-    case IO_STREAM_TERM:
-      return "terminated";
-    case IO_STREAM_CLOSED:
-      return "closed";
-    default:
-      tor_fragile_assert();
-      return "unknown";
-  }
-}
-
-/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be
- * modified. The resulting smartlist will consist of pointers to buf, so there
- * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated
- * string. <b>len</b> should be set to the length of the buffer excluding the
- * NUL. Non-printable characters (including NUL) will be replaced with "." */
-int
-tor_split_lines(smartlist_t *sl, char *buf, int len)
-{
-  /* Index in buf of the start of the current line */
-  int start = 0;
-  /* Index in buf of the current character being processed */
-  int cur = 0;
-  /* Are we currently in a line */
-  char in_line = 0;
-
-  /* Loop over string */
-  while (cur < len) {
-    /* Loop until end of line or end of string */
-    for (; cur < len; cur++) {
-      if (in_line) {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* End of line */
-          buf[cur] = '\0';
-          /* Point cur to the next line */
-          cur++;
-          /* Line starts at start and ends with a nul */
-          break;
-        } else {
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      } else {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* Skip leading vertical space */
-          ;
-        } else {
-          in_line = 1;
-          start = cur;
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      }
-    }
-    /* We are at the end of the line or end of string. If in_line is true there
-     * is a line which starts at buf+start and ends at a NUL. cur points to
-     * the character after the NUL. */
-    if (in_line)
-      smartlist_add(sl, (void *)(buf+start));
-    in_line = 0;
-  }
-  return smartlist_len(sl);
-}
-
-#ifdef _WIN32
-
-/** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (HANDLE *handle,
-                            enum stream_status *stream_status_out))
-{
-  int pos;
-  char stdout_buf[600] = {0};
-  smartlist_t *lines = NULL;
-
-  tor_assert(stream_status_out);
-
-  *stream_status_out = IO_STREAM_TERM;
-
-  pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL);
-  if (pos < 0) {
-    *stream_status_out = IO_STREAM_TERM;
-    return NULL;
-  }
-  if (pos == 0) {
-    *stream_status_out = IO_STREAM_EAGAIN;
-    return NULL;
-  }
-
-  /* End with a null even if there isn't a \r\n at the end */
-  /* TODO: What if this is a partial line? */
-  stdout_buf[pos] = '\0';
-
-  /* Split up the buffer */
-  lines = smartlist_new();
-  tor_split_lines(lines, stdout_buf, pos);
-
-  /* Currently 'lines' is populated with strings residing on the
-     stack. Replace them with their exact copies on the heap: */
-  SMARTLIST_FOREACH(lines, char *, line,
-                    SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line)));
-
-  *stream_status_out = IO_STREAM_OKAY;
-
-  return lines;
-}
-
-#else /* !(defined(_WIN32)) */
-
-/** Return a smartlist containing lines outputted from
- *  <b>fd</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
-{
-  enum stream_status stream_status;
-  char stdout_buf[400];
-  smartlist_t *lines = NULL;
-
-  while (1) {
-    memset(stdout_buf, 0, sizeof(stdout_buf));
-
-    stream_status = get_string_from_pipe(fd,
-                                         stdout_buf, sizeof(stdout_buf) - 1);
-    if (stream_status != IO_STREAM_OKAY)
-      goto done;
-
-    if (!lines) lines = smartlist_new();
-    smartlist_split_string(lines, stdout_buf, "\n", 0, 0);
-  }
-
- done:
-  *stream_status_out = stream_status;
-  return lines;
-}
-
-#endif /* defined(_WIN32) */
-
-/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
- *  sure it's below <b>count</b> bytes.
- *  If the string has a trailing newline, we strip it off.
- *
- * This function is specifically created to handle input from managed
- * proxies, according to the pluggable transports spec. Make sure it
- * fits your needs before using it.
- *
- * Returns:
- * IO_STREAM_CLOSED: If the stream is closed.
- * IO_STREAM_EAGAIN: If there is nothing to read and we should check back
- *  later.
- * IO_STREAM_TERM: If something is wrong with the stream.
- * IO_STREAM_OKAY: If everything went okay and we got a string
- *  in <b>buf_out</b>. */
-enum stream_status
-get_string_from_pipe(int fd, char *buf_out, size_t count)
-{
-  ssize_t ret;
-
-  tor_assert(count <= INT_MAX);
-
-  ret = read(fd, buf_out, count);
-
-  if (ret == 0)
-    return IO_STREAM_CLOSED;
-  else if (ret < 0 && errno == EAGAIN)
-    return IO_STREAM_EAGAIN;
-  else if (ret < 0)
-    return IO_STREAM_TERM;
-
-  if (buf_out[ret - 1] == '\n') {
-    /* Remove the trailing newline */
-    buf_out[ret - 1] = '\0';
-  } else
-    buf_out[ret] = '\0';
-
-  return IO_STREAM_OKAY;
-}
diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h
deleted file mode 100644
index aa3127d62d..0000000000
--- a/src/lib/process/subprocess.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.h
- * \brief Header for subprocess.c
- **/
-
-#ifndef TOR_SUBPROCESS_H
-#define TOR_SUBPROCESS_H
-
-#include "lib/cc/torint.h"
-#include "lib/testsupport/testsupport.h"
-#include <stddef.h>
-#ifdef _WIN32
-#include <windows.h>
-#endif
-
-struct smartlist_t;
-
-void tor_disable_spawning_background_processes(void);
-
-typedef struct process_handle_t process_handle_t;
-struct process_environment_t;
-int tor_spawn_background(const char *const filename, const char **argv,
-                         struct process_environment_t *env,
-                         process_handle_t **process_handle_out);
-
-#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code "
-
-/** Status of an I/O stream. */
-enum stream_status {
-  IO_STREAM_OKAY,
-  IO_STREAM_EAGAIN,
-  IO_STREAM_TERM,
-  IO_STREAM_CLOSED
-};
-
-const char *stream_status_to_string(enum stream_status stream_status);
-
-enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
-
-/* Values of process_handle_t.status. */
-#define PROCESS_STATUS_NOTRUNNING 0
-#define PROCESS_STATUS_RUNNING 1
-#define PROCESS_STATUS_ERROR -1
-
-#ifdef SUBPROCESS_PRIVATE
-struct waitpid_callback_t;
-
-/** Structure to represent the state of a process with which Tor is
- * communicating. The contents of this structure are private to util.c */
-struct process_handle_t {
-  /** One of the PROCESS_STATUS_* values */
-  int status;
-#ifdef _WIN32
-  HANDLE stdin_pipe;
-  HANDLE stdout_pipe;
-  HANDLE stderr_pipe;
-  PROCESS_INFORMATION pid;
-#else /* !(defined(_WIN32)) */
-  int stdin_pipe;
-  int stdout_pipe;
-  int stderr_pipe;
-  pid_t pid;
-  /** If the process has not given us a SIGCHLD yet, this has the
-   * waitpid_callback_t that gets invoked once it has. Otherwise this
-   * contains NULL. */
-  struct waitpid_callback_t *waitpid_cb;
-  /** The exit status reported by waitpid. */
-  int waitpid_exit_status;
-#endif /* defined(_WIN32) */
-};
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-/* Return values of tor_get_exit_code() */
-#define PROCESS_EXIT_RUNNING 1
-#define PROCESS_EXIT_EXITED 0
-#define PROCESS_EXIT_ERROR -1
-int tor_get_exit_code(process_handle_t *process_handle,
-                      int block, int *exit_code);
-int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
-#ifdef _WIN32
-ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                            const process_handle_t *process);
-#else
-ssize_t tor_read_all_handle(int fd, char *buf, size_t count,
-                            const process_handle_t *process,
-                            int *eof);
-#endif /* defined(_WIN32) */
-ssize_t tor_read_all_from_process_stdout(
-    const process_handle_t *process_handle, char *buf, size_t count);
-ssize_t tor_read_all_from_process_stderr(
-    const process_handle_t *process_handle, char *buf, size_t count);
-char *tor_join_win_cmdline(const char *argv[]);
-
-int tor_process_get_pid(process_handle_t *process_handle);
-#ifdef _WIN32
-HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#else
-int tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#endif
-
-#ifdef _WIN32
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle,
-                           enum stream_status *stream_status));
-#else
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd,
-                                       enum stream_status *stream_status));
-#endif /* defined(_WIN32) */
-
-int tor_terminate_process(process_handle_t *process_handle);
-
-MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle,
-                                            int also_terminate_process));
-
-#ifdef SUBPROCESS_PRIVATE
-/* Prototypes for private functions only used by util.c (and unit tests) */
-
-#ifndef _WIN32
-STATIC int format_helper_exit_status(unsigned char child_state,
-                              int saved_errno, char *hex_errno);
-
-/* Space for hex values of child state, a slash, saved_errno (with
-   leading minus) and newline (no null) */
-#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \
-                        1 + sizeof(int) * 2 + 1)
-#endif /* !defined(_WIN32) */
-
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-#endif
diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c
index 9b626394d2..2b38481aeb 100644
--- a/src/lib/process/waitpid.c
+++ b/src/lib/process/waitpid.c
@@ -16,7 +16,7 @@
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
-#include "ht.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c
new file mode 100644
index 0000000000..ad65886422
--- /dev/null
+++ b/src/lib/process/winprocess_sys.c
@@ -0,0 +1,66 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.c
+ * \brief Subsystem object for windows process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/process/winprocess_sys.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef _WIN32
+#include <windows.h>
+
+#define WINPROCESS_SYS_ENABLED true
+
+static int
+subsys_winprocess_initialize(void)
+{
+#ifndef HeapEnableTerminationOnCorruption
+#define HeapEnableTerminationOnCorruption 1
+#endif
+
+  /* On heap corruption, just give up; don't try to play along. */
+  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
+
+  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
+   * (On 64-bit Windows it always fails, and some compilers don't like the
+   * PSETDEP cast.)
+   * 32-bit Windows defines _WIN32.
+   * 64-bit Windows defines _WIN32 and _WIN64. */
+#ifndef _WIN64
+  /* Call SetProcessDEPPolicy to permanently enable DEP.
+     The function will not resolve on earlier versions of Windows,
+     and failure is not dangerous. */
+  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
+  if (hMod) {
+    typedef BOOL (WINAPI *PSETDEP)(DWORD);
+    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
+                           "SetProcessDEPPolicy");
+    if (setdeppolicy) {
+      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
+      setdeppolicy(3);
+    }
+  }
+#endif /* !defined(_WIN64) */
+
+  return 0;
+}
+#else /* !defined(_WIN32) */
+#define WINPROCESS_SYS_ENABLED false
+#define subsys_winprocess_initialize NULL
+#endif /* defined(_WIN32) */
+
+const subsys_fns_t sys_winprocess = {
+  .name = "winprocess",
+  /* HeapEnableTerminationOnCorruption and setdeppolicy() are security
+   * features, we want them to run first. */
+  .level = -100,
+  .supported = WINPROCESS_SYS_ENABLED,
+  .initialize = subsys_winprocess_initialize,
+};
diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h
new file mode 100644
index 0000000000..7ab2aa04a6
--- /dev/null
+++ b/src/lib/process/winprocess_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.h
+ * \brief Declare subsystem object for winprocess.c
+ **/
+
+#ifndef TOR_WINPROCESS_SYS_H
+#define TOR_WINPROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_winprocess;
+
+#endif /* !defined(TOR_WINPROCESS_SYS_H) */
diff --git a/src/lib/pubsub/.may_include b/src/lib/pubsub/.may_include
new file mode 100644
index 0000000000..5623492f00
--- /dev/null
+++ b/src/lib/pubsub/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/pubsub/*.h
+lib/string/*.h
diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am
new file mode 100644
index 0000000000..e2abebcd40
--- /dev/null
+++ b/src/lib/pubsub/include.am
@@ -0,0 +1,28 @@
+
+noinst_LIBRARIES += src/lib/libtor-pubsub.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_pubsub_a_SOURCES =			\
+	src/lib/pubsub/pubsub_build.c			\
+	src/lib/pubsub/pubsub_check.c			\
+	src/lib/pubsub/pubsub_publish.c
+
+src_lib_libtor_pubsub_testing_a_SOURCES = \
+	$(src_lib_libtor_pubsub_a_SOURCES)
+src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/pubsub/pub_binding_st.h			\
+	src/lib/pubsub/pubsub.h				\
+	src/lib/pubsub/pubsub_build.h			\
+	src/lib/pubsub/pubsub_builder_st.h		\
+	src/lib/pubsub/pubsub_connect.h			\
+	src/lib/pubsub/pubsub_flags.h			\
+	src/lib/pubsub/pubsub_macros.h			\
+	src/lib/pubsub/pubsub_publish.h
diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h
new file mode 100644
index 0000000000..d841bf3f54
--- /dev/null
+++ b/src/lib/pubsub/pub_binding_st.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pub_binding_st.h
+ * @brief Declaration of pub_binding_t.
+ *
+ * This is an internal type for the pubsub implementation.
+ */
+
+#ifndef TOR_PUB_BINDING_ST_H
+#define TOR_PUB_BINDING_ST_H
+
+#include "lib/dispatch/msgtypes.h"
+struct dispatch_t;
+
+/**
+ * A pub_binding_t is an opaque object that subsystems use to publish
+ * messages.  The DISPATCH_ADD_PUB*() macros set it up.
+ **/
+typedef struct pub_binding_t {
+  /**
+   * A pointer to a configured dispatch_t object.  This is filled in
+   * when the dispatch_t is finally constructed.
+   **/
+  struct dispatch_t *dispatch_ptr;
+  /**
+   * A template for the msg_t fields that are filled in for this message.
+   * This is copied into outgoing messages, ensuring that their fields are set
+   * corretly.
+   **/
+  msg_t msg_template;
+} pub_binding_t;
+
+#endif /* !defined(TOR_PUB_BINDING_ST_H) */
diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h
new file mode 100644
index 0000000000..5346b07517
--- /dev/null
+++ b/src/lib/pubsub/pubsub.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub.h
+ * @brief Header for OO publish-subscribe functionality.
+ *
+ * This module provides a wrapper around the "dispatch" module,
+ * ensuring type-safety and allowing us to do static analysis on
+ * publication and subscriptions.
+ *
+ * With this module, we enforce:
+ *  <ul>
+ *   <li>that every message has (potential) publishers and subscribers;
+ *   <li>that every message is published and subscribed from the correct
+ *       channels, with the correct type ID, every time it is published.
+ *   <li>that type IDs correspond to a single C type, and that the C types are
+ *       used correctly.
+ *   <li>that when a message is published or subscribed, it is done with
+ *       a correct subsystem identifier
+ * </ul>
+ *
+ * We do this by making "publication requests" and "subscription requests"
+ * into objects, and doing some computation on them before we create
+ * a dispatch_t with them.
+ *
+ * Rather than using the dispatch module directly, a publishing module
+ * receives a "binding" object that it uses to send messages with the right
+ * settings.
+ *
+ * Most users of this module will want to use this header, and the
+ * pubsub_macros.h header for convenience.
+ */
+
+/*
+ *
+ * Overview: Messages are sent over channels.  Before sending a message on a
+ * channel, or receiving a message on a channel, a subsystem needs to register
+ * that it publishes, or subscribes, to that message, on that channel.
+ *
+ * Messages, channels, and subsystems are represented internally as short
+ * integers, though they are associated with human-readable strings for
+ * initialization and debugging.
+ *
+ * When registering for a message, a subsystem must say whether it is an
+ * exclusive publisher/subscriber to that message type, or whether other
+ * subsystems may also publish/subscribe to it.
+ *
+ * All messages and their publishers/subscribers must be registered early in
+ * the initialization process.
+ *
+ * By default, it is an error for a message type to have publishers and no
+ * subscribers on a channel, or subscribers and no publishers on a channel.
+ *
+ * A subsystem may register for a message with a note that delivery or
+ * production is disabled -- for example, because the subsystem is
+ * disabled at compile-time. It is not an error for a message type to
+ * have all of its publishers or subscribers disabled.
+ *
+ * After a message is sent, it is delivered to every recipient.  This
+ * delivery happens from the top level of the event loop; it may be
+ * interleaved with network events, timers, etc.
+ *
+ * Messages may have associated data.  This data is typed, and is owned
+ * by the message.  Strings, byte-arrays, and integers have built-in
+ * support.  Other types may be added.  If objects are to be sent,
+ * they should be identified by handle.  If an object requires cleanup,
+ * it should be declared with an associated free function.
+ *
+ * Semantically, if two subsystems communicate only by this kind of
+ * message passing, neither is considered to depend on the other, though
+ * both are considered to have a dependency on the message and on any
+ * types it contains.
+ *
+ * (Or generational index?)
+ */
+#ifndef TOR_PUBSUB_PUBSUB_H
+#define TOR_PUBSUB_PUBSUB_H
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#endif /* !defined(TOR_PUBSUB_PUBSUB_H) */
diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c
new file mode 100644
index 0000000000..e44b7d76ec
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.c
@@ -0,0 +1,307 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.c
+ * @brief Construct a dispatch_t in safer, more OO way.
+ **/
+
+#define PUBSUB_PRIVATE
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+ #include <string.h>
+
+/** Construct and return a new empty pubsub_items_t. */
+static pubsub_items_t *
+pubsub_items_new(void)
+{
+  pubsub_items_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->items = smartlist_new();
+  cfg->type_items = smartlist_new();
+  return cfg;
+}
+
+/** Release all storage held in a pubsub_items_t. */
+void
+pubsub_items_free_(pubsub_items_t *cfg)
+{
+  if (! cfg)
+    return;
+  SMARTLIST_FOREACH(cfg->items, pubsub_cfg_t *, item, tor_free(item));
+  SMARTLIST_FOREACH(cfg->type_items,
+                    pubsub_type_cfg_t *, item, tor_free(item));
+  smartlist_free(cfg->items);
+  smartlist_free(cfg->type_items);
+  tor_free(cfg);
+}
+
+/** Construct and return a new pubsub_builder_t. */
+pubsub_builder_t *
+pubsub_builder_new(void)
+{
+  dispatch_naming_init();
+
+  pubsub_builder_t *pb = tor_malloc_zero(sizeof(*pb));
+  pb->cfg = dcfg_new();
+  pb->items = pubsub_items_new();
+  return pb;
+}
+
+/**
+ * Release all storage held by a pubsub_builder_t.
+ *
+ * You'll (mostly) only want to call this function on an error case: if you're
+ * constructing a dispatch_t instead, you should call
+ * pubsub_builder_finalize() to consume the pubsub_builder_t.
+ */
+void
+pubsub_builder_free_(pubsub_builder_t *pb)
+{
+  if (pb == NULL)
+    return;
+  pubsub_items_free(pb->items);
+  dcfg_free(pb->cfg);
+  tor_free(pb);
+}
+
+/**
+ * Create and return a pubsub_connector_t for the subsystem with ID
+ * <b>subsys</b> to use in adding publications, subscriptions, and types to
+ * <b>builder</b>.
+ **/
+pubsub_connector_t *
+pubsub_connector_for_subsystem(pubsub_builder_t *builder,
+                               subsys_id_t subsys)
+{
+  tor_assert(builder);
+  ++builder->n_connectors;
+
+  pubsub_connector_t *con = tor_malloc_zero(sizeof(*con));
+
+  con->builder = builder;
+  con->subsys_id = subsys;
+
+  return con;
+}
+
+/**
+ * Release all storage held by a pubsub_connector_t.
+ **/
+void
+pubsub_connector_free_(pubsub_connector_t *con)
+{
+  if (!con)
+    return;
+
+  if (con->builder) {
+    --con->builder->n_connectors;
+    tor_assert(con->builder->n_connectors >= 0);
+  }
+  tor_free(con);
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>.
+ **/
+int
+pubsub_add_pub_(pubsub_connector_t *con,
+                pub_binding_t *out,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  memset(out, 0, sizeof(*out));
+  cfg->is_publish = true;
+
+  out->msg_template.sender = cfg->subsys = con->subsys_id;
+  out->msg_template.channel = cfg->channel = channel;
+  out->msg_template.msg = cfg->msg = msg;
+  out->msg_template.type = cfg->type = type;
+
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  /* We're grabbing a pointer to the pub_binding_t so we can tell it about
+   * the dispatcher later on.
+   */
+  cfg->pub_binding = out;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>,
+ * passing them to the callback in <b>recv_fn</b>.
+ **/
+int
+pubsub_add_sub_(pubsub_connector_t *con,
+                recv_fn_t recv_fn,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  cfg->is_publish = false;
+  cfg->subsys = con->subsys_id;
+  cfg->channel = channel;
+  cfg->msg = msg;
+  cfg->type = type;
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  cfg->recv_fn = recv_fn;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+  if (! (flags & DISP_FLAG_STUB)) {
+    if (dcfg_add_recv(con->builder->cfg, msg, cfg->subsys, recv_fn) < 0)
+      goto err;
+  }
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to define the functions to use for manipulating the type
+ * <b>type</b>.  Any function pointers left as NULL will be implemented as
+ * no-ops.
+ **/
+int
+pubsub_connector_register_type_(pubsub_connector_t *con,
+                                msg_type_id_t type,
+                                dispatch_typefns_t *fns,
+                                const char *file,
+                                unsigned line)
+{
+  pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->type = type;
+  memcpy(&cfg->fns, fns, sizeof(*fns));
+  cfg->subsys = con->subsys_id;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  smartlist_add(con->builder->items->type_items, cfg);
+
+  if (dcfg_type_set_fns(con->builder->cfg, type, fns) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Initialize the dispatch_ptr field in every relevant publish binding
+ * for <b>d</b>.
+ */
+static void
+pubsub_items_install_bindings(pubsub_items_t *items,
+                              dispatch_t *d)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      // XXXX we could skip this for STUB publishers, and for any publishers
+      // XXXX where all subscribers are STUB.
+      cfg->pub_binding->dispatch_ptr = d;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Remove the dispatch_ptr fields for all the relevant publish bindings
+ * in <b>items</b>.  The prevents subsequent dispatch_pub_() calls from
+ * sending messages to a dispatcher that has been freed.
+ **/
+void
+pubsub_items_clear_bindings(pubsub_items_t *items)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      cfg->pub_binding->dispatch_ptr = NULL;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Create a new dispatcher as configured in a pubsub_builder_t.
+ *
+ * Consumes and frees its input.
+ **/
+dispatch_t *
+pubsub_builder_finalize(pubsub_builder_t *builder,
+                        pubsub_items_t **items_out)
+{
+  dispatch_t *dispatcher = NULL;
+  tor_assert_nonfatal(builder->n_connectors == 0);
+
+  if (pubsub_builder_check(builder) < 0)
+    goto err;
+
+  if (builder->n_errors) {
+    log_warn(LD_GENERAL, "At least one error occurred previously when "
+             "configuring the dispatcher.");
+    goto err;
+  }
+
+  dispatcher = dispatch_new(builder->cfg);
+
+  if (!dispatcher)
+    goto err;
+
+  pubsub_items_install_bindings(builder->items, dispatcher);
+  if (items_out) {
+    *items_out = builder->items;
+    builder->items = NULL; /* Prevent free */
+  }
+
+ err:
+  pubsub_builder_free(builder);
+  return dispatcher;
+}
diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h
new file mode 100644
index 0000000000..5a0c5f5bd3
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.h
@@ -0,0 +1,92 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.h
+ * @brief Header used for constructing the OO publish-subscribe facility.
+ *
+ * (See pubsub.h for more general information on this API.)
+ **/
+
+#ifndef TOR_PUBSUB_BUILD_H
+#define TOR_PUBSUB_BUILD_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct dispatch_t;
+struct pubsub_connector_t;
+
+/**
+ * A "dispatch builder" is an incomplete dispatcher, used when
+ * registering messages.  It does not have the same integrity guarantees
+ * as a dispatcher.  It cannot actually handle messages itself: once all
+ * subsystems have registered, it is converted into a dispatch_t.
+ **/
+typedef struct pubsub_builder_t pubsub_builder_t;
+
+/**
+ * A "pubsub items" holds the configuration items used to configure a
+ * pubsub_builder.  After the builder is finalized, this field is extracted,
+ * and used later to tear down pointers that enable publishing.
+ **/
+typedef struct pubsub_items_t pubsub_items_t;
+
+/**
+ * Create a new pubsub_builder. This should only happen in the
+ * main-init code.
+ */
+pubsub_builder_t *pubsub_builder_new(void);
+
+/** DOCDOC */
+int pubsub_builder_check(pubsub_builder_t *);
+
+/**
+ * Free a pubsub builder.  This should only happen on error paths, where
+ * we have decided not to construct a dispatcher for some reason.
+ */
+#define pubsub_builder_free(db) \
+  FREE_AND_NULL(pubsub_builder_t, pubsub_builder_free_, (db))
+
+/** Internal implementation of pubsub_builder_free(). */
+void pubsub_builder_free_(pubsub_builder_t *);
+
+/**
+ * Create a pubsub connector that a single subsystem will use to
+ * register its messages.  The main-init code does this during susbsystem
+ * initialization.
+ */
+struct pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *,
+                                                          subsys_id_t);
+
+/**
+ * The main-init code does this after subsystem initialization.
+ */
+#define pubsub_connector_free(c) \
+  FREE_AND_NULL(struct pubsub_connector_t, pubsub_connector_free_, (c))
+
+void pubsub_connector_free_(struct pubsub_connector_t *);
+
+/**
+ * Constructs a dispatcher from a dispatch_builder, after checking that the
+ * invariances on the messages, channels, and connections have been
+ * respected.
+ *
+ * This should happen after every subsystem has initialized, and before
+ * entering the mainloop.
+ */
+struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *,
+                                           pubsub_items_t **items_out);
+
+/**
+ * Clear all pub_binding_t backpointers in <b>items</b>.
+ **/
+void pubsub_items_clear_bindings(pubsub_items_t *items);
+
+#define pubsub_items_free(cfg) \
+  FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg))
+void pubsub_items_free_(pubsub_items_t *cfg);
+
+#endif /* !defined(TOR_PUBSUB_BUILD_H) */
diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h
new file mode 100644
index 0000000000..545aa3f3ef
--- /dev/null
+++ b/src/lib/pubsub/pubsub_builder_st.h
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_builder_st.h
+ *
+ * @brief private structures used for configuring dispatchers and messages.
+ */
+
+#ifndef TOR_PUBSUB_BUILDER_ST_H
+#define TOR_PUBSUB_BUILDER_ST_H
+
+#ifdef PUBSUB_PRIVATE
+
+#include <stdbool.h>
+#include <stddef.h>
+
+struct dispatch_cfg_t;
+struct smartlist_t;
+struct pub_binding_t;
+
+/**
+ * Configuration for a single publication or subscription request.
+ *
+ * These can be stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ *
+ * There are various fields in this request describing the message; all of
+ * them must match other descriptions of the message, or a bug has occurred.
+ **/
+typedef struct pubsub_cfg_t {
+  /** True if this is a publishing request; false for a subscribing request. */
+  bool is_publish;
+  /** The system making this request. */
+  subsys_id_t subsys;
+  /** The channel on which the message is to be sent. */
+  channel_id_t channel;
+  /** The message ID to be sent or received. */
+  message_id_t msg;
+  /** The C type associated with the message. */
+  msg_type_id_t type;
+  /** One or more DISP_FLAGS_* items, combined with bitwise OR. */
+  unsigned flags;
+
+  /**
+   * Publishing only: a pub_binding object that will receive the binding for
+   * this request.  We will finish filling this in when the dispatcher is
+   * constructed, so that the subsystem can publish then and not before.
+   */
+  struct pub_binding_t *pub_binding;
+
+  /**
+   * Subscribing only: a function to receive message objects for this request.
+   */
+  recv_fn_t recv_fn;
+
+  /** The file from which this message was configured */
+  const char *added_by_file;
+  /** The line at which this message was configured */
+  unsigned added_by_line;
+} pubsub_cfg_t;
+
+/**
+ * Configuration request for a single C type.
+ *
+ * These are stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ **/
+typedef struct pubsub_type_cfg_t {
+  /**
+   * The identifier for this type.
+   */
+  msg_type_id_t type;
+  /**
+   * Functions to use when manipulating the type.
+   */
+  dispatch_typefns_t fns;
+
+  /** The subsystem that configured this type. */
+  subsys_id_t subsys;
+  /** The file from which this type was configured */
+  const char *added_by_file;
+  /** The line at which this type was configured */
+  unsigned added_by_line;
+} pubsub_type_cfg_t;
+
+/**
+ * The set of configuration requests for a dispatcher, as made by various
+ * subsystems.
+ **/
+struct pubsub_items_t {
+  /** List of pubsub_cfg_t. */
+  struct smartlist_t *items;
+  /** List of pubsub_type_cfg_t. */
+  struct smartlist_t *type_items;
+};
+
+/**
+ * Type used to construct a dispatcher.  We use this type to build up the
+ * configuration for a dispatcher, and then pass ownership of that
+ * configuration to the newly constructed dispatcher.
+ **/
+struct pubsub_builder_t {
+  /** Number of outstanding pubsub_connector_t objects pointing to this
+   * pubsub_builder_t. */
+  int n_connectors;
+  /** Number of errors encountered while constructing this object so far. */
+  int n_errors;
+  /** In-progress configuration that we're constructing, as a list of the
+   * requests that have been made. */
+  struct pubsub_items_t *items;
+  /** In-progress configuration that we're constructing, in a form that can
+   * be converted to a dispatch_t. */
+  struct dispatch_cfg_t *cfg;
+};
+
+/**
+ * Type given to a subsystem when adding connections to a pubsub_builder_t.
+ * We use this type to force each subsystem to get blamed for the
+ * publications, subscriptions, and types that it adds.
+ **/
+struct pubsub_connector_t {
+  /** The pubsub_builder that this connector refers to. */
+  struct pubsub_builder_t *builder;
+  /** The subsystem that has been given this connector. */
+  subsys_id_t subsys_id;
+};
+
+/**
+ * Helper structure used when constructing a dispatcher that sorts the
+ * pubsub_cfg_t objects in various ways.
+ **/
+typedef struct pubsub_adjmap_t {
+  /* XXXX The next three fields are currently constructed but not yet
+   * XXXX used. I believe we'll want them in the future, though. -nickm
+   */
+  /** Number of subsystems; length of the *_by_subsys arrays. */
+  size_t n_subsystems;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **pub_by_subsys;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **sub_by_subsys;
+
+  /** Number of message IDs; length of the *_by_msg arrays. */
+  size_t n_msgs;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **pub_by_msg;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **sub_by_msg;
+} pubsub_adjmap_t;
+
+#endif /* defined(PUBSUB_PRIVATE) */
+
+#endif /* !defined(TOR_PUBSUB_BUILDER_ST_H) */
diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c
new file mode 100644
index 0000000000..bf1196df2c
--- /dev/null
+++ b/src/lib/pubsub/pubsub_check.c
@@ -0,0 +1,412 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_check.c
+ * @brief Enforce various requirements on a pubsub_builder.
+ **/
+
+#define PUBSUB_PRIVATE
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_build.h"
+
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_string.h"
+
+#include <string.h>
+
+static void pubsub_adjmap_add(pubsub_adjmap_t *map,
+                                const pubsub_cfg_t *item);
+
+/**
+ * Helper: contruct and return a new pubsub_adjacency_map from <b>cfg</b>.
+ * Return NULL on error.
+ **/
+static pubsub_adjmap_t *
+pubsub_build_adjacency_map(const pubsub_items_t *cfg)
+{
+  pubsub_adjmap_t *map = tor_malloc_zero(sizeof(*map));
+  const size_t n_subsystems = get_num_subsys_ids();
+  const size_t n_msgs = get_num_message_ids();
+
+  map->n_subsystems = n_subsystems;
+  map->n_msgs = n_msgs;
+
+  map->pub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->sub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->pub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+  map->sub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+
+  SMARTLIST_FOREACH_BEGIN(cfg->items, const pubsub_cfg_t *, item) {
+    pubsub_adjmap_add(map, item);
+  } SMARTLIST_FOREACH_END(item);
+
+  return map;
+}
+
+/**
+ * Helper: add a single pubsub_cfg_t to an adjacency map.
+ **/
+static void
+pubsub_adjmap_add(pubsub_adjmap_t *map,
+                  const pubsub_cfg_t *item)
+{
+  smartlist_t **by_subsys;
+  smartlist_t **by_msg;
+
+  tor_assert(item->subsys < map->n_subsystems);
+  tor_assert(item->msg < map->n_msgs);
+
+  if (item->is_publish) {
+    by_subsys = &map->pub_by_subsys[item->subsys];
+    by_msg = &map->pub_by_msg[item->msg];
+  } else {
+    by_subsys = &map->sub_by_subsys[item->subsys];
+    by_msg = &map->sub_by_msg[item->msg];
+  }
+
+  if (! *by_subsys)
+    *by_subsys = smartlist_new();
+  if (! *by_msg)
+    *by_msg = smartlist_new();
+  smartlist_add(*by_subsys, (void*) item);
+  smartlist_add(*by_msg, (void *) item);
+}
+
+/**
+ * Release all storage held by m and set m to NULL.
+ **/
+#define pubsub_adjmap_free(m) \
+  FREE_AND_NULL(pubsub_adjmap_t, pubsub_adjmap_free_, m)
+
+/**
+ * Free every element of an <b>n</b>-element array of smartlists, then
+ * free the array itself.
+ **/
+static void
+pubsub_adjmap_free_helper(smartlist_t **lsts, size_t n)
+{
+  if (!lsts)
+    return;
+
+  for (unsigned i = 0; i < n; ++i) {
+    smartlist_free(lsts[i]);
+  }
+  tor_free(lsts);
+}
+
+/**
+ * Release all storage held by <b>map</b>.
+ **/
+static void
+pubsub_adjmap_free_(pubsub_adjmap_t *map)
+{
+  if (!map)
+    return;
+  pubsub_adjmap_free_helper(map->pub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->sub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->pub_by_msg, map->n_msgs);
+  pubsub_adjmap_free_helper(map->sub_by_msg, map->n_msgs);
+  tor_free(map);
+}
+
+/**
+ * Helper: return the length of <b>sl</b>, or 0 if sl is NULL.
+ **/
+static int
+smartlist_len_opt(const smartlist_t *sl)
+{
+  if (sl)
+    return smartlist_len(sl);
+  else
+    return 0;
+}
+
+/** Return a pointer to a statically allocated string encoding the
+ * dispatcher flags in <b>flags</b>. */
+static const char *
+format_flags(unsigned flags)
+{
+  static char buf[32];
+  buf[0] = 0;
+  if (flags & DISP_FLAG_EXCL) {
+    strlcat(buf, " EXCL", sizeof(buf));
+  }
+  if (flags & DISP_FLAG_STUB) {
+    strlcat(buf, " STUB", sizeof(buf));
+  }
+  return buf[0] ? buf+1 : buf;
+}
+
+/**
+ * Log a message containing a description of <b>cfg</b> at severity, prefixed
+ * by the string <b>prefix</b>.
+ */
+static void
+pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix)
+{
+  tor_assert(prefix);
+
+  tor_log(severity, LD_MESG,
+          "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]",
+          prefix,
+          get_subsys_id_name(cfg->subsys),
+          cfg->is_publish ? "PUB" : "SUB",
+          get_message_id_name(cfg->msg),
+          get_msg_type_id_name(cfg->type),
+          get_channel_id_name(cfg->channel),
+          format_flags(cfg->flags),
+          cfg->subsys, cfg->msg, cfg->type, cfg->channel, cfg->flags,
+          cfg->added_by_file, cfg->added_by_line);
+}
+
+/**
+ * Helper: fill a bitarray <b>out</b> with entries corresponding to the
+ * subsystems listed in <b>items</b>.
+ **/
+static void
+get_message_bitarray(const pubsub_adjmap_t *map,
+                     const smartlist_t *items,
+                     bitarray_t **out)
+{
+  *out = bitarray_init_zero((unsigned)map->n_subsystems);
+  if (! items)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) {
+    bitarray_set(*out, cfg->subsys);
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists obey our local graph topology rules.
+ *
+ * (Right now this is just a matter of "each subsystem only
+ * publishes/subscribes once; no subsystem is a publisher and subscriber for
+ * the same message.")
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+lint_message_graph(const pubsub_adjmap_t *map,
+                   message_id_t msg,
+                   const smartlist_t *pub,
+                   const smartlist_t *sub)
+{
+  bitarray_t *published_by = NULL;
+  bitarray_t *subscribed_by = NULL;
+  bool ok = true;
+
+  get_message_bitarray(map, pub, &published_by);
+  get_message_bitarray(map, sub, &subscribed_by);
+
+  /* Check whether any subsystem is publishing and subscribing the same
+   * message. [??]
+   */
+  for (unsigned i = 0; i < map->n_subsystems; ++i) {
+    if (bitarray_is_set(published_by, i) &&
+        bitarray_is_set(subscribed_by, i)) {
+      log_warn(LD_MESG|LD_BUG,
+               "Message \"%s\" is published and subscribed by the same "
+               "subsystem \"%s\".",
+               get_message_id_name(msg),
+               get_subsys_id_name(i));
+      ok = false;
+    }
+  }
+
+  bitarray_free(published_by);
+  bitarray_free(subscribed_by);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists have compatible flags, channels, and types.
+ **/
+static int
+lint_message_consistency(message_id_t msg,
+                         const smartlist_t *pub,
+                         const smartlist_t *sub)
+{
+  if (!smartlist_len_opt(pub) && !smartlist_len_opt(sub))
+    return 0; // LCOV_EXCL_LINE -- this was already checked.
+
+  /* The 'all' list has the publishers and the subscribers. */
+  smartlist_t *all = smartlist_new();
+  if (pub)
+    smartlist_add_all(all, pub);
+  if (sub)
+    smartlist_add_all(all, sub);
+
+  const pubsub_cfg_t *item0 = smartlist_get(all, 0);
+
+  /* Indicates which subsystems we've found publishing/subscribing here. */
+  bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true;
+
+  /* Simple message consistency properties across messages.
+   */
+  SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) {
+    chan_same &= (cfg->channel == item0->channel);
+    type_same &= (cfg->type == item0->type);
+    if (cfg->is_publish)
+      pub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+    else
+      sub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+  } SMARTLIST_FOREACH_END(cfg);
+
+  bool ok = true;
+
+  if (! chan_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "channels.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (! type_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "message types.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Enforce exclusive-ness for publishers and subscribers that have asked for
+   * it.
+   */
+  if (pub_excl && smartlist_len_opt(pub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple publishers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (sub_excl && smartlist_len_opt(sub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple subscribers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  smartlist_free(all);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check whether there are any errors or inconsistencies for the message
+ * described by <b>msg</b> in <b>map</b>.  If there are problems, log about
+ * them, and return -1.  Otherwise return 0.
+ **/
+static int
+lint_message(const pubsub_adjmap_t *map, message_id_t msg)
+{
+  /* NOTE: Some of the checks in this function are maybe over-zealous, and we
+   * might not want to have them forever.  I've marked them with [?] below.
+   */
+  if (BUG(msg >= map->n_msgs))
+    return 0; // LCOV_EXCL_LINE
+
+  const smartlist_t *pub = map->pub_by_msg[msg];
+  const smartlist_t *sub = map->sub_by_msg[msg];
+
+  const size_t n_pub = smartlist_len_opt(pub);
+  const size_t n_sub = smartlist_len_opt(sub);
+
+  if (n_pub == 0 && n_sub == 0) {
+    log_info(LD_MESG, "Nobody is publishing or subscribing to message "
+             "\"%s\".",
+             get_message_id_name(msg));
+    return 0; // No publishers or subscribers: nothing to do.
+  }
+  /* We'll set this to false if there are any problems. */
+  bool ok = true;
+
+  /* First make sure that if there are publishers, there are subscribers. */
+  if (n_pub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has subscribers, but no publishers.",
+             get_message_id_name(msg));
+    ok = false;
+  } else if (n_sub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has publishers, but no subscribers.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Check the message graph topology. */
+  if (lint_message_graph(map, msg, pub, sub) < 0)
+    ok = false;
+
+  /* Check whether the messages have the same fields set on them. */
+  if (lint_message_consistency(msg, pub, sub) < 0)
+    ok = false;
+
+  if (!ok) {
+    /* There was a problem -- let's log all the publishers and subscribers on
+     * this message */
+    if (pub) {
+      SMARTLIST_FOREACH(pub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+    if (sub) {
+      SMARTLIST_FOREACH(sub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+  }
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check all the messages in <b>map</b> for consistency.  Return 0 on success,
+ * -1 on problems.
+ **/
+static int
+pubsub_adjmap_check(const pubsub_adjmap_t *map)
+{
+  bool all_ok = true;
+  for (unsigned i = 0; i < map->n_msgs; ++i) {
+    if (lint_message(map, i) < 0) {
+      all_ok = false;
+    }
+  }
+  return all_ok ? 0 : -1;
+}
+
+/**
+ * Check builder for consistency and various constraints. Return 0 on success,
+ * -1 on failure.
+ **/
+int
+pubsub_builder_check(pubsub_builder_t *builder)
+{
+  pubsub_adjmap_t *map = pubsub_build_adjacency_map(builder->items);
+  int rv = -1;
+
+  if (!map)
+    goto err; // should be impossible
+
+  if (pubsub_adjmap_check(map) < 0)
+    goto err;
+
+  rv = 0;
+ err:
+  pubsub_adjmap_free(map);
+  return rv;
+}
diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h
new file mode 100644
index 0000000000..0ad106044e
--- /dev/null
+++ b/src/lib/pubsub/pubsub_connect.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_connect.h
+ * @brief Header for functions that add relationships to a pubsub builder.
+ *
+ * These functions are used by modules that need to add publication and
+ * subscription requests.  Most users will want to call these functions
+ * indirectly, via the macros in pubsub_macros.h.
+ **/
+
+#ifndef TOR_PUBSUB_CONNECT_H
+#define TOR_PUBSUB_CONNECT_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct pub_binding_t;
+/**
+ * A "dispatch connector" is a view of the dispatcher that a subsystem
+ * uses while initializing itself.  It is specific to the subsystem, and
+ * ensures that each subsystem doesn't need to identify itself
+ * repeatedly while registering its messages.
+ **/
+typedef struct pubsub_connector_t pubsub_connector_t;
+
+int pubsub_add_pub_(struct pubsub_connector_t *con,
+                    struct pub_binding_t *out,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_add_sub_(struct pubsub_connector_t *con,
+                    recv_fn_t recv_fn,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_connector_register_type_(struct pubsub_connector_t *,
+                                    msg_type_id_t,
+                                    dispatch_typefns_t *,
+                                    const char *file,
+                                    unsigned line);
+
+#endif /* !defined(TOR_PUBSUB_CONNECT_H) */
diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h
new file mode 100644
index 0000000000..53c6e49565
--- /dev/null
+++ b/src/lib/pubsub/pubsub_flags.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_flags.h
+ * @brief Flags that can be set on publish/subscribe messages.
+ **/
+
+#ifndef TOR_PUBSUB_FLAGS_H
+#define TOR_PUBSUB_FLAGS_H
+
+/**
+ * Flag for registering a message: declare that no other module is allowed to
+ * publish this message if we are publishing it, or subscribe to it if we are
+ * subscribing to it.
+ */
+#define DISP_FLAG_EXCL (1u<<0)
+
+/**
+ * Flag for registering a message: declare that this message is a stub, and we
+ * will not actually publish/subscribe it, but that the dispatcher should
+ * treat us as if we did when typechecking.
+ *
+ * We use this so that messages aren't treated as "dangling" if they are
+ * potentially used by some other build of Tor.
+ */
+#define DISP_FLAG_STUB (1u<<1)
+
+#endif /* !defined(TOR_PUBSUB_FLAGS_H) */
diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h
new file mode 100644
index 0000000000..357e59fd54
--- /dev/null
+++ b/src/lib/pubsub/pubsub_macros.h
@@ -0,0 +1,373 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pubsub_macros.h
+ * \brief Macros to help with the publish/subscribe dispatch API.
+ *
+ * The dispatch API allows different subsystems of Tor to communicate with
+ * another asynchronously via a shared "message" system.  Some subsystems
+ * declare that they publish a given message, and others declare that they
+ * subscribe to it.  Both subsystems depend on the message, but not upon one
+ * another.
+ *
+ * To declare a message, use DECLARE_MESSAGE() (for messages that take their
+ * data as a pointer) or DECLARE_MESSAGE_INT() (for messages that take their
+ * data as an integer.  For example, you might say
+ *
+ *     DECLARE_MESSAGE(new_circuit, circ, circuit_handle_t *);
+ * or
+ *     DECLARE_MESSAGE_INT(shutdown_requested, boolean, bool);
+ *
+ * Every message has a unique name, a "type name" that the dispatch system
+ * uses to manage associated data, and a C type name.  You'll want to put
+ * these declarations in a header, to be included by all publishers and all
+ * subscribers.
+ *
+ * When a subsystem wants to publish a message, it uses DECLARE_PUBLISH() at
+ * file scope to create necessary static functions.  Then, in its subsystem
+ * initialization (in the "bind to dispatcher" callback) (TODO: name this
+ * properly!), it calls DISPATCH_ADD_PUB() to tell the dispatcher about its
+ * intent to publish.  When it actually wants to publish, it uses the
+ * PUBLISH() macro.  For example:
+ *
+ *     // At file scope
+ *     DECLARE_PUBLISH(shutdown_requested);
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_PUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ *     // somewhere in a function
+ *        {
+ *            PUBLISH(shutdown_requested, true);
+ *        }
+ *
+ * When a subsystem wants to subscribe to a message, it uses
+ * DECLARE_SUBSCRIBE() at file scope to declare static functions.  It must
+ * declare a hook function that receives the message type.  Then, in its "bind
+ * to dispatcher" function, it calls DISPATCHER_ADD_SUB() to tell the
+ * dispatcher about its intent to subscribe.  When another module publishes
+ * the message, the dispatcher will call the provided hook function.
+ *
+ *     // At file scope.  The first argument is the message that you're
+ *     // subscribing to; the second argument is the hook function to declare.
+ *     DECLARE_SUBSCRIBE(shutdown_requested, on_shutdown_req_cb);
+ *
+ *     // You need to declare this function.
+ *     static void on_shutdown_req_cb(const msg_t *msg,
+ *                                    bool value)
+ *     {
+ *         // (do something here.)
+ *     }
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_SUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ * Where did these types come from?  Somewhere in the code, you need to call
+ * DISPATCH_REGISTER_TYPE() to make sure that the dispatcher can manage the
+ * message auxiliary data.  It associates a vtbl-like structure with the
+ * type name, so that the dispatcher knows how to manipulate the type you're
+ * giving it.
+ *
+ * For example, the "boolean" type we're using above could be defined as:
+ *
+ *    static char *boolean_fmt(msg_aux_data_t d)
+ *    {
+ *        // This is used for debugging and dumping messages.
+ *        if (d.u64)
+ *            return tor_strdup("true");
+ *        else
+ *            return tor_strdup("false");
+ *    }
+ *
+ *    static void boolean_free(msg_aux_data_t d)
+ *    {
+ *        // We don't actually need to do anything to free a boolean.
+ *        // We could use "NULL" instead of this function, but I'm including
+ *        // it as an example.
+ *    }
+ *
+ *    static void bind_to_dispatcher(pubsub_connector_t *con)
+ *    {
+ *        dispatch_typefns_t boolean_fns = {
+ *            .fmt_fn = boolean_fmt,
+ *            .free_fn = boolean_free,
+ *        };
+ *        DISPATCH_REGISTER_TYPE(con, boolean, &boolean_fns);
+ *    }
+ *
+ *
+ *
+ * So, how does this all work?  (You can stop reading here, unless you're
+ * debugging something.)
+ *
+ * When you declare a message in a header with DECLARE_MESSAGE() or
+ * DECLARE_MESSAGE_INT(), it creates five things:
+ *
+ *    * two typedefs for the message argument (constant and non-constant
+ *      variants).
+ *    * a constant string to hold the declared message type name
+ *    * two inline functions, to coerce the message argument type to and from
+ *      a "msg_aux_data_t" union.
+ *
+ * All of these declarations have names based on the message name.
+ *
+ * Later, when you say DECLARE_PUBLISH() or DECLARE_SUBSCRIBE(), we use the
+ * elements defined by DECLARE_MESSAGE() to make sure that the publish
+ * function takes the correct argument type, and that the subscription hook is
+ * declared with the right argument type.
+ **/
+
+#ifndef TOR_DISPATCH_MSG_H
+#define TOR_DISPATCH_MSG_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+/* Implemenation notes:
+ *
+ * For a messagename "foo", the DECLARE_MESSAGE*() macros must declare:
+ *
+ * msg_arg_type__foo -- a typedef for the argument type of the foo message.
+ * msg_arg_consttype__foo -- a typedef for the const argument type of the
+ *     foo message.
+ * msg_arg_name__foo[] -- a static string constant holding the unique
+ *      identifier for the type of the foo message.
+ * msg_arg_get__foo() -- an inline function taking a msg_aux_data_t and
+ *      returning the C data type.
+ * msg_arg_set__foo() -- an inline function taking a msg_aux_data_t and
+ *      the C type, setting the msg_aux_data_t to hold the C type.
+ *
+ * For a messagename "foo", the DECLARE_PUBLISH() macro must declare:
+ *
+ * pub_binding__foo -- A static pub_binding_t object used to send messages
+ *      from this module.
+ * publish_fn__foo -- A function taking an argument of the appropriate
+ *      C type, to be invoked by PUBLISH().
+ *
+ * For a messagename "foo", the DECLARE_SUBSCRIBE() macro must declare:
+ *
+ * hookfn -- A user-provided function name, with the correct signature.
+ * recv_fn__foo -- A wrapper callback that takes a msg_t *, and calls
+ *      hookfn with the appropriate arguments.
+ */
+
+/* Macro to declare common elements shared by DECLARE_MESSAGE and
+ * DECLARE_MESSAGE_INT.  Don't call this directly.
+ *
+ * Note that the "msg_arg_name" string constant is defined in each
+ * translation unit.  This might be undesirable; we can tweak it in the
+ * future if need be.
+ */
+#define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)         \
+  typedef c_type msg_arg_type__ ##messagename;                          \
+  typedef const c_type msg_arg_consttype__ ##messagename;               \
+  ATTR_UNUSED static const char msg_arg_name__ ##messagename[] = # typename;
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking a pointer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.
+ * It needs to be defined somewhere in Tor, using
+ * "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_ptr_type" is a C pointer type (like "char *" or "struct foo *").
+ * The "*" needs to be included.
+ */
+#define DECLARE_MESSAGE(messagename, typename, c_ptr_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_ptr_type)           \
+  ATTR_UNUSED static inline c_ptr_type                                  \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return m.ptr;                                                       \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_ptr_type v)          \
+  {                                                                     \
+    m->ptr = v;                                                         \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking an integer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.  It
+ * needs to be defined somewhere in Tor, using "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_type" is a C integer type, like "int" or "bool".  It needs to fit inside
+ * a uint64_t.
+ */
+#define DECLARE_MESSAGE_INT(messagename, typename, c_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)               \
+  ATTR_UNUSED static inline c_type                                      \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return (c_type)m.u64;                                               \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v)              \
+  {                                                                     \
+    m->u64 = (uint64_t)v;                                               \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C module declare that we'll be publishing a given
+ * message type from within this module.
+ *
+ * It creates necessary functions and wrappers to publish a message whose
+ * unique identifier is "messagename".
+ *
+ * Before you use this, you need to include the header where DECLARE_MESSAGE*()
+ * was used for this message.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_PUBLISH(messagename)                                    \
+  static pub_binding_t pub_binding__ ##messagename;                     \
+  static void                                                           \
+  publish_fn__ ##messagename(msg_arg_type__ ##messagename arg)          \
+  {                                                                     \
+    msg_aux_data_t data;                                                \
+    msg_arg_set__ ##messagename(&data, arg);                            \
+    pubsub_pub_(&pub_binding__ ##messagename, data);                    \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C file to declare that we're subscribing to a
+ * given message and associating it with a given "hook function".  It
+ * declares the hook function static, and helps with strong typing.
+ *
+ * Before you use this, you need to include the header where
+ * DECLARE_MESSAGE*() was used for the message whose unique identifier is
+ * "messagename".
+ *
+ * You will need to define a function with the name that you provide for
+ * "hookfn".  The type of this function will be:
+ *     static void hookfn(const msg_t *, const c_type)
+ * where c_type is the c type that you declared in the header.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_SUBSCRIBE(messagename, hookfn) \
+  static void hookfn(const msg_t *,                             \
+                     const msg_arg_consttype__ ##messagename);  \
+  static void recv_fn__ ## messagename(const msg_t *m)          \
+  {                                                             \
+    msg_arg_type__ ## messagename arg;                          \
+    arg = msg_arg_get__ ##messagename(m->aux_data__);           \
+    hookfn(m, arg);                                             \
+  }                                                             \
+  EAT_SEMICOLON
+
+/**
+ * Add a fake use of the publish function for 'messagename', so that
+ * the compiler does not call it unused.
+ */
+#define DISPATCH__FAKE_USE_OF_PUBFN_(messagename)                       \
+  ( 0 ? (publish_fn__ ##messagename((msg_arg_type__##messagename)0), 1) \
+    : 1)
+
+/*
+ * This macro is for internal use.  It backs DISPATCH_ADD_PUB*()
+ */
+#define DISPATCH_ADD_PUB_(connector, channel, messagename, flags)       \
+  (                                                                     \
+    DISPATCH__FAKE_USE_OF_PUBFN_(messagename),                          \
+    pubsub_add_pub_((connector),                                        \
+                      &pub_binding__ ##messagename,                     \
+                      get_channel_id(# channel),                        \
+                    get_message_id(# messagename),                      \
+                      get_msg_type_id(msg_arg_name__ ## messagename),   \
+                      (flags),                                          \
+                      __FILE__,                                         \
+                      __LINE__)                                         \
+    )
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB(connector, channel, messagename)       \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, 0)
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type, and that no other subsystem is allowed to.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/*
+ * This macro is for internal use. It backs DISPATCH_ADD_SUB*()
+ */
+#define DISPATCH_ADD_SUB_(connector, channel, messagename, flags)       \
+  pubsub_add_sub_((connector),                                          \
+                    recv_fn__ ##messagename,                            \
+                    get_channel_id(#channel),                           \
+                    get_message_id(# messagename),                      \
+                    get_msg_type_id(msg_arg_name__ ##messagename),      \
+                    (flags),                                            \
+                    __FILE__,                                           \
+                    __LINE__)
+/*
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB(connector, channel, messagename)       \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, 0)
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type, and that no other subsystem is allowed to do
+ * so.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/**
+ * Publish a given message with a given argument.  (Takes ownership of the
+ * argument if it is a pointer.)
+ */
+#define PUBLISH(messagename, arg)               \
+  publish_fn__ ##messagename(arg)
+
+/**
+ * Use a given connector to declare that the functions to be used to manipuate
+ * a certain C type.
+ **/
+#define DISPATCH_REGISTER_TYPE(con, type, fns)                  \
+  pubsub_connector_register_type_((con),                        \
+                                  get_msg_type_id(#type),       \
+                                  (fns),                        \
+                                  __FILE__,                     \
+                                  __LINE__)
+
+#endif /* !defined(TOR_DISPATCH_MSG_H) */
diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c
new file mode 100644
index 0000000000..454a335a78
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.c
@@ -0,0 +1,72 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_publish.c
+ * @brief Header for functions to publish using a pub_binding_t.
+ **/
+
+#define PUBSUB_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Publish a message from the publication binding <b>pub</b> using the
+ * auxiliary data <b>auxdata</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata)
+{
+  dispatch_t *d = pub->dispatch_ptr;
+  if (BUG(! d)) {
+    /* Tried to publish a message before the dispatcher was configured. */
+    /* (Without a dispatcher, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.type >= d->n_types)) {
+    /* The type associated with this message is not known to the dispatcher. */
+    /* (Without a correct type, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.msg >= d->n_msgs) ||
+      BUG(pub->msg_template.channel >= d->n_queues)) {
+    /* The message ID or channel ID was out of bounds. */
+    // LCOV_EXCL_START
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return -1;
+    // LCOV_EXCL_STOP
+  }
+
+  if (! d->table[pub->msg_template.msg]) {
+    /* Fast path: nobody wants this data. */
+
+    // XXXX Faster path: we could store this in the pub_binding_t.
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return 0;
+  }
+
+  /* Construct the message object */
+  msg_t *m = tor_malloc(sizeof(msg_t));
+  memcpy(m, &pub->msg_template, sizeof(msg_t));
+  m->aux_data__ = auxdata;
+
+  return dispatch_send_msg_unchecked(d, m);
+}
diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h
new file mode 100644
index 0000000000..0686a465de
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.h
@@ -0,0 +1,15 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PUBSUB_PUBLISH_H
+#define TOR_PUBSUB_PUBLISH_H
+
+#include "lib/dispatch/msgtypes.h"
+struct pub_binding_t;
+
+int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata);
+
+#endif /* !defined(TOR_PUBSUB_PUBLISH_H) */
diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include
index 84906dfb3d..853dae7880 100644
--- a/src/lib/sandbox/.may_include
+++ b/src/lib/sandbox/.may_include
@@ -5,11 +5,10 @@ lib/container/*.h
 lib/err/*.h
 lib/log/*.h
 lib/malloc/*.h
-lib/net/*.h
 lib/sandbox/*.h
 lib/sandbox/*.inc
 lib/string/*.h
 
-ht.h
-siphash.h
-tor_queue.h
+ext/ht.h
+ext/siphash.h
+ext/tor_queue.h
diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am
index adfda6bde5..e81f14b55f 100644
--- a/src/lib/sandbox/include.am
+++ b/src/lib/sandbox/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_sandbox_a_SOURCES =			\
 	src/lib/sandbox/sandbox.c
 
@@ -13,6 +14,7 @@ src_lib_libtor_sandbox_testing_a_SOURCES = \
 src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/sandbox/linux_syscalls.inc		\
 	src/lib/sandbox/sandbox.h
diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c
index 8f577b0660..3515e8854e 100644
--- a/src/lib/sandbox/sandbox.c
+++ b/src/lib/sandbox/sandbox.c
@@ -38,13 +38,12 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "lib/cc/torint.h"
-#include "lib/net/resolve.h"
 #include "lib/malloc/malloc.h"
 #include "lib/string/scanf.h"
 
-#include "tor_queue.h"
-#include "ht.h"
-#include "siphash.h"
+#include "ext/tor_queue.h"
+#include "ext/ht.h"
+#include "ext/siphash.h"
 
 #define DEBUGGING_CLOSE
 
@@ -144,6 +143,7 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(clock_gettime),
     SCMP_SYS(close),
     SCMP_SYS(clone),
+    SCMP_SYS(dup),
     SCMP_SYS(epoll_create),
     SCMP_SYS(epoll_wait),
 #ifdef __NR_epoll_pwait
@@ -295,6 +295,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   unsigned i;
   int rc;
   int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD,
+                  SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, SIGIO,
 #ifdef SIGXFSZ
       SIGXFSZ
 #endif
@@ -444,7 +445,7 @@ libc_uses_openat_for_everything(void)
     return 1;
   else
     return 0;
-#else /* !(defined(CHECK_LIBC_VERSION)) */
+#else /* !defined(CHECK_LIBC_VERSION) */
   return 0;
 #endif /* defined(CHECK_LIBC_VERSION) */
 }
@@ -798,6 +799,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   if (rc)
     return rc;
 
+  rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
+      SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
+      SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN));
+  if (rc)
+    return rc;
+
 #ifdef HAVE_SYSTEMD
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
       SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
@@ -1517,7 +1524,6 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // marking the sandbox as active
   sandbox_active = 1;
-  tor_make_getaddrinfo_cache_active();
 
  end:
   seccomp_release(ctx);
@@ -1764,9 +1770,4 @@ sandbox_is_active(void)
   return 0;
 }
 
-void
-sandbox_disable_getaddrinfo_cache(void)
-{
-}
-
 #endif /* !defined(USE_LIBSECCOMP) */
diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h
index 5bec09a36a..b4ae6e5c07 100644
--- a/src/lib/sandbox/sandbox.h
+++ b/src/lib/sandbox/sandbox.h
@@ -108,7 +108,7 @@ typedef struct {
  * it matches the parameter.
  */
 const char* sandbox_intern_string(const char *param);
-#else /* !(defined(USE_LIBSECCOMP)) */
+#else /* !defined(USE_LIBSECCOMP) */
 #define sandbox_intern_string(s) (s)
 #endif /* defined(USE_LIBSECCOMP) */
 
diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include
index a8507761a4..2f0c8d341e 100644
--- a/src/lib/smartlist_core/.may_include
+++ b/src/lib/smartlist_core/.may_include
@@ -4,4 +4,4 @@ lib/malloc/*.h
 lib/err/*.h
 lib/string/*.h
 lib/smartlist_core/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am
index 99d65f0b23..548179bc4f 100644
--- a/src/lib/smartlist_core/include.am
+++ b/src/lib/smartlist_core/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_smartlist_core_a_SOURCES =			\
 	src/lib/smartlist_core/smartlist_core.c                 \
 	src/lib/smartlist_core/smartlist_split.c
@@ -15,6 +16,7 @@ src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \
 	$(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/smartlist_core/smartlist_core.h         \
 	src/lib/smartlist_core/smartlist_foreach.h      \
diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c
index ac85a6cc84..6b0a305a93 100644
--- a/src/lib/smartlist_core/smartlist_core.c
+++ b/src/lib/smartlist_core/smartlist_core.c
@@ -88,6 +88,30 @@ smartlist_ensure_capacity(smartlist_t *sl, size_t size)
 #undef MAX_CAPACITY
 }
 
+/** Expand <b>sl</b> so that its length is at least <b>new_size</b>,
+ * filling in previously unused entries with NULL>
+ *
+ * Do nothing if <b>sl</b> already had at least <b>new_size</b> elements.
+ */
+void
+smartlist_grow(smartlist_t *sl, size_t new_size)
+{
+  smartlist_ensure_capacity(sl, new_size);
+
+  if (new_size > (size_t)sl->num_used) {
+    /* This memset() should be a no-op: everything else in the smartlist code
+     * tries to make sure that unused entries are always NULL.  Still, that is
+     * meant as a safety mechanism, so let's clear the memory here.
+     */
+    memset(sl->list + sl->num_used, 0,
+           sizeof(void *) * (new_size - sl->num_used));
+
+    /* This cast is safe, since we already asserted that we were below
+     * MAX_CAPACITY in smartlist_ensure_capacity(). */
+    sl->num_used = (int)new_size;
+  }
+}
+
 /** Append element to the end of the list. */
 void
 smartlist_add(smartlist_t *sl, void *element)
@@ -153,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element)
       sl->list[i++] = sl->list[j];
     }
   }
+  memset(sl->list + sl->num_used, 0,
+         sizeof(void *) * (num_used_orig - sl->num_used));
 }
 
 /** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h
index a7fbaa099b..36f23e2009 100644
--- a/src/lib/smartlist_core/smartlist_core.h
+++ b/src/lib/smartlist_core/smartlist_core.h
@@ -43,6 +43,7 @@ void smartlist_clear(smartlist_t *sl);
 void smartlist_add(smartlist_t *sl, void *element);
 void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
 void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
+void smartlist_grow(smartlist_t *sl, size_t new_size);
 
 void smartlist_remove(smartlist_t *sl, const void *element);
 void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
@@ -76,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
   raw_assert(sl->num_used > idx);
   sl->list[idx] = val;
 }
-#else /* !(defined(DEBUG_SMARTLIST)) */
+#else /* !defined(DEBUG_SMARTLIST) */
 #define smartlist_len(sl) ((sl)->num_used)
 #define smartlist_get(sl, idx) ((sl)->list[idx])
 #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
@@ -97,4 +98,4 @@ void smartlist_del(smartlist_t *sl, int idx);
 void smartlist_del_keeporder(smartlist_t *sl, int idx);
 void smartlist_insert(smartlist_t *sl, int idx, void *val);
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_CORE_H) */
diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h
index 0f6fe30074..a1fbcd444c 100644
--- a/src/lib/smartlist_core/smartlist_foreach.h
+++ b/src/lib/smartlist_core/smartlist_foreach.h
@@ -83,6 +83,19 @@
          ++var ## _sl_idx) {                                    \
       var = (sl)->list[var ## _sl_idx];
 
+/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to
+ *  SMARTLIST_FOREACH_BEGIN
+ *
+ * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT.
+ */
+#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var)  \
+  STMT_BEGIN                                                       \
+    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;             \
+    type var;                                                      \
+    for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0;   \
+         --var ## _sl_idx) {                                       \
+      var = (sl)->list[var ## _sl_idx];
+
 #define SMARTLIST_FOREACH_END(var)              \
     var = NULL;                                 \
     (void) var ## _sl_idx;                      \
diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h
index 4f72376125..e2cc7245b7 100644
--- a/src/lib/smartlist_core/smartlist_split.h
+++ b/src/lib/smartlist_core/smartlist_split.h
@@ -17,4 +17,4 @@
 int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
                            int flags, int max);
 
-#endif
+#endif /* !defined(TOR_SMARTLIST_SPLIT_H) */
diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include
index ec5c769831..1fb9127f19 100644
--- a/src/lib/string/.may_include
+++ b/src/lib/string/.may_include
@@ -6,5 +6,5 @@ lib/malloc/*.h
 lib/ctime/*.h
 lib/string/*.h
 
-strlcat.c
-strlcpy.c
+ext/strlcat.c
+ext/strlcpy.c
diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c
index e125c921a4..187f784be5 100644
--- a/src/lib/string/compat_string.c
+++ b/src/lib/string/compat_string.c
@@ -14,10 +14,10 @@
 
 /* Inline the strl functions if the platform doesn't have them. */
 #ifndef HAVE_STRLCPY
-#include "strlcpy.c"
+#include "ext/strlcpy.c"
 #endif
 #ifndef HAVE_STRLCAT
-#include "strlcat.c"
+#include "ext/strlcat.c"
 #endif
 
 #include <stdlib.h>
diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h
index a0e37bb6dc..ffc892c3e5 100644
--- a/src/lib/string/compat_string.h
+++ b/src/lib/string/compat_string.h
@@ -25,20 +25,23 @@ static inline int strncasecmp(const char *a, const char *b, size_t n);
 static inline int strncasecmp(const char *a, const char *b, size_t n) {
   return _strnicmp(a,b,n);
 }
-#endif
+#endif /* !defined(HAVE_STRNCASECMP) */
 #ifndef HAVE_STRCASECMP
 static inline int strcasecmp(const char *a, const char *b);
 static inline int strcasecmp(const char *a, const char *b) {
   return _stricmp(a,b);
 }
-#endif
-#endif
+#endif /* !defined(HAVE_STRCASECMP) */
+#endif /* defined(_WIN32) */
 
 #if defined __APPLE__
 /* On OSX 10.9 and later, the overlap-checking code for strlcat would
  * appear to have a severe bug that can sometimes cause aborts in Tor.
  * Instead, use the non-checking variants.  This is sad.
  *
+ * (If --enable-fragile-hardening is passed to configure, we use the hardened
+ * variants, which do not suffer from this issue.)
+ *
  * See https://trac.torproject.org/projects/tor/ticket/15205
  */
 #undef strlcat
@@ -59,4 +62,4 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
 #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts)
 #endif
 
-#endif
+#endif /* !defined(TOR_COMPAT_STRING_H) */
diff --git a/src/lib/string/include.am b/src/lib/string/include.am
index edd74b8a3e..82d35cc5af 100644
--- a/src/lib/string/include.am
+++ b/src/lib/string/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-string-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_string_a_SOURCES =			\
 	src/lib/string/compat_ctype.c			\
 	src/lib/string/compat_string.c			\
@@ -18,6 +19,7 @@ src_lib_libtor_string_testing_a_SOURCES = \
 src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/string/compat_ctype.h			\
 	src/lib/string/compat_string.h			\
diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h
index 925547942e..50d48b44c5 100644
--- a/src/lib/string/parse_int.h
+++ b/src/lib/string/parse_int.h
@@ -22,4 +22,4 @@ double tor_parse_double(const char *s, double min, double max, int *ok,
 uint64_t tor_parse_uint64(const char *s, int base, uint64_t min,
                          uint64_t max, int *ok, char **next);
 
-#endif
+#endif /* !defined(TOR_PARSE_INT_H) */
diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c
index a5cb71ce09..26203932e4 100644
--- a/src/lib/string/printf.c
+++ b/src/lib/string/printf.c
@@ -117,8 +117,8 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = NULL;
     return -1;
   }
-  strp_tmp = tor_malloc(len + 1);
-  r = _vsnprintf(strp_tmp, len+1, fmt, args);
+  strp_tmp = tor_malloc((size_t)len + 1);
+  r = _vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
@@ -153,9 +153,9 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = tor_strdup(buf);
     return len;
   }
-  strp_tmp = tor_malloc(len+1);
+  strp_tmp = tor_malloc((size_t)len+1);
   /* use of tor_vsnprintf() will ensure string is null terminated */
-  r = tor_vsnprintf(strp_tmp, len+1, fmt, args);
+  r = tor_vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h
index 2cc13d6bee..6e90770f81 100644
--- a/src/lib/string/printf.h
+++ b/src/lib/string/printf.h
@@ -27,4 +27,4 @@ int tor_asprintf(char **strp, const char *fmt, ...)
 int tor_vasprintf(char **strp, const char *fmt, va_list args)
   CHECK_PRINTF(2,0);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_PRINTF_H) */
diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h
index 6673173de5..b642e242db 100644
--- a/src/lib/string/scanf.h
+++ b/src/lib/string/scanf.h
@@ -21,4 +21,4 @@ int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \
 int tor_sscanf(const char *buf, const char *pattern, ...)
   CHECK_SCANF(2, 3);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_SCANF_H) */
diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c
index f934f66f02..f5061a11d2 100644
--- a/src/lib/string/util_string.c
+++ b/src/lib/string/util_string.c
@@ -71,7 +71,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle)
 
 /** Return true iff the 'len' bytes at 'mem' are all zero. */
 int
-tor_mem_is_zero(const char *mem, size_t len)
+fast_mem_is_zero(const char *mem, size_t len)
 {
   static const char ZERO[] = {
     0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
@@ -95,17 +95,14 @@ tor_mem_is_zero(const char *mem, size_t len)
 int
 tor_digest_is_zero(const char *digest)
 {
-  static const uint8_t ZERO_DIGEST[] = {
-    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
-  };
-  return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN);
+  return safe_mem_is_zero(digest, DIGEST_LEN);
 }
 
 /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
 int
 tor_digest256_is_zero(const char *digest)
 {
-  return tor_mem_is_zero(digest, DIGEST256_LEN);
+  return safe_mem_is_zero(digest, DIGEST256_LEN);
 }
 
 /** Remove from the string <b>s</b> every character which appears in
@@ -212,21 +209,6 @@ strcmpstart(const char *s1, const char *s2)
   return strncmp(s1, s2, n);
 }
 
-/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>,
- * without depending on a terminating nul in s1.  Sorting order is first by
- * length, then lexically; return values are as for strcmp.
- */
-int
-strcmp_len(const char *s1, const char *s2, size_t s1_len)
-{
-  size_t s2_len = strlen(s2);
-  if (s1_len < s2_len)
-    return -1;
-  if (s1_len > s2_len)
-    return 1;
-  return fast_memcmp(s1, s2, s2_len);
-}
-
 /** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
  * strcasecmp.
  */
@@ -541,3 +523,16 @@ string_is_utf8(const char *str, size_t len)
   }
   return true;
 }
+
+/** As string_is_utf8(), but returns false if the string begins with a UTF-8
+ * byte order mark (BOM).
+ */
+int
+string_is_utf8_no_bom(const char *str, size_t len)
+{
+  if (len >= 3 && (!strcmpstart(str, "\uFEFF") ||
+                   !strcmpstart(str, "\uFFFE"))) {
+    return false;
+  }
+  return string_is_utf8(str, len);
+}
diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h
index d9fbf8c61e..b3c6841d41 100644
--- a/src/lib/string/util_string.h
+++ b/src/lib/string/util_string.h
@@ -20,7 +20,10 @@ const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
                        size_t nlen);
 const void *tor_memstr(const void *haystack, size_t hlen,
                        const char *needle);
-int tor_mem_is_zero(const char *mem, size_t len);
+int fast_mem_is_zero(const char *mem, size_t len);
+#define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN)
+#define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN)
+
 int tor_digest_is_zero(const char *digest);
 int tor_digest256_is_zero(const char *digest);
 
@@ -33,7 +36,6 @@ int tor_strisnonupper(const char *s);
 int tor_strisspace(const char *s);
 int strcmp_opt(const char *s1, const char *s2);
 int strcmpstart(const char *s1, const char *s2);
-int strcmp_len(const char *s1, const char *s2, size_t len);
 int strcasecmpstart(const char *s1, const char *s2);
 int strcmpend(const char *s1, const char *s2);
 int strcasecmpend(const char *s1, const char *s2);
@@ -53,5 +55,6 @@ const char *find_str_at_start_of_line(const char *haystack,
 int string_is_C_identifier(const char *string);
 
 int string_is_utf8(const char *str, size_t len);
+int string_is_utf8_no_bom(const char *str, size_t len);
 
 #endif /* !defined(TOR_UTIL_STRING_H) */
diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include
new file mode 100644
index 0000000000..2b06e8519c
--- /dev/null
+++ b/src/lib/subsys/.may_include
@@ -0,0 +1 @@
+orconfig.h
diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am
new file mode 100644
index 0000000000..c9ab54ca73
--- /dev/null
+++ b/src/lib/subsys/include.am
@@ -0,0 +1,4 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS += \
+	src/lib/subsys/subsys.h
diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h
new file mode 100644
index 0000000000..21f984f32d
--- /dev/null
+++ b/src/lib/subsys/subsys.h
@@ -0,0 +1,95 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SUBSYS_T
+#define TOR_SUBSYS_T
+
+#include <stdbool.h>
+
+struct pubsub_connector_t;
+
+/**
+ * A subsystem is a part of Tor that is initialized, shut down, configured,
+ * and connected to other parts of Tor.
+ *
+ * All callbacks are optional -- if a callback is set to NULL, the subsystem
+ * manager will treat it as a no-op.
+ *
+ * You should use c99 named-field initializers with this structure: we
+ * will be adding more fields, often in the middle of the structure.
+ **/
+typedef struct subsys_fns_t {
+  /**
+   * The name of this subsystem.  It should be a programmer-readable
+   * identifier.
+   **/
+  const char *name;
+
+  /**
+   * Whether this subsystem is supported -- that is, whether it is compiled
+   * into Tor.  For most subsystems, this should be true.
+   **/
+  bool supported;
+
+  /**
+   * The 'initialization level' for the subsystem.  It should run from -100
+   * through +100.  The subsystems are initialized from lowest level to
+   * highest, and shut down from highest level to lowest.
+   **/
+  int level;
+
+  /**
+   * Initialize any global components of this subsystem.
+   *
+   * This function MAY rely on any lower-level subsystem being initialized.
+   *
+   * This function MUST NOT rely on any runtime configuration information;
+   * it is only for global state or pre-configuration state.
+   *
+   * (If you need to do any setup that depends on configuration, you'll need
+   * to declare a configuration callback. (Not yet designed))
+   *
+   * This function MUST NOT have any parts that can fail.
+   **/
+  int (*initialize)(void);
+
+  /**
+   * Connect a subsystem to the message dispatch system.
+   **/
+  int (*add_pubsub)(struct pubsub_connector_t *);
+
+  /**
+   * Perform any necessary pre-fork cleanup.  This function may not fail.
+   */
+  void (*prefork)(void);
+
+  /**
+   * Perform any necessary post-fork setup. This function may not fail.
+   */
+  void (*postfork)(void);
+
+  /**
+   * Free any thread-local resources held by this subsystem. Called before
+   * the thread exits.
+   */
+  void (*thread_cleanup)(void);
+
+  /**
+   * Free all resources held by this subsystem.
+   *
+   * This function is not allowed to fail.
+   **/
+  void (*shutdown)(void);
+
+} subsys_fns_t;
+
+#define MIN_SUBSYS_LEVEL -100
+#define MAX_SUBSYS_LEVEL 100
+
+/* All tor "libraries" (in src/libs) should have a subsystem level equal to or
+ * less than this value. */
+#define SUBSYS_LEVEL_LIBS -10
+
+#endif /* !defined(TOR_SUBSYS_T) */
diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include
index c93a06e59e..306fa57b7a 100644
--- a/src/lib/term/.may_include
+++ b/src/lib/term/.may_include
@@ -5,5 +5,4 @@ lib/log/*.h
 lib/term/*.h
 lib/malloc/*.h
 
-# From src/ext
-tor_readpassphrase.h
+ext/tor_readpassphrase.h
diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c
index c6459f250f..8741344acf 100644
--- a/src/lib/term/getpass.c
+++ b/src/lib/term/getpass.c
@@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #elif defined(HAVE_READPASSPHRASE_H)
 #include <readpassphrase.h>
 #else
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #endif /* defined(_WIN32) || ... */
 
 #include <stdlib.h>
diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h
index a9c146ea8f..aa597ec423 100644
--- a/src/lib/term/getpass.h
+++ b/src/lib/term/getpass.h
@@ -15,4 +15,4 @@
 
 ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
 
-#endif
+#endif /* !defined(TOR_GETPASS_H) */
diff --git a/src/lib/term/include.am b/src/lib/term/include.am
index 55fe548ebc..a120bba0cb 100644
--- a/src/lib/term/include.am
+++ b/src/lib/term/include.am
@@ -11,6 +11,7 @@ else
 readpassphrase_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_term_a_SOURCES =			\
 	src/lib/term/getpass.c			\
 	$(readpassphrase_source)
@@ -20,5 +21,6 @@ src_lib_libtor_term_testing_a_SOURCES = \
 src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/term/getpass.h
diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am
index b2aa620985..a5ed46eb67 100644
--- a/src/lib/testsupport/include.am
+++ b/src/lib/testsupport/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/testsupport/testsupport.h
diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h
index 9363a9ba66..90b7c43b19 100644
--- a/src/lib/testsupport/testsupport.h
+++ b/src/lib/testsupport/testsupport.h
@@ -21,7 +21,7 @@
  * tests. */
 #define STATIC
 #define EXTERN(type, name) extern type name;
-#else
+#else /* !defined(TOR_UNIT_TESTS) */
 #define STATIC static
 #define EXTERN(type, name)
 #endif /* defined(TOR_UNIT_TESTS) */
@@ -90,7 +90,7 @@
   do {                                          \
     func = func ##__real;                       \
   } while (0)
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
 #define MOCK_DECL(rv, funcname, arglist) \
   rv funcname arglist
 #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include
index fc56f46836..02711348c5 100644
--- a/src/lib/thread/.may_include
+++ b/src/lib/thread/.may_include
@@ -2,6 +2,7 @@ orconfig.h
 lib/cc/*.h
 lib/lock/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/wallclock/*.h
diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c
index 94ab021c52..5c8ffa55c6 100644
--- a/src/lib/thread/compat_threads.c
+++ b/src/lib/thread/compat_threads.c
@@ -14,9 +14,11 @@
 #include "orconfig.h"
 #include <stdlib.h>
 #include "lib/thread/threads.h"
+#include "lib/thread/thread_sys.h"
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
+#include "lib/subsys/subsys.h"
 
 #include <string.h>
 
@@ -65,7 +67,15 @@ atomic_counter_init(atomic_counter_t *counter)
   memset(counter, 0, sizeof(*counter));
   tor_mutex_init_nonrecursive(&counter->mutex);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -109,3 +119,19 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return oldval;
 }
 #endif /* !defined(HAVE_WORKING_STDATOMIC) */
+
+static int
+subsys_threads_initialize(void)
+{
+  tor_threads_init();
+  return 0;
+}
+
+const subsys_fns_t sys_threads = {
+  .name = "threads",
+  .supported = true,
+  /* Threads is used by logging, which is a diagnostic feature, we want it to
+   * init right after low-level error handling and approx time. */
+  .level = -95,
+  .initialize = subsys_threads_initialize,
+};
diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am
index 9ec23d166e..cd8016b5df 100644
--- a/src/lib/thread/include.am
+++ b/src/lib/thread/include.am
@@ -12,6 +12,7 @@ if THREADS_WIN32
 threads_impl_source=src/lib/thread/compat_winthreads.c
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_thread_a_SOURCES =			\
 	src/lib/thread/compat_threads.c			\
 	src/lib/thread/numcpus.c			\
@@ -22,6 +23,8 @@ src_lib_libtor_thread_testing_a_SOURCES = \
 src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/thread/threads.h			\
-	src/lib/thread/numcpus.h
+	src/lib/thread/numcpus.h			\
+	src/lib/thread/thread_sys.h			\
+	src/lib/thread/threads.h
diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h
index 3f0a29ce7c..2f1ea16eb9 100644
--- a/src/lib/thread/numcpus.h
+++ b/src/lib/thread/numcpus.h
@@ -13,4 +13,4 @@
 
 int compute_num_cpus(void);
 
-#endif
+#endif /* !defined(TOR_NUMCPUS_H) */
diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h
new file mode 100644
index 0000000000..c0daf2b5e9
--- /dev/null
+++ b/src/lib/thread/thread_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file threads_sys.h
+ * \brief Declare subsystem object for threads library
+ **/
+
+#ifndef TOR_THREADS_SYS_H
+#define TOR_THREADS_SYS_H
+
+extern const struct subsys_fns_t sys_threads;
+
+#endif /* !defined(TOR_THREADS_SYS_H) */
diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h
index ecf60641b5..4b42b9abd9 100644
--- a/src/lib/thread/threads.h
+++ b/src/lib/thread/threads.h
@@ -107,7 +107,7 @@ typedef struct atomic_counter_t {
   atomic_size_t val;
 } atomic_counter_t;
 #define ATOMIC_LINKAGE static
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 typedef struct atomic_counter_t {
   tor_mutex_t mutex;
   size_t val;
@@ -131,7 +131,17 @@ atomic_counter_init(atomic_counter_t *counter)
 {
   atomic_init(&counter->val, 0);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * This usage note applies to the compat_threads implementation of
+ * atomic_counter_destroy():
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 static inline void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -162,7 +172,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return atomic_exchange(&counter->val, newval);
 }
 
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 #endif /* defined(HAVE_WORKING_STDATOMIC) */
 
 #endif /* !defined(TOR_COMPAT_THREADS_H) */
diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include
index 2c7e37a836..ae01431b60 100644
--- a/src/lib/time/.may_include
+++ b/src/lib/time/.may_include
@@ -4,8 +4,10 @@ lib/cc/*.h
 lib/err/*.h
 lib/intmath/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/time/*.h
 lib/wallclock/*.h
+lib/defs/time.h
 
 # For load_windows_system_lib.
 lib/fs/winlib.h
\ No newline at end of file
diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c
index 98854bad2c..ab45224a7f 100644
--- a/src/lib/time/compat_time.c
+++ b/src/lib/time/compat_time.c
@@ -164,6 +164,8 @@ static int64_t last_tick_count = 0;
  * to be monotonic; increments them as appropriate so that they actually
  * _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC int64_t
 ratchet_performance_counter(int64_t count_raw)
@@ -202,6 +204,8 @@ static struct timeval timeofday_offset = { 0, 0 };
  * supposed to be monotonic; increments them as appropriate so that they
  * actually _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC void
 ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
@@ -270,7 +274,9 @@ monotime_init_internal(void)
 }
 
 /**
- * Set "out" to the most recent monotonic time value
+ * Set "out" to the most recent monotonic time value.
+ *
+ * The returned time may be the same as the previous returned time.
  */
 void
 monotime_get(monotime_t *out)
@@ -298,10 +304,12 @@ monotime_coarse_get(monotime_coarse_t *out)
 #endif /* defined(TOR_UNIT_TESTS) */
   out->abstime_ = mach_approximate_time();
 }
-#endif
+#endif /* defined(HAVE_MACH_APPROXIMATE_TIME) */
 
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ *
+ * The returned value may be equal to zero.
  */
 int64_t
 monotime_diff_nsec(const monotime_t *start,
@@ -522,7 +530,9 @@ monotime_init_internal(void)
     GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void))
       GetProcAddress(h, "GetTickCount64");
   }
-  // FreeLibrary(h) ?
+  // We can't call FreeLibrary(h) here, because freeing the handle may
+  // unload the library, and cause future calls to GetTickCount64_fn()
+  // to fail. See 29642 for details.
 }
 
 void
@@ -757,7 +767,7 @@ monotime_coarse_zero(monotime_coarse_t *out)
 {
   memset(out, 0, sizeof(*out));
 }
-#endif
+#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 
 int64_t
 monotime_diff_usec(const monotime_t *start,
@@ -787,8 +797,8 @@ monotime_absolute_nsec(void)
   return monotime_diff_nsec(&initialized_at, &now);
 }
 
-uint64_t
-monotime_absolute_usec(void)
+MOCK_IMPL(uint64_t,
+monotime_absolute_usec,(void))
 {
   return monotime_absolute_nsec() / 1000;
 }
@@ -823,7 +833,7 @@ monotime_coarse_absolute_msec(void)
 {
   return monotime_coarse_absolute_nsec() / ONE_MILLION;
 }
-#else
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define initialized_at_coarse initialized_at
 #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 
@@ -855,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
     mach_time_info.numer;
   return abstime_val >> monotime_shift;
 }
-#else
+#else /* !defined(__APPLE__) */
 uint64_t
 monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
 {
@@ -866,4 +876,4 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
 {
   return (msec * STAMP_TICKS_PER_SECOND) / 1000;
 }
-#endif
+#endif /* defined(__APPLE__) */
diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h
index 480d426ac7..4d16effd29 100644
--- a/src/lib/time/compat_time.h
+++ b/src/lib/time/compat_time.h
@@ -15,6 +15,120 @@
  * of tens of milliseconds.
  */
 
+/* Q: When should I use monotonic time?
+ *
+ * A: If you need a time that never decreases, use monotonic time. If you need
+ * to send a time to a user or another process, or store a time, use the
+ * wall-clock time.
+ *
+ * Q: Should you use monotime or monotime_coarse as your source?
+ *
+ * A: Generally, you get better precision with monotime, but better
+ * performance with monotime_coarse.
+ *
+ * Q: What is a "monotonic" time, exactly?
+ *
+ * A: Monotonic times are strictly non-decreasing. The difference between any
+ * previous monotonic time, and the current monotonic time, is always greater
+ * than *or equal to* zero.
+ * Zero deltas happen more often:
+ *  - on Windows (due to an OS bug),
+ *  - when using monotime_coarse, or on systems with low-resolution timers,
+ *  - on platforms where we emulate monotonic time using wall-clock time, and
+ *  - when using time units that are larger than nanoseconds (due to
+ *    truncation on division).
+ *
+ * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use
+ *    usec? msec? "stamp units?"
+ *
+ * A: Using monotime_t and monotime_coarse_t directly is most time-efficient,
+ * since no conversion needs to happen.  But they can potentially use more
+ * memory than you would need for a usec/msec/"stamp unit" count.
+ *
+ * Converting to usec or msec on some platforms, and working with them in
+ * general, creates a risk of doing a 64-bit division.  64-bit division is
+ * expensive on 32-bit platforms, which still do exist.
+ *
+ * The "stamp unit" type is designed to give a type that is cheap to convert
+ * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a
+ * 32-bit integer.  Its downside is that it does not correspond directly
+ * to a natural unit of time.
+ *
+ * There is not much point in using "coarse usec" or "coarse nsec", since the
+ * current coarse monotime implementations give you on the order of
+ * milliseconds of precision.
+ *
+ * Q: So, what backends is monotime_coarse using?
+ *
+ * A: Generally speaking, it uses "whatever monotonic-ish time implemenation
+ * does not require a context switch."  The various implementations provide
+ * this by having a view of the current time in a read-only memory page that
+ * is updated with a frequency corresponding to the kernel's tick count.
+ *
+ * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on
+ * obsolete systems].  MSDN claims that the resolution is "typically in the
+ * range of 10-16 msec", but it has said that for years.  Storing
+ * monotime_coarse_t uses 8 bytes.
+ *
+ * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where
+ * available, and falls back to regular monotime. The precision is not
+ * documented, but the implementation is open-source: it reads from a page
+ * that the kernel updates. Storing monotime_coarse_t uses 8 bytes.
+ *
+ * On unixy systems, monotime_coarse uses clock_gettime() with
+ * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC.
+ * It typically uses vdso tricks to read from a page that the kernel updates.
+ * Its precision fixed, but you can get it with clock_getres(): on my Linux
+ * desktop, it claims to be 1 msec, but it will depend on the system HZ
+ * setting. Storing monotime_coarse_t uses 16 bytes.
+ *
+ * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.]
+ *
+ * Q: What backends is regular monotonic time using?
+ *
+ * A: In general, regular monotime uses something that requires a system call.
+ * On platforms where system calls are cheap, you win!  Otherwise, you lose.
+ *
+ * On Windows, monotonic time uses QuereyPerformanceCounter.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On OSX/Apple, monotonic time uses mach_absolute_time.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On unixy systems, monotonic time uses CLOCK_MONOTONIC.  Storing
+ * monotime_t costs 16 bytes.
+ *
+ * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec
+ *    count.
+ *
+ * A: Windows, coarse: Cheap, since it's all multiplication.
+ *
+ * Windows, precise: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Apple, all: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Unixy, all: Fairly cheap, since the only division required is dividing
+ * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value.
+ *
+ * All, "timestamp units": Cheap everywhere: it never divides.
+ *
+ * Q: This is only somewhat related, but how much precision could I hope for
+ *    from a libevent time?
+ *
+ * A: Actually, it's _very_ related if you're timing in order to have a
+ * timeout happen.
+ *
+ * On Windows, it uses select: you could in theory have a microsecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On Linux, you have epoll: It has a millisecond resolution.  Some recent
+ * Libevents can also use timerfd for higher resolution if
+ * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag.
+ */
+
 #ifndef TOR_COMPAT_TIME_H
 #define TOR_COMPAT_TIME_H
 
@@ -86,26 +200,36 @@ void monotime_init(void);
 void monotime_get(monotime_t *out);
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
  */
 int64_t monotime_diff_nsec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of microseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of milliseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_msec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of nanoseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
  */
 uint64_t monotime_absolute_nsec(void);
 /**
  * Return the number of microseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
-uint64_t monotime_absolute_usec(void);
+MOCK_DECL(uint64_t, monotime_absolute_usec,(void));
 /**
  * Return the number of milliseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_absolute_msec(void);
 
@@ -129,10 +253,13 @@ void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec);
  * Set <b>out</b> to the current coarse time.
  */
 void monotime_coarse_get(monotime_coarse_t *out);
+/**
+ * Like monotime_absolute_*(), but faster on some platforms.
+ */
 uint64_t monotime_coarse_absolute_nsec(void);
 uint64_t monotime_coarse_absolute_usec(void);
 uint64_t monotime_coarse_absolute_msec(void);
-#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define monotime_coarse_get monotime_get
 #define monotime_coarse_absolute_nsec monotime_absolute_nsec
 #define monotime_coarse_absolute_usec monotime_absolute_usec
@@ -152,23 +279,32 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t);
 /**
  * Convert a difference, expressed in the units of monotime_coarse_to_stamp,
  * into an approximate number of milliseconds.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units);
 uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec);
 uint32_t monotime_coarse_get_stamp(void);
 
 #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)
+/**
+ * Like monotime_diff_*(), but faster on some platforms.
+ */
 int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
+/**
+ * Like monotime_*(), but faster on some platforms.
+ */
 void monotime_coarse_zero(monotime_coarse_t *out);
 int monotime_coarse_is_zero(const monotime_coarse_t *val);
 void monotime_coarse_add_msec(monotime_coarse_t *out,
                               const monotime_coarse_t *val, uint32_t msec);
-#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 #define monotime_coarse_diff_nsec monotime_diff_nsec
 #define monotime_coarse_diff_usec monotime_diff_usec
 #define monotime_coarse_diff_msec monotime_diff_msec
@@ -182,6 +318,9 @@ void monotime_coarse_add_msec(monotime_coarse_t *out,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
                                      const monotime_coarse_t *end);
@@ -191,6 +330,9 @@ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 static inline int32_t
 monotime_coarse_diff_msec32(const monotime_coarse_t *start,
@@ -202,7 +344,7 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start,
 #else
 #define USING_32BIT_MSEC_HACK
   return monotime_coarse_diff_msec32_(start, end);
-#endif
+#endif /* SIZEOF_VOID_P == 8 */
 }
 
 #ifdef TOR_UNIT_TESTS
diff --git a/src/lib/time/include.am b/src/lib/time/include.am
index a3f93a3744..dcb199b142 100644
--- a/src/lib/time/include.am
+++ b/src/lib/time/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-time-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_time_a_SOURCES =	\
 		src/lib/time/compat_time.c	\
+		src/lib/time/time_sys.c		\
 		src/lib/time/tvdiff.c
 
 src_lib_libtor_time_testing_a_SOURCES = \
@@ -14,6 +16,8 @@ src_lib_libtor_time_testing_a_SOURCES = \
 src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/time/compat_time.h	\
+		src/lib/time/time_sys.h		\
 		src/lib/time/tvdiff.h
diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c
new file mode 100644
index 0000000000..8b9aa2856c
--- /dev/null
+++ b/src/lib/time/time_sys.c
@@ -0,0 +1,28 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.c
+ * \brief Subsystem object for monotime setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/time/time_sys.h"
+#include "lib/time/compat_time.h"
+
+static int
+subsys_time_initialize(void)
+{
+  monotime_init();
+  return 0;
+}
+
+const subsys_fns_t sys_time = {
+  .name = "time",
+  /* Monotonic time depends on logging, and a lot of other modules depend on
+   * monotonic time. */
+  .level = -80,
+  .supported = true,
+  .initialize = subsys_time_initialize,
+};
diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h
new file mode 100644
index 0000000000..6a860ffd08
--- /dev/null
+++ b/src/lib/time/time_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.h
+ * \brief Declare subsystem object for the time module.
+ **/
+
+#ifndef TOR_TIME_SYS_H
+#define TOR_TIME_SYS_H
+
+extern const struct subsys_fns_t sys_time;
+
+#endif /* !defined(TOR_TIME_SYS_H) */
diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c
index a87d0d96dc..d7c245f57a 100644
--- a/src/lib/time/tvdiff.c
+++ b/src/lib/time/tvdiff.c
@@ -11,6 +11,7 @@
 #include "lib/time/tvdiff.h"
 
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/time.h"
 #include "lib/log/log.h"
 
 #ifdef _WIN32
@@ -20,8 +21,6 @@
 #include <sys/time.h>
 #endif
 
-#define TOR_USEC_PER_SEC 1000000
-
 /** Return the difference between start->tv_sec and end->tv_sec.
  * Returns INT64_MAX on overflow and underflow.
  */
diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h
index 724af1528a..657cb99553 100644
--- a/src/lib/time/tvdiff.h
+++ b/src/lib/time/tvdiff.h
@@ -20,4 +20,4 @@ int64_t tv_to_msec(const struct timeval *tv);
 
 time_t time_diff(const time_t from, const time_t to);
 
-#endif
+#endif /* !defined(TOR_TVDIFF_H) */
diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include
index 2840e590b8..c550bde024 100644
--- a/src/lib/tls/.may_include
+++ b/src/lib/tls/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/crypt_ops/*.h
@@ -11,7 +12,7 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/tls/*.h
-
-ciphers.inc
+lib/tls/*.inc
diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c
index b570216df0..ed0f821ce8 100644
--- a/src/lib/tls/buffers_tls.c
+++ b/src/lib/tls/buffers_tls.c
@@ -12,7 +12,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am
index a664b29fb2..7e05ef4f8c 100644
--- a/src/lib/tls/include.am
+++ b/src/lib/tls/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-tls-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_tls_a_SOURCES =			\
 	src/lib/tls/buffers_tls.c		\
 	src/lib/tls/tortls.c			\
@@ -29,6 +30,7 @@ src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_tls_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/tls/ciphers.inc			\
 	src/lib/tls/buffers_tls.h		\
@@ -36,5 +38,6 @@ noinst_HEADERS +=				\
 	src/lib/tls/tortls.h			\
 	src/lib/tls/tortls_internal.h		\
 	src/lib/tls/tortls_st.h			\
+	src/lib/tls/tortls_sys.h		\
 	src/lib/tls/x509.h			\
 	src/lib/tls/x509_internal.h
diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h
index 8b31603923..47f220c4c1 100644
--- a/src/lib/tls/nss_countbytes.h
+++ b/src/lib/tls/nss_countbytes.h
@@ -22,4 +22,4 @@ int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd,
                                    uint64_t *n_read_out,
                                    uint64_t *n_written_out);
 
-#endif
+#endif /* !defined(TOR_NSS_COUNTBYTES_H) */
diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c
index 4ca7c7d9d3..1aff40c437 100644
--- a/src/lib/tls/tortls.c
+++ b/src/lib/tls/tortls.c
@@ -7,6 +7,7 @@
 #define TOR_X509_PRIVATE
 #include "lib/tls/x509.h"
 #include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls_sys.h"
 #include "lib/tls/tortls.h"
 #include "lib/tls/tortls_st.h"
 #include "lib/tls/tortls_internal.h"
@@ -15,6 +16,7 @@
 #include "lib/crypt_ops/crypto_rsa.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/net/socket.h"
+#include "lib/subsys/subsys.h"
 
 #ifdef _WIN32
   #include <winsock2.h>
@@ -440,3 +442,15 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity)
 
   return rv;
 }
+
+static void
+subsys_tortls_shutdown(void)
+{
+  tor_tls_free_all();
+}
+
+const subsys_fns_t sys_tortls = {
+  .name = "tortls",
+  .level = -50,
+  .shutdown = subsys_tortls_shutdown
+};
diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h
index 8efc7a1c98..799bd6aaeb 100644
--- a/src/lib/tls/tortls.h
+++ b/src/lib/tls/tortls.h
@@ -25,12 +25,12 @@ struct ssl_ctx_st;
 struct ssl_session_st;
 typedef struct ssl_ctx_st tor_tls_context_impl_t;
 typedef struct ssl_st tor_tls_impl_t;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 struct PRFileDesc;
 typedef struct PRFileDesc tor_tls_context_impl_t;
 typedef struct PRFileDesc tor_tls_impl_t;
-#endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+#endif /* defined(TORTLS_PRIVATE) */
 
 struct tor_x509_cert_t;
 
@@ -144,9 +144,9 @@ void check_no_tls_errors_(const char *fname, int line);
 
 void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
                            int severity, int domain, const char *doing);
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 #define check_no_tls_errors() STMT_NIL
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 int tor_tls_get_my_certs(int server,
                          const struct tor_x509_cert_t **link_cert_out,
diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h
index 071c506561..866483a94c 100644
--- a/src/lib/tls/tortls_internal.h
+++ b/src/lib/tls/tortls_internal.h
@@ -61,8 +61,8 @@ STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret,
                             void *arg);
 STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
                              uint16_t cipher);
-#endif
-#endif
+#endif /* defined(TORTLS_OPENSSL_PRIVATE) */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef TOR_UNIT_TESTS
 extern int tor_tls_object_ex_data_index;
@@ -73,4 +73,4 @@ extern uint64_t total_bytes_written_over_tls;
 extern uint64_t total_bytes_written_by_tls;
 #endif /* defined(TOR_UNIT_TESTS) */
 
-#endif /* defined(TORTLS_INTERNAL_H) */
+#endif /* !defined(TORTLS_INTERNAL_H) */
diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c
index 80b0df301f..5bafcf676d 100644
--- a/src/lib/tls/tortls_openssl.c
+++ b/src/lib/tls/tortls_openssl.c
@@ -25,7 +25,7 @@
    * <winsock.h> and mess things up, in at least some openssl versions. */
   #include <winsock2.h>
   #include <ws2tcpip.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rand.h"
@@ -318,7 +318,7 @@ tor_tls_init(void)
 #else
     SSL_library_init();
     SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))
@@ -383,7 +383,7 @@ static const char SERVER_CIPHER_LIST[] =
    * conclude that it has no valid ciphers if it's running with TLS1.3.
    */
   TLS1_3_TXT_AES_128_GCM_SHA256 ":"
-#endif
+#endif /* defined(TLS1_3_TXT_AES_128_GCM_SHA256) */
   TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":"
   TLS1_TXT_DHE_RSA_WITH_AES_128_SHA;
 
@@ -464,7 +464,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 /** List of ciphers that clients should advertise, omitting items that
  * our OpenSSL doesn't know about. */
 static const char CLIENT_CIPHER_LIST[] =
-#include "ciphers.inc"
+#include "lib/tls/ciphers.inc"
   /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version
    * of any cipher we say. */
   "!SSLv2"
@@ -657,7 +657,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     if (r < 0)
       goto error;
   }
-#else
+#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */
   if (! is_client) {
     int nid;
     EC_KEY *ec_key;
@@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
       SSL_CTX_set_tmp_ecdh(result->ctx, ec_key);
     EC_KEY_free(ec_key);
   }
-#endif
+#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */
   SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER,
                      always_accept_verify_cb);
   /* let us realloc bufs that we're writing from */
@@ -764,7 +764,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher);
     return c != NULL;
   }
-#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */
+#else /* !defined(HAVE_SSL_CIPHER_FIND) */
 
 # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR)
   if (m && m->get_cipher_by_char) {
@@ -1062,7 +1062,7 @@ tor_tls_new(tor_socket_t sock, int isServer)
     /* We can't actually use TLS 1.3 until this bug is fixed. */
     SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION);
   }
-#endif
+#endif /* defined(SSL_CTRL_SET_MAX_PROTO_VERSION) */
 
   if (!SSL_set_cipher_list(result->ssl,
                      isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) {
@@ -1728,7 +1728,7 @@ tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
     else
       return -1;
   }
-#endif
+#endif /* defined(TLS1_3_VERSION) */
 
   return (r == 1) ? 0 : -1;
 }
diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h
index 3f7ea8ac6a..73f6e6ecca 100644
--- a/src/lib/tls/tortls_st.h
+++ b/src/lib/tls/tortls_st.h
@@ -64,7 +64,7 @@ struct tor_tls_t {
   void (*negotiated_callback)(tor_tls_t *tls, void *arg);
   /** Argument to pass to negotiated_callback. */
   void *callback_arg;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
   /** Last values retried from tor_get_prfiledesc_byte_counts(). */
   uint64_t last_write_count;
@@ -72,4 +72,4 @@ struct tor_tls_t {
 #endif
 };
 
-#endif
+#endif /* !defined(TOR_TORTLS_ST_H) */
diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h
new file mode 100644
index 0000000000..4b04f85f0c
--- /dev/null
+++ b/src/lib/tls/tortls_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tortls_sys.h
+ * \brief Declare subsystem object for the tortls module
+ **/
+
+#ifndef TOR_TORTLS_SYS_H
+#define TOR_TORTLS_SYS_H
+
+extern const struct subsys_fns_t sys_tortls;
+
+#endif /* !defined(TOR_TORTLS_SYS_H) */
diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h
index 5e6660de5c..0390a5464d 100644
--- a/src/lib/tls/x509.h
+++ b/src/lib/tls/x509.h
@@ -35,7 +35,7 @@ struct tor_x509_cert_t {
   common_digests_t cert_digests;
   common_digests_t pkey_digests;
 };
-#endif
+#endif /* defined(TOR_X509_PRIVATE) */
 
 void tor_tls_pick_certificate_lifetime(time_t now,
                                        unsigned cert_lifetime,
@@ -47,7 +47,7 @@ tor_x509_cert_t *tor_x509_cert_replace_expiration(
                                                const tor_x509_cert_t *inp,
                                                time_t new_expiration_time,
                                                crypto_pk_t *signing_key);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 
@@ -72,4 +72,4 @@ int tor_tls_cert_is_valid(int severity,
                           time_t now,
                           int check_rsa_1024);
 
-#endif
+#endif /* !defined(TOR_X509_H) */
diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h
index bf2bec9689..f858baae98 100644
--- a/src/lib/tls/x509_internal.h
+++ b/src/lib/tls/x509_internal.h
@@ -50,4 +50,4 @@ int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert);
 #define tor_x509_cert_set_cached_der_encoding(cert) (0)
 #endif
 
-#endif
+#endif /* !defined(TOR_X509_INTERNAL_H) */
diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c
index fb4af54c52..e04afaf07b 100644
--- a/src/lib/tls/x509_nss.c
+++ b/src/lib/tls/x509_nss.c
@@ -120,13 +120,13 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                      der.data, der.len,
                                      (SECKEYPrivateKey *)signing_key,//const
                                      &cert->signature);
-#else
+#else /* !(0) */
   s = SEC_DerSignData(cert->arena,
                       &signed_der,
                       der.data, der.len,
                       (SECKEYPrivateKey *)signing_key,//const
                       SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION);
-#endif
+#endif /* 0 */
 
   if (s != SECSuccess)
     goto err;
@@ -145,7 +145,7 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                &result_cert->signatureWrap, issuer_pk, NULL);
     tor_assert(cert_ok == SECSuccess);
   }
-#endif
+#endif /* 1 */
 
  err:
   if (subject_spki)
@@ -455,4 +455,4 @@ tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
 
   return newcert ? tor_x509_cert_new(newcert) : NULL;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c
index a344279c22..7724288279 100644
--- a/src/lib/tls/x509_openssl.c
+++ b/src/lib/tls/x509_openssl.c
@@ -59,12 +59,12 @@ ENABLE_GCC_WARNING(redundant-decls)
 #define X509_get_notAfter(cert) \
     X509_getm_notAfter(cert)
 #endif
-#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
+#else /* !defined(OPENSSL_1_1_API) */
 #define X509_get_notBefore_const(cert) \
   ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
 #define X509_get_notAfter_const(cert) \
   ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 /** Return a newly allocated X509 name with commonName <b>cname</b>. */
 static X509_NAME *
diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h
index e35616cf50..92bb95c883 100644
--- a/src/lib/trace/debug.h
+++ b/src/lib/trace/debug.h
@@ -27,4 +27,4 @@
                         "\"" XSTR(subsystem) "\" hit. " \
                         "(line "XSTR(__LINE__) ")")
 
-#endif /* TOR_TRACE_LOG_DEBUG_H */
+#endif /* !defined(TOR_TRACE_LOG_DEBUG_H) */
diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h
index 1e1e7b9d16..9de86d63f2 100644
--- a/src/lib/trace/events.h
+++ b/src/lib/trace/events.h
@@ -34,12 +34,12 @@
 #include "lib/trace/debug.h"
 #endif
 
-#else /* TOR_EVENT_TRACING_ENABLED */
+#else /* !defined(TOR_EVENT_TRACING_ENABLED) */
 
 /* Reaching this point, we NOP every event declaration because event tracing
  * is not been enabled at compile time. */
 #define tor_trace(subsystem, name, args...)
 
-#endif /* TOR_EVENT_TRACING_ENABLED */
+#endif /* defined(TOR_EVENT_TRACING_ENABLED) */
 
-#endif /* TOR_TRACE_EVENTS_H */
+#endif /* !defined(TOR_TRACE_EVENTS_H) */
diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am
index 6f10c98744..98098c87f4 100644
--- a/src/lib/trace/include.am
+++ b/src/lib/trace/include.am
@@ -2,6 +2,7 @@
 noinst_LIBRARIES += \
 	src/lib/libtor-trace.a
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 TRACEHEADERS = \
 	src/lib/trace/trace.h \
 	src/lib/trace/events.h
@@ -11,7 +12,7 @@ TRACEHEADERS += \
 	src/lib/trace/debug.h
 endif
 
-# Library source files.
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_trace_a_SOURCES = \
 	src/lib/trace/trace.c
 
diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h
index 606d435568..5001b28a1d 100644
--- a/src/lib/trace/trace.h
+++ b/src/lib/trace/trace.h
@@ -11,4 +11,4 @@
 
 void tor_trace_init(void);
 
-#endif // TOR_TRACE_TRACE_H
+#endif /* !defined(TOR_TRACE_TRACE_H) */
diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include
new file mode 100644
index 0000000000..d159ceb41f
--- /dev/null
+++ b/src/lib/version/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+micro-revision.i
+lib/version/*.h
\ No newline at end of file
diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c
index 7d27549cad..900a1e12a0 100644
--- a/src/lib/log/git_revision.c
+++ b/src/lib/version/git_revision.c
@@ -4,7 +4,7 @@
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "lib/log/git_revision.h"
+#include "lib/version/git_revision.h"
 
 /** String describing which Tor Git repository version the source was
  * built from.  This string is generated by a bit of shell kludging in
diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h
index 79e3c6684b..79e3c6684b 100644
--- a/src/lib/log/git_revision.h
+++ b/src/lib/version/git_revision.h
diff --git a/src/lib/version/include.am b/src/lib/version/include.am
new file mode 100644
index 0000000000..0ae31be1b2
--- /dev/null
+++ b/src/lib/version/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-version.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-version-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_version_a_SOURCES =			\
+	src/lib/version/git_revision.c			\
+	src/lib/version/version.c
+
+src_lib_libtor_version_testing_a_SOURCES = \
+	$(src_lib_libtor_version_a_SOURCES)
+src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# Declare that these object files depend on micro-revision.i.  Without this
+# rule, we could try to build them before micro-revision.i was created.
+src/lib/version/git_revision.$(OBJEXT) \
+  src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \
+	micro-revision.i
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/version/git_revision.h			\
+	src/lib/version/torversion.h
diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h
new file mode 100644
index 0000000000..7b0fb66ec0
--- /dev/null
+++ b/src/lib/version/torversion.h
@@ -0,0 +1,12 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_VERSION_H
+#define TOR_VERSION_H
+
+const char *get_version(void);
+const char *get_short_version(void);
+
+#endif /* !defined(TOR_VERSION_H) */
diff --git a/src/lib/version/version.c b/src/lib/version/version.c
new file mode 100644
index 0000000000..434e6fb424
--- /dev/null
+++ b/src/lib/version/version.c
@@ -0,0 +1,50 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/version/torversion.h"
+#include "lib/version/git_revision.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/** A shorter version of this Tor process's version, for export in our router
+ *  descriptor.  (Does not include the git version, if any.) */
+static const char the_short_tor_version[] =
+  VERSION
+#ifdef TOR_BUILD_TAG
+  " ("TOR_BUILD_TAG")"
+#endif
+  "";
+
+#define MAX_VERSION_LEN 128
+
+/** The version of this Tor process, possibly including git version */
+static char the_tor_version[MAX_VERSION_LEN] = "";
+
+/** Return the current Tor version. */
+const char *
+get_version(void)
+{
+  if (the_tor_version[0] == 0) {
+    if (strlen(tor_git_revision)) {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s (git-%s)", the_short_tor_version, tor_git_revision);
+    } else {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s", the_short_tor_version);
+    }
+    the_tor_version[sizeof(the_tor_version)-1] = 0;
+  }
+
+  return the_tor_version;
+}
+
+/** Return the current Tor version, without any git tag. */
+const char *
+get_short_version(void)
+{
+  return the_short_tor_version;
+}
diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include
index dc010da063..ce7a26472b 100644
--- a/src/lib/wallclock/.may_include
+++ b/src/lib/wallclock/.may_include
@@ -3,4 +3,5 @@ lib/cc/*.h
 lib/err/*.h
 lib/wallclock/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c
index ee498702d5..77eeddaf56 100644
--- a/src/lib/wallclock/approx_time.c
+++ b/src/lib/wallclock/approx_time.c
@@ -9,7 +9,9 @@
  **/
 
 #include "orconfig.h"
+#include "lib/subsys/subsys.h"
 #include "lib/wallclock/approx_time.h"
+#include "lib/wallclock/wallclock_sys.h"
 
 /* =====
  * Cached time
@@ -41,3 +43,19 @@ update_approx_time(time_t now)
   cached_approx_time = now;
 }
 #endif /* !defined(TIME_IS_FAST) */
+
+static int
+subsys_wallclock_initialize(void)
+{
+  update_approx_time(time(NULL));
+  return 0;
+}
+
+const subsys_fns_t sys_wallclock = {
+  .name = "wallclock",
+  .supported = true,
+  /* Approximate time is a diagnostic feature, we want it to init right after
+   * low-level error handling. */
+  .level = -98,
+  .initialize = subsys_wallclock_initialize,
+};
diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h
index e6b53f2c27..e7da160122 100644
--- a/src/lib/wallclock/approx_time.h
+++ b/src/lib/wallclock/approx_time.h
@@ -22,4 +22,4 @@ time_t approx_time(void);
 void update_approx_time(time_t now);
 #endif /* defined(TIME_IS_FAST) */
 
-#endif
+#endif /* !defined(TOR_APPROX_TIME_H) */
diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am
index 1961639bd7..2b50d6ccbb 100644
--- a/src/lib/wallclock/include.am
+++ b/src/lib/wallclock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_wallclock_a_SOURCES =			\
 	src/lib/wallclock/approx_time.c			\
 	src/lib/wallclock/time_to_tm.c			\
@@ -15,8 +16,10 @@ src_lib_libtor_wallclock_testing_a_SOURCES = \
 src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/wallclock/approx_time.h			\
 	src/lib/wallclock/timeval.h			\
 	src/lib/wallclock/time_to_tm.h			\
-	src/lib/wallclock/tor_gettimeofday.h
+	src/lib/wallclock/tor_gettimeofday.h		\
+	src/lib/wallclock/wallclock_sys.h
diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h
index abe78c0efe..da27fcaba1 100644
--- a/src/lib/wallclock/time_to_tm.h
+++ b/src/lib/wallclock/time_to_tm.h
@@ -19,4 +19,4 @@ struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result,
 struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result,
                             char **err_out);
 
-#endif
+#endif /* !defined(TOR_WALLCLOCK_TIME_TO_TM_H) */
diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h
index 4967e939bf..e632d04a04 100644
--- a/src/lib/wallclock/timeval.h
+++ b/src/lib/wallclock/timeval.h
@@ -20,6 +20,27 @@
 #include <sys/time.h>
 #endif
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we use a slower definition of these macros without
+ * branches, to make coverage consistent. */
+#undef timeradd
+#undef timersub
+#define timeradd(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec;    \
+    (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;      \
+    (tvout)->tv_usec %= 1000000;                        \
+  } while (0)
+#define timersub(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec - 1;            \
+    (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec + 1000000;   \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;                  \
+    (tvout)->tv_usec %= 1000000;                                    \
+  } while (0)
+#endif /* defined(TOR_COVERAGE) */
+
 #ifndef timeradd
 /** Replacement for timeradd on platforms that do not have it: sets tvout to
  * the sum of tv1 and tv2. */
@@ -62,4 +83,4 @@
    ((tv1)->tv_sec op (tv2)->tv_sec))
 #endif /* !defined(timercmp) */
 
-#endif
+#endif /* !defined(TOR_TIMEVAL_H) */
diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h
index c7fff9747a..6fec2fc893 100644
--- a/src/lib/wallclock/tor_gettimeofday.h
+++ b/src/lib/wallclock/tor_gettimeofday.h
@@ -17,4 +17,4 @@ struct timeval;
 
 MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
 
-#endif
+#endif /* !defined(TOR_GETTIMEOFDAY_H) */
diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h
new file mode 100644
index 0000000000..a30912b8fb
--- /dev/null
+++ b/src/lib/wallclock/wallclock_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file wallclock_sys.h
+ * \brief Declare subsystem object for the wallclock module.
+ **/
+
+#ifndef TOR_WALLCLOCK_SYS_H
+#define TOR_WALLCLOCK_SYS_H
+
+extern const struct subsys_fns_t sys_wallclock;
+
+#endif /* !defined(TOR_WALLCLOCK_SYS_H) */
diff --git a/src/rust/build.rs b/src/rust/build.rs
index 123d5c0682..5626b35f75 100644
--- a/src/rust/build.rs
+++ b/src/rust/build.rs
@@ -149,8 +149,9 @@ pub fn main() {
             cfg.component("tor-sandbox-testing");
             cfg.component("tor-encoding-testing");
             cfg.component("tor-fs-testing");
-            cfg.component("tor-time-testing");
             cfg.component("tor-net-testing");
+            cfg.component("tor-buf-testing");
+            cfg.component("tor-time-testing");
             cfg.component("tor-thread-testing");
             cfg.component("tor-memarea-testing");
             cfg.component("tor-log-testing");
@@ -162,6 +163,7 @@ pub fn main() {
             cfg.component("tor-malloc");
             cfg.component("tor-wallclock");
             cfg.component("tor-err-testing");
+            cfg.component("tor-version-testing");
             cfg.component("tor-intmath-testing");
             cfg.component("tor-ctime-testing");
             cfg.component("curve25519_donna");
diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs
index 6ee63adb10..14170d0353 100644
--- a/src/rust/protover/ffi.rs
+++ b/src/rust/protover/ffi.rs
@@ -30,6 +30,8 @@ fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> {
         7 => Ok(Protocol::Desc),
         8 => Ok(Protocol::Microdesc),
         9 => Ok(Protocol::Cons),
+        10 => Ok(Protocol::Padding),
+        11 => Ok(Protocol::FlowCtrl),
         _ => Err(ProtoverError::UnknownProtocol),
     }
 }
diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs
index 06fdf56c69..0ca960bd69 100644
--- a/src/rust/protover/protover.rs
+++ b/src/rust/protover/protover.rs
@@ -46,6 +46,8 @@ pub enum Protocol {
     LinkAuth,
     Microdesc,
     Relay,
+    Padding,
+    FlowCtrl,
 }
 
 impl fmt::Display for Protocol {
@@ -73,6 +75,8 @@ impl FromStr for Protocol {
             "LinkAuth" => Ok(Protocol::LinkAuth),
             "Microdesc" => Ok(Protocol::Microdesc),
             "Relay" => Ok(Protocol::Relay),
+            "Padding" => Ok(Protocol::Padding),
+            "FlowCtrl" => Ok(Protocol::FlowCtrl),
             _ => Err(ProtoverError::UnknownProtocol),
         }
     }
@@ -163,7 +167,9 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
              Link=1-5 \
              LinkAuth=3 \
              Microdesc=1-2 \
-             Relay=1-2"
+             Relay=1-2 \
+             Padding=2 \
+             FlowCtrl=1"
         )
     } else {
         cstr!(
@@ -176,7 +182,9 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
              Link=1-5 \
              LinkAuth=1,3 \
              Microdesc=1-2 \
-             Relay=1-2"
+             Relay=1-2 \
+             Padding=2 \
+             FlowCtrl=1"
         )
     }
 }
diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs
index 98fccba5a9..bbaf97129c 100644
--- a/src/rust/tor_log/tor_log.rs
+++ b/src/rust/tor_log/tor_log.rs
@@ -99,14 +99,14 @@ pub mod log {
     /// Domain log types. These mirror definitions in src/lib/log/log.h
     /// C_RUST_COUPLED: src/lib/log/log.c, log severity types
     extern "C" {
-        static LD_NET_: u32;
-        static LD_GENERAL_: u32;
+        static LD_NET_: u64;
+        static LD_GENERAL_: u64;
     }
 
     /// Translate Rust defintions of log domain levels to C. This exposes a 1:1
     /// mapping between types.
     #[inline]
-    pub unsafe fn translate_domain(domain: LogDomain) -> u32 {
+    pub unsafe fn translate_domain(domain: LogDomain) -> u64 {
         match domain {
             LogDomain::Net => LD_NET_,
             LogDomain::General => LD_GENERAL_,
@@ -128,7 +128,7 @@ pub mod log {
     extern "C" {
         pub fn tor_log_string(
             severity: c_int,
-            domain: u32,
+            domain: u64,
             function: *const c_char,
             string: *const c_char,
         );
diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs
index 2a19458f46..ede42c6ea8 100644
--- a/src/rust/tor_util/strings.rs
+++ b/src/rust/tor_util/strings.rs
@@ -105,11 +105,7 @@ macro_rules! cstr {
     ($($bytes:expr),*) => (
         ::std::ffi::CStr::from_bytes_with_nul(
             concat!($($bytes),*, "\0").as_bytes()
-        ).unwrap_or(
-            unsafe{
-                ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0")
-            }
-        )
+        ).unwrap_or_default()
     )
 }
 
diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake
index cfbe281b94..ca6a84cf8a 100644
--- a/src/test/Makefile.nmake
+++ b/src/test/Makefile.nmake
@@ -1,4 +1,4 @@
-all: test.exe test-child.exe bench.exe
+all: test.exe bench.exe
 
 CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \
     /I ..\ext
@@ -19,6 +19,7 @@ TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \
 	test_cell_formats.obj test_relay.obj test_replay.obj \
 	test_channelpadding.obj \
 	test_circuitstats.obj \
+	test_circuitpadding.obj \
 	test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj
 
 tinytest.obj: ..\ext\tinytest.c
@@ -30,8 +31,5 @@ test.exe: $(TEST_OBJECTS)
 bench.exe: bench.obj
 	$(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@
 
-test-child.exe: test-child.obj
-	$(CC) $(CFLAGS) test-child.obj /Fe$@
-
 clean:
-	del *.obj *.lib test.exe bench.exe test-child.exe
+	del *.obj *.lib test.exe bench.exe
diff --git a/src/test/bench.c b/src/test/bench.c
index 06c616c3b0..cf732df593 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -14,16 +14,19 @@
 #include "core/crypto/onion_tap.h"
 #include "core/crypto/relay_crypto.h"
 
+#include "lib/intmath/weakrng.h"
+
 #ifdef ENABLE_OPENSSL
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #include <openssl/ec.h>
 #include <openssl/ecdh.h>
 #include <openssl/obj_mac.h>
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "core/or/circuitlist.h"
 #include "app/config/config.h"
+#include "app/main/subsysmgr.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "core/crypto/onion_ntor.h"
@@ -38,6 +41,9 @@
 #include "lib/crypt_ops/digestset.h"
 #include "lib/crypt_ops/crypto_init.h"
 
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/nodelist/microdesc.h"
+
 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
 static uint64_t nanostart;
 static inline uint64_t
@@ -332,6 +338,65 @@ bench_ed25519(void)
 }
 
 static void
+bench_rand_len(int len)
+{
+  const int N = 100000;
+  int i;
+  char *buf = tor_malloc(len);
+  uint64_t start,end;
+
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_rand(buf, len);
+  }
+  end = perftime();
+  printf("crypto_rand(%d): %f nsec.\n", len, NANOCOUNT(start,end,N));
+
+  crypto_fast_rng_t *fr = crypto_fast_rng_new();
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_fast_rng_getbytes(fr,(uint8_t*)buf,len);
+  }
+  end = perftime();
+  printf("crypto_fast_rng_getbytes(%d): %f nsec.\n", len,
+         NANOCOUNT(start,end,N));
+  crypto_fast_rng_free(fr);
+
+  if (len <= 32) {
+    start = perftime();
+    for (i = 0; i < N; ++i) {
+      crypto_strongest_rand((uint8_t*)buf, len);
+    }
+    end = perftime();
+    printf("crypto_strongest_rand(%d): %f nsec.\n", len,
+           NANOCOUNT(start,end,N));
+  }
+
+  if (len == 4) {
+    tor_weak_rng_t weak;
+    tor_init_weak_random(&weak, 1337);
+
+    start = perftime();
+    uint32_t t=0;
+    for (i = 0; i < N; ++i) {
+      t += tor_weak_random(&weak);
+    }
+    end = perftime();
+    printf("weak_rand(4): %f nsec.\n", NANOCOUNT(start,end,N));
+  }
+
+  tor_free(buf);
+}
+
+static void
+bench_rand(void)
+{
+  bench_rand_len(4);
+  bench_rand_len(16);
+  bench_rand_len(128);
+}
+
+static void
 bench_cell_aes(void)
 {
   uint64_t start, end;
@@ -636,7 +701,42 @@ bench_ecdh_p224(void)
 {
   bench_ecdh_impl(NID_secp224r1, "P-224");
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+
+static void
+bench_md_parse(void)
+{
+  uint64_t start, end;
+  const int N = 100000;
+  // selected arbitrarily
+  const char md_text[] =
+    "@last-listed 2018-12-14 18:14:14\n"
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n"
+    "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n"
+    "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n"
+    "family $00E9A86E7733240E60D8435A7BBD634A23894098 "
+    "$329BD7545DEEEBBDC8C4285F243916F248972102 "
+    "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D "
+    "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n"
+    "p accept 53,80,443,5222-5223,25565\n"
+    "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n";
+
+  reset_perftime();
+  start = perftime();
+  for (int i = 0; i < N; ++i) {
+    smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1,
+                                                  SAVED_IN_CACHE, NULL);
+    SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md));
+    smartlist_free(s);
+  }
+
+  end = perftime();
+  printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N));
+}
 
 typedef void (*bench_fn)(void);
 
@@ -656,6 +756,7 @@ static struct benchmark_t benchmarks[] = {
   ENT(onion_TAP),
   ENT(onion_ntor),
   ENT(ed25519),
+  ENT(rand),
 
   ENT(cell_aes),
   ENT(cell_ops),
@@ -665,6 +766,8 @@ static struct benchmark_t benchmarks[] = {
   ENT(ecdh_p256),
   ENT(ecdh_p224),
 #endif
+
+  ENT(md_parse),
   {NULL,NULL,0}
 };
 
@@ -690,9 +793,10 @@ main(int argc, const char **argv)
   char *errmsg;
   or_options_t *options;
 
-  tor_threads_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
+
   tor_compress_init();
-  init_logging(1);
 
   if (argc == 4 && !strcmp(argv[1], "diff")) {
     const int N = 200;
@@ -702,11 +806,13 @@ main(int argc, const char **argv)
       perror("X");
       return 1;
     }
+    size_t f1len = strlen(f1);
+    size_t f2len = strlen(f2);
     for (i = 0; i < N; ++i) {
-      char *diff = consensus_diff_generate(f1, f2);
+      char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
       tor_free(diff);
     }
-    char *diff = consensus_diff_generate(f1, f2);
+    char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
     printf("%s", diff);
     tor_free(f1);
     tor_free(f2);
@@ -737,7 +843,6 @@ main(int argc, const char **argv)
 
   init_protocol_warning_severity_level();
   options = options_new();
-  init_logging(1);
   options->command = CMD_RUN_UNITTESTS;
   options->DataDirectory = tor_strdup("");
   options->KeyDirectory = tor_strdup("");
diff --git a/src/test/conf_examples/badnick_1/error b/src/test/conf_examples/badnick_1/error
new file mode 100644
index 0000000000..3e92ddc832
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/error
@@ -0,0 +1 @@
+nicknames must be between 1 and 19 characters inclusive
diff --git a/src/test/conf_examples/badnick_1/torrc b/src/test/conf_examples/badnick_1/torrc
new file mode 100644
index 0000000000..41ee4894f1
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/torrc
@@ -0,0 +1,2 @@
+# This nickname is too long; we won't accept it.
+Nickname TooManyCharactersInThisNickname
diff --git a/src/test/conf_examples/badnick_2/error b/src/test/conf_examples/badnick_2/error
new file mode 100644
index 0000000000..ceac99f012
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/error
@@ -0,0 +1 @@
+must contain only the characters \[a-zA-Z0-9\]
diff --git a/src/test/conf_examples/badnick_2/torrc b/src/test/conf_examples/badnick_2/torrc
new file mode 100644
index 0000000000..07acc61698
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/torrc
@@ -0,0 +1,2 @@
+# this nickname has spaces in it and won't work.
+Nickname has a space
diff --git a/src/test/conf_examples/contactinfo_notutf8/error b/src/test/conf_examples/contactinfo_notutf8/error
new file mode 100644
index 0000000000..6d165152ce
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/error
@@ -0,0 +1 @@
+ContactInfo config option must be UTF-8
diff --git a/src/test/conf_examples/contactinfo_notutf8/torrc b/src/test/conf_examples/contactinfo_notutf8/torrc
new file mode 100644
index 0000000000..2ee4d093c5
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/torrc
@@ -0,0 +1 @@
+ContactInfo ÄëÄëÄë@example.com
diff --git a/src/test/conf_examples/example_1/expected b/src/test/conf_examples/example_1/expected
new file mode 100644
index 0000000000..9d6688a565
--- /dev/null
+++ b/src/test/conf_examples/example_1/expected
@@ -0,0 +1,2 @@
+ContactInfo tor_tellini@example.com
+SocksPort 80
diff --git a/src/test/conf_examples/example_1/torrc b/src/test/conf_examples/example_1/torrc
new file mode 100644
index 0000000000..bff7fa0aa2
--- /dev/null
+++ b/src/test/conf_examples/example_1/torrc
@@ -0,0 +1,5 @@
+
+# Here is a simple example torrc.
+  SocksPort  80
+
+ContactInfo "tor_tellini@example.com"
diff --git a/src/test/conf_examples/example_2/error b/src/test/conf_examples/example_2/error
new file mode 100644
index 0000000000..ce18b68db4
--- /dev/null
+++ b/src/test/conf_examples/example_2/error
@@ -0,0 +1 @@
+Unknown option 'JumpingJellyjars'
diff --git a/src/test/conf_examples/example_2/torrc b/src/test/conf_examples/example_2/torrc
new file mode 100644
index 0000000000..8ec8133b24
--- /dev/null
+++ b/src/test/conf_examples/example_2/torrc
@@ -0,0 +1 @@
+JumpingJellyjars 1
diff --git a/src/test/conf_examples/example_3/cmdline b/src/test/conf_examples/example_3/cmdline
new file mode 100644
index 0000000000..5b2fadcebb
--- /dev/null
+++ b/src/test/conf_examples/example_3/cmdline
@@ -0,0 +1 @@
+--socksport 99
diff --git a/src/test/conf_examples/example_3/expected b/src/test/conf_examples/example_3/expected
new file mode 100644
index 0000000000..867fb8bcc8
--- /dev/null
+++ b/src/test/conf_examples/example_3/expected
@@ -0,0 +1 @@
+SocksPort 99
diff --git a/src/test/conf_examples/example_3/torrc b/src/test/conf_examples/example_3/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/example_3/torrc
diff --git a/src/test/conf_examples/include_1/expected b/src/test/conf_examples/include_1/expected
new file mode 100644
index 0000000000..4bbf52ce9f
--- /dev/null
+++ b/src/test/conf_examples/include_1/expected
@@ -0,0 +1,3 @@
+ContactInfo includefile@example.com
+Nickname nested
+ORPort 8008
diff --git a/src/test/conf_examples/include_1/included.inc b/src/test/conf_examples/include_1/included.inc
new file mode 100644
index 0000000000..8d1834345d
--- /dev/null
+++ b/src/test/conf_examples/include_1/included.inc
@@ -0,0 +1,4 @@
+
+ContactInfo includefile@example.com
+
+%include "nested.inc"
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/nested.inc b/src/test/conf_examples/include_1/nested.inc
new file mode 100644
index 0000000000..789b044a2b
--- /dev/null
+++ b/src/test/conf_examples/include_1/nested.inc
@@ -0,0 +1,2 @@
+
+Nickname nested
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/torrc b/src/test/conf_examples/include_1/torrc
new file mode 100644
index 0000000000..2ed4074f6e
--- /dev/null
+++ b/src/test/conf_examples/include_1/torrc
@@ -0,0 +1,4 @@
+
+%include "included.inc"
+
+ORPort 8008
diff --git a/src/test/conf_examples/include_bug_31408/expected b/src/test/conf_examples/include_bug_31408/expected
new file mode 100644
index 0000000000..2e822f1a78
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/expected
@@ -0,0 +1,2 @@
+Nickname test31408
+ORPort 31408
diff --git a/src/test/conf_examples/include_bug_31408/included/01_nickname.inc b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
new file mode 100644
index 0000000000..508dd89a35
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
@@ -0,0 +1 @@
+Nickname test31408
diff --git a/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
new file mode 100644
index 0000000000..140e927f19
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
@@ -0,0 +1,3 @@
+# Bug 31048 is triggered when the last file in a config directory:
+#   * contains no configuration options,
+#   * but is non-empty: that is, it contains comments or whitespace.
diff --git a/src/test/conf_examples/include_bug_31408/torrc b/src/test/conf_examples/include_bug_31408/torrc
new file mode 100644
index 0000000000..a42685e93c
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/torrc
@@ -0,0 +1,2 @@
+%include "included"
+ORPort 31408
diff --git a/src/test/conf_examples/large_1/expected b/src/test/conf_examples/large_1/expected
new file mode 100644
index 0000000000..5866f5823e
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected
@@ -0,0 +1,159 @@
+AccountingMax 10737418240
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2147483647
+BandwidthRate 1610612736
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientAutoIPv6ORPort 1
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirPortFrontPage /dirport/frontpage
+DirPort 99
+DirReqStatistics 0
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 1260
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 300
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPortCookieAuthFile /foobar
+ExtORPort 99
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 691200
+HeartbeatPeriod 2700
+IPv6Exit 1
+KeepalivePeriod 540
+KeyDirectory /keyz
+KISTSchedRunInterval 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 60000
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 3600
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 2629728
+MaxMemInQueues 314572800
+MaxOnionQueueDelay 60000
+MaxUnparseableDescSizeToLog 1048576
+MiddleNodes grommit,truffle,parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 7200
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PerConnBWBurst 10485760
+PerConnBWRate 102400
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 600
+RephistTrackTime 600
+SafeLogging 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10
+SigningKeyLifetime 4838400
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 600
+SSLKeyLifetime 86400
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1000
+TrackHostExits www.example.com
+TrackHostExitsExpire 3600
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/large_1/torrc b/src/test/conf_examples/large_1/torrc
new file mode 100644
index 0000000000..e99acd9fb7
--- /dev/null
+++ b/src/test/conf_examples/large_1/torrc
@@ -0,0 +1,167 @@
+AccountingMax 10 GB
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2 GB
+BandwidthRate 1.5 GB
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitPadding 1
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientAutoIPv6ORPort 1
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirReqStatistics 0
+DirPort 99
+DirPortFrontPage /dirport/frontpage
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 21 minutes
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 5 minutes
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPort 99
+ExtORPortCookieAuthFile /foobar
+ExtraInfoStatistics 1
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchHidServDescriptors 1
+FetchServerDescriptors 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 8 days
+HeartbeatPeriod 45 minutes
+IPv6Exit 1
+KeepalivePeriod 9 minutes
+KeyDirectory /keyz
+KISTSchedRunInterval 1 msec
+LearnCircuitBuildTimeout 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 1 minute
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 1 hour
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 1 month
+MaxMemInQueues 300 MB
+MaxOnionQueueDelay 60 seconds
+MaxUnparseableDescSizeToLog 1 MB
+MiddleNodes grommit, truffle, parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 2 hours
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PaddingStatistics 1
+PerConnBWBurst 10 MB
+PerConnBWRate 100 kb
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 10 minutes
+RephistTrackTime 10 minutes
+SafeLogging 0
+SafeSocks 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10 seconds
+SigningKeyLifetime 8 weeks
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 10 minutes
+SSLKeyLifetime 1 day
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1 second
+TrackHostExits www.example.com
+TrackHostExitsExpire 1 hour
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseEntryGuards 1
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/obsolete_1/expected b/src/test/conf_examples/obsolete_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/expected
diff --git a/src/test/conf_examples/obsolete_1/torrc b/src/test/conf_examples/obsolete_1/torrc
new file mode 100644
index 0000000000..3cd9a6d777
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/torrc
@@ -0,0 +1,68 @@
+# These options are obsolete as of 0.4.2
+AllowDotExit
+AllowInvalidNodes
+AllowSingleHopCircuits
+AllowSingleHopExits
+AlternateHSAuthority
+AuthDirBadDir
+AuthDirBadDirCCs
+AuthDirRejectUnlisted
+AuthDirListBadDirs
+AuthDirMaxServersPerAuthAddr
+CircuitIdleTimeout
+ControlListenAddress
+DirListenAddress
+DisableIOCP
+DisableV2DirectoryInfo_
+DynamicDHGroups
+DNSListenAddress
+TestingEnableTbEmptyEvent
+ExcludeSingleHopRelays
+FallbackNetworkstatusFile
+FastFirstHopPK
+FetchV2Networkstatus
+Group
+HidServDirectoryV2
+CloseHSClientCircuitsImmediatelyOnTimeout
+CloseHSServiceRendCircuitsImmediatelyOnTimeout
+MaxOnionsPending
+NamingAuthoritativeDirectory
+NATDListenAddress
+PredictedPortsRelevanceTime
+WarnUnsafeSocks
+ORListenAddress
+PathBiasDisableRate
+PathBiasScaleFactor
+PathBiasMultFactor
+PathBiasUseCloseCounts
+PortForwarding
+PortForwardingHelper
+PreferTunneledDirConns
+RecommendedPackages
+RunTesting
+SchedulerLowWaterMark__
+SchedulerHighWaterMark__
+SchedulerMaxFlushCells__
+SocksListenAddress
+StrictEntryNodes
+StrictExitNodes
+Support022HiddenServices
+Tor2webMode
+Tor2webRendezvousPoints
+TLSECGroup
+TransListenAddress
+TunnelDirConns
+UseEntryGuardsAsDirGuards
+UseNTorHandshake
+UserspaceIOCPBuffers
+V1AuthoritativeDirectory
+V2AuthoritativeDirectory
+VoteOnHidServDirectoriesV2
+UseFilteringSSLBufferevents
+__UseFilteringSSLBufferevents
+TestingConsensusMaxDownloadTries
+ClientBootstrapConsensusMaxDownloadTries
+ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries
+TestingDescriptorMaxDownloadTries
+TestingMicrodescMaxDownloadTries
+TestingCertMaxDownloadTries
diff --git a/src/test/conf_examples/obsolete_2/expected b/src/test/conf_examples/obsolete_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/expected
diff --git a/src/test/conf_examples/obsolete_2/torrc b/src/test/conf_examples/obsolete_2/torrc
new file mode 100644
index 0000000000..4f78d47625
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/torrc
@@ -0,0 +1,2 @@
+# This option has been obsolete for some time
+AllowDotExit
diff --git a/src/test/conf_examples/ops_1/cmdline b/src/test/conf_examples/ops_1/cmdline
new file mode 100644
index 0000000000..2bb9bfa132
--- /dev/null
+++ b/src/test/conf_examples/ops_1/cmdline
@@ -0,0 +1 @@
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/expected b/src/test/conf_examples/ops_1/expected
new file mode 100644
index 0000000000..84be6a70e2
--- /dev/null
+++ b/src/test/conf_examples/ops_1/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/torrc b/src/test/conf_examples/ops_1/torrc
new file mode 100644
index 0000000000..daf8ae60fe
--- /dev/null
+++ b/src/test/conf_examples/ops_1/torrc
@@ -0,0 +1,3 @@
+# We'll replace this option on the command line.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_2/cmdline b/src/test/conf_examples/ops_2/cmdline
new file mode 100644
index 0000000000..fdd48a045c
--- /dev/null
+++ b/src/test/conf_examples/ops_2/cmdline
@@ -0,0 +1 @@
+/ORPort
diff --git a/src/test/conf_examples/ops_2/expected b/src/test/conf_examples/ops_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_2/expected
diff --git a/src/test/conf_examples/ops_2/torrc b/src/test/conf_examples/ops_2/torrc
new file mode 100644
index 0000000000..21fcc93f9a
--- /dev/null
+++ b/src/test/conf_examples/ops_2/torrc
@@ -0,0 +1,3 @@
+# We'll remove this option on the command line, and not replace it.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_3/cmdline b/src/test/conf_examples/ops_3/cmdline
new file mode 100644
index 0000000000..e4965d26f8
--- /dev/null
+++ b/src/test/conf_examples/ops_3/cmdline
@@ -0,0 +1 @@
++ORPort 1000
diff --git a/src/test/conf_examples/ops_3/expected b/src/test/conf_examples/ops_3/expected
new file mode 100644
index 0000000000..569d26b577
--- /dev/null
+++ b/src/test/conf_examples/ops_3/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9999
+ORPort 1000
diff --git a/src/test/conf_examples/ops_3/torrc b/src/test/conf_examples/ops_3/torrc
new file mode 100644
index 0000000000..14adf87d7f
--- /dev/null
+++ b/src/test/conf_examples/ops_3/torrc
@@ -0,0 +1,3 @@
+# We will extend this option on the command line
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_4/expected b/src/test/conf_examples/ops_4/expected
new file mode 100644
index 0000000000..bf52f6a330
--- /dev/null
+++ b/src/test/conf_examples/ops_4/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/torrc b/src/test/conf_examples/ops_4/torrc
new file mode 100644
index 0000000000..dcec2aa95d
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc
@@ -0,0 +1,3 @@
+# This value is unadorned, so replaces the one from defaults.torrc.
+
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/torrc.defaults b/src/test/conf_examples/ops_4/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_5/expected b/src/test/conf_examples/ops_5/expected
new file mode 100644
index 0000000000..288721da53
--- /dev/null
+++ b/src/test/conf_examples/ops_5/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9000
+ORPort 9099
diff --git a/src/test/conf_examples/ops_5/torrc b/src/test/conf_examples/ops_5/torrc
new file mode 100644
index 0000000000..3284fc1c55
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc
@@ -0,0 +1,3 @@
+# This value has a plus, and so extends the one from defaults.torrc.
+
++ORPort 9099
diff --git a/src/test/conf_examples/ops_5/torrc.defaults b/src/test/conf_examples/ops_5/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_6/expected b/src/test/conf_examples/ops_6/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_6/expected
diff --git a/src/test/conf_examples/ops_6/torrc b/src/test/conf_examples/ops_6/torrc
new file mode 100644
index 0000000000..4d51caaff7
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc
@@ -0,0 +1,3 @@
+# This value has a slash, and so clears the one from defaults.torrc.
+
+/ORPort
diff --git a/src/test/conf_examples/ops_6/torrc.defaults b/src/test/conf_examples/ops_6/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/relpath_rad/error b/src/test/conf_examples/relpath_rad/error
new file mode 100644
index 0000000000..e131744475
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/error
@@ -0,0 +1 @@
+RunAsDaemon is not compatible with relative paths.
diff --git a/src/test/conf_examples/relpath_rad/torrc b/src/test/conf_examples/relpath_rad/torrc
new file mode 100644
index 0000000000..fe02441c3f
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/torrc
@@ -0,0 +1,4 @@
+
+# Relative-path data directories are incompatible with RunAsDaemon
+DataDirectory ./datadir
+RunAsDaemon 1
diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh
index 68efc1abc5..f730d532a5 100755
--- a/src/test/fuzz/fixup_filenames.sh
+++ b/src/test/fuzz/fixup_filenames.sh
@@ -8,9 +8,9 @@ if [ ! -d "$1" ] ; then
 fi
 
 for fn in "$1"/* ; do
-    prev=`basename "$fn"`
-    post=`sha256sum "$fn" | sed -e 's/ .*//;'`
-    if [ "$prev" == "$post" ] ; then
+    prev=$(basename "$fn")
+    post=$(sha256sum "$fn" | sed -e 's/ .*//;')
+    if [ "$prev" = "$post" ] ; then
       echo "OK $prev"
     else
       echo "mv $prev $post"
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
index 5947a3f48c..656ef0bdb2 100644
--- a/src/test/fuzz/fuzz_consensus.c
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -61,13 +61,13 @@ int
 fuzz_main(const uint8_t *data, size_t sz)
 {
   networkstatus_t *ns;
-  char *str = tor_memdup_nulterm(data, sz);
   const char *eos = NULL;
   networkstatus_type_t tp = NS_TYPE_CONSENSUS;
   if (tor_memstr(data, MIN(sz, 1024), "tus vote"))
     tp = NS_TYPE_VOTE;
   const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote";
-  ns = networkstatus_parse_vote_from_string(str,
+  ns = networkstatus_parse_vote_from_string((const char *)data,
+                                            sz,
                                             &eos,
                                             tp);
   if (ns) {
@@ -76,6 +76,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   } else {
     log_debug(LD_GENERAL, "Parsing as %s failed", what);
   }
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c
index 1bc60e50ee..a31445666c 100644
--- a/src/test/fuzz/fuzz_diff.c
+++ b/src/test/fuzz/fuzz_diff.c
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -42,28 +44,34 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_generate(c1, c2);
+  const char *cp = memchr(c1, 0, c1_len);
+  if (cp)
+    c1_len = cp - c1;
+
+  cp = memchr(c2, 0, c2_len);
+  if (cp)
+    c2_len = cp - c2;
+
+  char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len);
 
   if (c3) {
-    char *c4 = consensus_diff_apply(c1, c3);
+    char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3));
     tor_assert(c4);
-    if (strcmp(c2, c4)) {
-      printf("%s\n", escaped(c1));
-      printf("%s\n", escaped(c2));
+    int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len);
+    if (! equal) {
+      //printf("%s\n", escaped(c1));
+      //printf("%s\n", escaped(c2));
       printf("%s\n", escaped(c3));
       printf("%s\n", escaped(c4));
     }
-    tor_assert(! strcmp(c2, c4));
+    tor_assert(equal);
     tor_free(c3);
     tor_free(c4);
   }
-  tor_free(c1);
-  tor_free(c2);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c
index 9bd3cb0bf8..d8a0f9e590 100644
--- a/src/test/fuzz/fuzz_diff_apply.c
+++ b/src/test/fuzz/fuzz_diff_apply.c
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_apply(c1, c2);
+  char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len);
 
-  tor_free(c1);
-  tor_free(c2);
   tor_free(c3);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c
index 2cbd655898..9d4a6dbb55 100644
--- a/src/test/fuzz/fuzz_hsdescv3.c
+++ b/src/test/fuzz/fuzz_hsdescv3.c
@@ -35,16 +35,21 @@ mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert,
 
 static size_t
 mock_decrypt_desc_layer(const hs_descriptor_t *desc,
-                        const uint8_t *encrypted_blob,
-                        size_t encrypted_blob_size,
                         const uint8_t *descriptor_cookie,
-                        int is_superencrypted_layer,
+                        bool is_superencrypted_layer,
                         char **decrypted_out)
 {
   (void)is_superencrypted_layer;
   (void)desc;
   (void)descriptor_cookie;
   const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
+
   if (encrypted_blob_size < overhead)
     return 0;
   *decrypted_out = tor_memdup_nulterm(
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
index 2798c47d23..44393b3a10 100644
--- a/src/test/fuzz/fuzz_http.c
+++ b/src/test/fuzz/fuzz_http.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/dircache/dircache.h"
diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c
index a60fc36804..2a597cae74 100644
--- a/src/test/fuzz/fuzz_http_connect.c
+++ b/src/test/fuzz/fuzz_http_connect.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh
index b4a17ed8cb..406ab498d9 100755
--- a/src/test/fuzz/fuzz_multi.sh
+++ b/src/test/fuzz/fuzz_multi.sh
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 MEMLIMIT_BYTES=21990500990976
 
 N_CPUS=1
@@ -6,9 +8,9 @@ if [ $# -ge 1 ]; then
     shift
 fi
 
-FILTER=echo
+FILTER="echo"
 
-for i in `seq -w "$N_CPUS"`; do
+for i in $(seq -w "$N_CPUS"); do
     if [ "$i" -eq 1 ]; then
         if [ "$N_CPUS" -eq 1 ]; then
             INSTANCE=""
diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c
index 06cb08391e..d6c416a0f9 100644
--- a/src/test/fuzz/fuzz_socks.c
+++ b/src/test/fuzz/fuzz_socks.c
@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/err/backtrace.h"
 #include "lib/log/log.h"
 #include "core/proto/proto_socks.h"
diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c
new file mode 100644
index 0000000000..459b4e21aa
--- /dev/null
+++ b/src/test/fuzz/fuzz_strops.c
@@ -0,0 +1,253 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fuzz_strops.c
+ * \brief Fuzzers for various string encoding/decoding operations
+ **/
+
+#include "orconfig.h"
+
+#include "lib/cc/torint.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/confline.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/muldiv.h"
+
+#include "test/fuzz/fuzzing.h"
+
+#include <stdio.h>
+#include <string.h>
+
+int
+fuzz_init(void)
+{
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+typedef struct chunk_t {
+  uint8_t *buf;
+  size_t len;
+} chunk_t;
+
+#define chunk_free(ch)                          \
+  FREE_AND_NULL(chunk_t, chunk_free_, (ch))
+
+static chunk_t *
+chunk_new(size_t len)
+{
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = tor_malloc(len);
+  ch->len = len;
+  return ch;
+}
+static void
+chunk_free_(chunk_t *ch)
+{
+  if (!ch)
+    return;
+  tor_free(ch->buf);
+  tor_free(ch);
+}
+static bool
+chunk_eq(const chunk_t *a, const chunk_t *b)
+{
+  return a->len == b->len && fast_memeq(a->buf, b->buf, a->len);
+}
+
+static chunk_t *
+b16_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2));
+  int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b16_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len * 2 + 1);
+  base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  return ch;
+}
+
+static chunk_t *
+b32_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX
+  int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b32_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(base32_encoded_size(inp->len));
+  base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+b64_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter.
+  int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b64_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len));
+  base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+c_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  char *r = NULL;
+  (void) unescape_string(s, &r, &ch->len);
+  tor_free(s);
+  ch->buf = (uint8_t*) r;
+  if (!ch->buf) {
+    tor_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+c_enc(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = (uint8_t*)esc_for_log(s);
+  tor_free(s);
+  ch->len = strlen((char*)ch->buf);
+  return ch;
+}
+
+static int kv_flags = 0;
+static config_line_t *
+kv_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  config_line_t *res = kvline_parse(s, kv_flags);
+  tor_free(s);
+  return res;
+}
+static chunk_t *
+kv_enc(const config_line_t *inp)
+{
+  char *s = kvline_encode(inp, kv_flags);
+  if (!s)
+    return NULL;
+  chunk_t *res = tor_malloc(sizeof(chunk_t));
+  res->buf = (uint8_t*)s;
+  res->len = strlen(s);
+  return res;
+}
+
+/* Given an encoder function, a decoder function, and a function to free
+ * the decoded object, check whether any string that successfully decoded
+ * will then survive an encode-decode-encode round-trip unchanged.
+ */
+#define ENCODE_ROUNDTRIP(E,D,FREE)              \
+  STMT_BEGIN {                                  \
+    bool err = false;                           \
+    a = D(&inp);                                \
+    if (!a)                                     \
+      return 0;                                 \
+    b = E(a);                                   \
+    tor_assert(b);                              \
+    c = D(b);                                   \
+    tor_assert(c);                              \
+    d = E(c);                                   \
+    tor_assert(d);                              \
+    if (!chunk_eq(b,d)) {                       \
+      printf("Unequal chunks: %s\n",            \
+             hex_str((char*)b->buf, b->len));   \
+      printf("             vs %s\n",            \
+             hex_str((char*)d->buf, d->len));   \
+      err = true;                               \
+    }                                           \
+    FREE(a);                                    \
+    chunk_free(b);                              \
+    FREE(c);                                    \
+    chunk_free(d);                              \
+    tor_assert(!err);                           \
+  } STMT_END
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  if (!data_size)
+    return 0;
+
+  chunk_t inp = { (uint8_t*)stdin_buf, data_size };
+  chunk_t *b=NULL,*d=NULL;
+  void *a=NULL,*c=NULL;
+
+  switch (stdin_buf[0]) {
+    case 0:
+      ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_);
+      break;
+    case 1:
+      ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_);
+      break;
+    case 2:
+      ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_);
+      break;
+    case 3:
+      ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_);
+      break;
+    case 5:
+      kv_flags = KV_QUOTED|KV_OMIT_KEYS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 6:
+      kv_flags = 0;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 7:
+      kv_flags = KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 8:
+      kv_flags = KV_QUOTED;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 9:
+      kv_flags = KV_QUOTED|KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    }
+
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
index 967397d1af..7b61b8df2d 100644
--- a/src/test/fuzz/fuzz_vrs.c
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -3,6 +3,7 @@
 #define NS_PARSE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #include "core/or/or.h"
+#include "feature/dirauth/dirvote.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/dirparse/unparseable.h"
 #include "lib/memarea/memarea.h"
@@ -35,9 +36,12 @@ fuzz_init(void)
   dummy_vote = tor_malloc_zero(sizeof(*dummy_vote));
   dummy_vote->known_flags = smartlist_new();
   smartlist_split_string(dummy_vote->known_flags,
-                         "Authority BadExit Exit Fast Guard HSDir "
-                         "NoEdConsensus Running Stable V2Dir Valid",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          " ", 0, 0);
+  smartlist_split_string(dummy_vote->known_flags,
+                         DIRVOTE_OPTIONAL_FLAGS,
+                         " ", 0, 0);
+  smartlist_sort_strings(dummy_vote->known_flags);
   return 0;
 }
 
@@ -53,24 +57,24 @@ fuzz_cleanup(void)
 int
 fuzz_main(const uint8_t *data, size_t sz)
 {
-  char *str = tor_memdup_nulterm(data, sz);
   const char *s;
   routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL;
   vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs));
   smartlist_t *tokens = smartlist_new();
+  const char *eos = (const char *)data + sz;
 
-  s = str;
-  rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_md = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_MICRODESC);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                               dummy_vote, vrs, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
@@ -82,6 +86,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   vote_routerstatus_free(vrs);
   memarea_clear(area);
   smartlist_free(tokens);
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h
index 150ac4aa7d..2d278825ec 100644
--- a/src/test/fuzz/fuzzing.h
+++ b/src/test/fuzz/fuzzing.h
@@ -9,5 +9,5 @@ int fuzz_main(const uint8_t *data, size_t sz);
 
 void disable_signature_checking(void);
 
-#endif /* FUZZING_H */
+#endif /* !defined(FUZZING_H) */
 
diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c
index 8ea4898522..604aba7a7f 100644
--- a/src/test/fuzz/fuzzing_common.c
+++ b/src/test/fuzz/fuzzing_common.c
@@ -1,14 +1,17 @@
 /* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define CRYPTO_ED25519_PRIVATE
+#define CONFIG_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
+#include "app/main/subsysmgr.h"
 #include "lib/err/backtrace.h"
 #include "app/config/config.h"
 #include "test/fuzz/fuzzing.h"
 #include "lib/compress/compress.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 static or_options_t *mock_options = NULL;
 static const or_options_t *
@@ -94,12 +97,10 @@ disable_signature_checking(void)
 static void
 global_init(void)
 {
-  tor_threads_init();
-  tor_compress_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
 
-  /* Initialise logging first */
-  init_logging(1);
-  configure_backtrace_handler(get_version());
+  tor_compress_init();
 
   if (crypto_global_init(0, NULL, NULL) < 0)
     abort();
@@ -111,7 +112,7 @@ global_init(void)
   }
 
   /* set up the options. */
-  mock_options = tor_malloc_zero(sizeof(or_options_t));
+  mock_options = options_new();
   MOCK(get_options, mock_get_options);
 
   /* Make BUG() and nonfatal asserts crash */
@@ -137,7 +138,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
   return fuzz_main(Data, Size);
 }
 
-#else /* Not LLVM_FUZZ, so AFL. */
+#else /* !defined(LLVM_FUZZ) */
 
 int
 main(int argc, char **argv)
@@ -166,7 +167,7 @@ main(int argc, char **argv)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
 
@@ -189,9 +190,9 @@ main(int argc, char **argv)
   if (fuzz_cleanup() < 0)
     abort();
 
-  tor_free(mock_options);
+  or_options_free(mock_options);
   UNMOCK(get_options);
   return 0;
 }
 
-#endif
+#endif /* defined(LLVM_FUZZ) */
diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am
index 27eeced8c5..d0711f05d6 100644
--- a/src/test/fuzz/include.am
+++ b/src/test/fuzz/include.am
@@ -153,6 +153,16 @@ src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_strops_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_strops.c
+src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_fuzz_vrs_SOURCES = \
 	src/test/fuzz/fuzzing_common.c \
 	src/test/fuzz/fuzz_vrs.c
@@ -176,6 +186,7 @@ FUZZERS = \
 	src/test/fuzz/fuzz-iptsv2 \
 	src/test/fuzz/fuzz-microdesc \
 	src/test/fuzz/fuzz-socks \
+	src/test/fuzz/fuzz-strops \
 	src/test/fuzz/fuzz-vrs
 endif
 
@@ -291,6 +302,15 @@ src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_strops_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_lf_fuzz_vrs_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
@@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \
 	src/test/fuzz/lf-fuzz-iptsv2 \
 	src/test/fuzz/lf-fuzz-microdesc \
 	src/test/fuzz/lf-fuzz-socks \
+	src/test/fuzz/lf-fuzz-strops \
 	src/test/fuzz/lf-fuzz-vrs
 
 else
@@ -406,6 +427,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
@@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \
 	src/test/fuzz/liboss-fuzz-iptsv2.a \
 	src/test/fuzz/liboss-fuzz-microdesc.a \
 	src/test/fuzz/liboss-fuzz-socks.a \
+	src/test/fuzz/liboss-fuzz-strops.a \
 	src/test/fuzz/liboss-fuzz-vrs.a
 
 else
diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh
index 87d3dda13c..ce43812bb8 100755
--- a/src/test/fuzz/minimize.sh
+++ b/src/test/fuzz/minimize.sh
@@ -7,7 +7,7 @@ if [ ! -d "$1" ] ; then
     exit 1
 fi
 
-which=`basename "$1"`
+which=$(basename "$1")
 
 mkdir "$1.out"
 afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}"
diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh
index f7b3adffb1..b883352402 100755
--- a/src/test/fuzz_static_testcases.sh
+++ b/src/test/fuzz_static_testcases.sh
@@ -14,7 +14,7 @@ fi
 
 
 for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do
-    f=`basename $fuzzer`
+    f=$(basename "$fuzzer")
     case="${f#fuzz-}"
     if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then
         echo "Running tests for ${case}"
diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c
index f2ae8398df..0a21fe576b 100644
--- a/src/test/hs_test_helpers.c
+++ b/src/test/hs_test_helpers.c
@@ -21,26 +21,35 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
   /* For a usable intro point we need at least two link specifiers: One legacy
    * keyid and one ipv4 */
   {
-    hs_desc_link_specifier_t *ls_legacy = tor_malloc_zero(sizeof(*ls_legacy));
-    hs_desc_link_specifier_t *ls_v4 = tor_malloc_zero(sizeof(*ls_v4));
-    ls_legacy->type = LS_LEGACY_ID;
-    memcpy(ls_legacy->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8",
-           DIGEST_LEN);
-    ls_v4->u.ap.port = 9001;
-    int family = tor_addr_parse(&ls_v4->u.ap.addr, addr);
+    tor_addr_t a;
+    tor_addr_make_unspec(&a);
+    link_specifier_t *ls_legacy = link_specifier_new();
+    link_specifier_t *ls_ip = link_specifier_new();
+    link_specifier_set_ls_type(ls_legacy, LS_LEGACY_ID);
+    memset(link_specifier_getarray_un_legacy_id(ls_legacy), 'C',
+           link_specifier_getlen_un_legacy_id(ls_legacy));
+    int family = tor_addr_parse(&a, addr);
     switch (family) {
     case AF_INET:
-          ls_v4->type = LS_IPV4;
+          link_specifier_set_ls_type(ls_ip, LS_IPV4);
+          link_specifier_set_un_ipv4_addr(ls_ip, tor_addr_to_ipv4h(&a));
+          link_specifier_set_un_ipv4_port(ls_ip, 9001);
           break;
         case AF_INET6:
-          ls_v4->type = LS_IPV6;
+          link_specifier_set_ls_type(ls_ip, LS_IPV6);
+          memcpy(link_specifier_getarray_un_ipv6_addr(ls_ip),
+                 tor_addr_to_in6_addr8(&a),
+                 link_specifier_getlen_un_ipv6_addr(ls_ip));
+          link_specifier_set_un_ipv6_port(ls_ip, 9001);
           break;
         default:
-          /* Stop the test, not suppose to have an error. */
-          tt_int_op(family, OP_EQ, AF_INET);
+          /* Stop the test, not supposed to have an error.
+           * Compare with -1 to show the actual family.
+           */
+          tt_int_op(family, OP_EQ, -1);
     }
     smartlist_add(ip->link_specifiers, ls_legacy);
-    smartlist_add(ip->link_specifiers, ls_v4);
+    smartlist_add(ip->link_specifiers, ls_ip);
   }
 
   ret = ed25519_keypair_generate(&auth_kp, 0);
@@ -202,7 +211,6 @@ void
 hs_helper_desc_equal(const hs_descriptor_t *desc1,
                      const hs_descriptor_t *desc2)
 {
-  char *addr1 = NULL, *addr2 = NULL;
   /* Plaintext data section. */
   tt_int_op(desc1->plaintext_data.version, OP_EQ,
             desc2->plaintext_data.version);
@@ -291,35 +299,57 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
       tt_int_op(smartlist_len(ip1->link_specifiers), ==,
                 smartlist_len(ip2->link_specifiers));
       for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) {
-        hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
-                                 *ls2 = smartlist_get(ip2->link_specifiers, j);
-        tt_int_op(ls1->type, ==, ls2->type);
-        switch (ls1->type) {
+        link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
+                         *ls2 = smartlist_get(ip2->link_specifiers, j);
+        tt_int_op(link_specifier_get_ls_type(ls1), ==,
+                  link_specifier_get_ls_type(ls2));
+        switch (link_specifier_get_ls_type(ls1)) {
           case LS_IPV4:
+            {
+              uint32_t addr1 = link_specifier_get_un_ipv4_addr(ls1);
+              uint32_t addr2 = link_specifier_get_un_ipv4_addr(ls2);
+              tt_int_op(addr1, OP_EQ, addr2);
+              uint16_t port1 = link_specifier_get_un_ipv4_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv4_port(ls2);
+              tt_int_op(port1, ==, port2);
+            }
+            break;
           case LS_IPV6:
             {
-              addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr);
-              addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr);
-              tt_str_op(addr1, OP_EQ, addr2);
-              tor_free(addr1);
-              tor_free(addr2);
-              tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port);
+              const uint8_t *addr1 =
+                link_specifier_getconstarray_un_ipv6_addr(ls1);
+              const uint8_t *addr2 =
+                link_specifier_getconstarray_un_ipv6_addr(ls2);
+              tt_int_op(link_specifier_getlen_un_ipv6_addr(ls1), OP_EQ,
+                        link_specifier_getlen_un_ipv6_addr(ls2));
+              tt_mem_op(addr1, OP_EQ, addr2,
+                        link_specifier_getlen_un_ipv6_addr(ls1));
+              uint16_t port1 = link_specifier_get_un_ipv6_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv6_port(ls2);
+              tt_int_op(port1, ==, port2);
             }
             break;
           case LS_LEGACY_ID:
-            tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id,
-                      sizeof(ls1->u.legacy_id));
+            {
+              const uint8_t *id1 =
+                link_specifier_getconstarray_un_legacy_id(ls1);
+              const uint8_t *id2 =
+                link_specifier_getconstarray_un_legacy_id(ls2);
+              tt_int_op(link_specifier_getlen_un_legacy_id(ls1), OP_EQ,
+                        link_specifier_getlen_un_legacy_id(ls2));
+              tt_mem_op(id1, OP_EQ, id2,
+                        link_specifier_getlen_un_legacy_id(ls1));
+            }
             break;
           default:
             /* Unknown type, caught it and print its value. */
-            tt_int_op(ls1->type, OP_EQ, -1);
+            tt_int_op(link_specifier_get_ls_type(ls1), OP_EQ, -1);
         }
       }
     }
   }
 
  done:
-  tor_free(addr1);
-  tor_free(addr2);
+  ;
 }
 
diff --git a/src/test/include.am b/src/test/include.am
index ecb7689579..d8e25dea9f 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -23,7 +23,9 @@ TESTSCRIPTS = \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh \
-	src/test/test_switch_id.sh
+	src/test/test_switch_id.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh
 
 if USE_RUST
 TESTSCRIPTS += \
@@ -31,9 +33,20 @@ TESTSCRIPTS += \
 endif
 
 if USEPYTHON
-TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += \
+	src/test/test_ntor.sh \
+	src/test/test_hs_ntor.sh \
+	src/test/test_bt.sh \
+	scripts/maint/practracker/test_practracker.sh
+
+if COVERAGE_ENABLED
+# ...
+else
+# Only do this when coverage is not on, since it invokes lots of code
+# in a kind of unpredictable way.
 TESTSCRIPTS += src/test/test_rebind.sh
 endif
+endif
 
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue \
@@ -46,10 +59,8 @@ TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \
 	single-onion-v23
 # only run if we can ping6 ::1 (localhost)
-# IPv6-only v3 single onion services don't work yet, so we don't test the
-# single-onion-v23-ipv6-md flavor
 TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \
-	single-onion-ipv6-md
+	single-onion-v23-ipv6-md
 # only run if we can find a stable (or simply another) version of tor
 TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2
 
@@ -65,10 +76,11 @@ noinst_PROGRAMS+= \
 	src/test/test \
 	src/test/test-slow \
 	src/test/test-memwipe \
-	src/test/test-child \
+	src/test/test-process \
 	src/test/test_workqueue \
 	src/test/test-switch-id \
-	src/test/test-timers
+	src/test/test-timers \
+	src/test/test-rng
 endif
 
 src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
@@ -84,22 +96,28 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
 src_test_test_SOURCES =
 
 if UNITTESTS_ENABLED
+
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_test_test_SOURCES += \
 	src/test/log_test_helpers.c \
 	src/test/hs_test_helpers.c \
 	src/test/rend_test_helpers.c \
+	src/test/resolve_test_helpers.c \
+	src/test/rng_test_helpers.c \
 	src/test/test.c \
 	src/test/test_accounting.c \
 	src/test/test_addr.c \
 	src/test/test_address.c \
 	src/test/test_address_set.c \
 	src/test/test_bridges.c \
+	src/test/test_btrack.c \
 	src/test/test_buffers.c \
 	src/test/test_bwmgt.c \
 	src/test/test_cell_formats.c \
 	src/test/test_cell_queue.c \
 	src/test/test_channel.c \
 	src/test/test_channelpadding.c \
+	src/test/test_circuitpadding.c \
 	src/test/test_channeltls.c \
 	src/test/test_checkdir.c \
 	src/test/test_circuitlist.c \
@@ -109,6 +127,8 @@ src_test_test_SOURCES += \
 	src/test/test_circuitstats.c \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
+	src/test/test_confmgr.c \
+	src/test/test_confparse.c \
 	src/test/test_connection.c \
 	src/test/test_conscache.c \
 	src/test/test_consdiff.c \
@@ -118,10 +138,12 @@ src_test_test_SOURCES += \
 	src/test/test_controller_events.c \
 	src/test/test_crypto.c \
 	src/test/test_crypto_ope.c \
+	src/test/test_crypto_rng.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
 	src/test/test_dir_common.c \
 	src/test/test_dir_handle_get.c \
+	src/test/test_dispatch.c \
 	src/test/test_dos.c \
 	src/test/test_entryconn.c \
 	src/test/test_entrynodes.c \
@@ -140,12 +162,15 @@ src_test_test_SOURCES += \
 	src/test/test_handles.c \
 	src/test/test_hs_cache.c \
 	src/test/test_hs_descriptor.c \
+	src/test/test_hs_dos.c \
 	src/test/test_introduce.c \
 	src/test/test_keypin.c \
 	src/test/test_link_handshake.c \
 	src/test/test_logging.c \
 	src/test/test_mainloop.c \
 	src/test/test_microdesc.c \
+	src/test/test_namemap.c \
+	src/test/test_netinfo.c \
 	src/test/test_nodelist.c \
 	src/test/test_oom.c \
 	src/test/test_oos.c \
@@ -153,11 +178,16 @@ src_test_test_SOURCES += \
 	src/test/test_pem.c \
 	src/test/test_periodic_event.c \
 	src/test/test_policy.c \
+	src/test/test_process.c \
+	src/test/test_process_descs.c \
+	src/test/test_prob_distr.c \
 	src/test/test_procmon.c \
 	src/test/test_proto_http.c \
 	src/test/test_proto_misc.c \
 	src/test/test_protover.c \
 	src/test/test_pt.c \
+	src/test/test_pubsub_build.c \
+	src/test/test_pubsub_msg.c \
 	src/test/test_relay.c \
 	src/test/test_relaycell.c \
 	src/test/test_relaycrypt.c \
@@ -168,19 +198,23 @@ src_test_test_SOURCES += \
 	src/test/test_routerlist.c \
 	src/test/test_routerset.c \
 	src/test/test_scheduler.c \
+	src/test/test_sendme.c \
 	src/test/test_shared_random.c \
 	src/test/test_socks.c \
 	src/test/test_status.c \
 	src/test/test_storagedir.c \
 	src/test/test_threads.c \
+	src/test/test_token_bucket.c \
 	src/test/test_tortls.c \
 	src/test/test_util.c \
 	src/test/test_util_format.c \
 	src/test/test_util_process.c \
+	src/test/test_voting_flags.c \
 	src/test/test_voting_schedule.c \
 	src/test/test_x509.c \
 	src/test/test_helpers.c \
 	src/test/test_dns.c \
+	src/test/test_parsecommon.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -198,9 +232,13 @@ endif
 src_test_test_slow_SOURCES =
 if UNITTESTS_ENABLED
 src_test_test_slow_SOURCES += \
+	src/test/rng_test_helpers.c \
 	src/test/test_slow.c \
 	src/test/test_crypto_slow.c \
-	src/test/test_util_slow.c \
+	src/test/test_process_slow.c \
+	src/test/test_prob_distr.c \
+	src/test/ptr_helpers.c \
+	src/test/test_ptr_slow.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -251,6 +289,12 @@ src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS)
 src_test_test_slow_LDADD = $(src_test_test_LDADD)
 src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS)
 
+src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS)
+src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS)
+src_test_test_rng_SOURCES = src/test/test_rng.c
+src_test_test_rng_LDFLAGS =  $(src_test_test_LDFLAGS)
+src_test_test_rng_LDADD = $(src_test_test_LDADD)
+
 src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS)
 # Don't use bugtrap cflags here: memwipe tests require memory violations.
 src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS)
@@ -292,12 +336,16 @@ src_test_test_timers_LDADD = \
 	@TOR_LZMA_LIBS@
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS+= \
 	src/test/fakechans.h \
 	src/test/hs_test_helpers.h \
 	src/test/log_test_helpers.h \
 	src/test/rend_test_helpers.h \
+	src/test/resolve_test_helpers.h \
+	src/test/rng_test_helpers.h \
 	src/test/test.h \
+	src/test/ptr_helpers.h \
 	src/test/test_helpers.h \
 	src/test/test_dir_common.h \
 	src/test/test_connection.h \
@@ -369,7 +417,9 @@ EXTRA_DIST += \
 	src/test/test_workqueue_efd2.sh \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
-	src/test/test_workqueue_socketpair.sh
+	src/test/test_workqueue_socketpair.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh
 
 test-rust:
 	$(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh"
diff --git a/src/test/ope_ref.py b/src/test/ope_ref.py
index f9bd97c546..b2f7012563 100644
--- a/src/test/ope_ref.py
+++ b/src/test/ope_ref.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # Copyright 2018-2019, The Tor Project, Inc. See LICENSE for licensing info.
 
 # Reference implementation for our rudimentary OPE code, used to
diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c
new file mode 100644
index 0000000000..425733dc1b
--- /dev/null
+++ b/src/test/prob_distr_mpfr_ref.c
@@ -0,0 +1,64 @@
+/* Copyright 2012-2019, The Tor Project, Inc
+ * See LICENSE for licensing information */
+
+/** prob_distr_mpfr_ref.c
+ *
+ * Example reference file for GNU MPFR vectors tested in test_prob_distr.c .
+ * Code by Riastradh.
+ */
+
+#include <complex.h>
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+
+/* Must come after <stdio.h> so we get mpfr_printf.  */
+#include <mpfr.h>
+
+/*  gcc -o mpfr prob_distr_mpfr_ref.c -lmpfr -lm */
+
+/* Computes logit(p) for p = .49999 */
+int
+main(void)
+{
+  mpfr_t p, q, r;
+  mpfr_init(p);
+  mpfr_set_prec(p, 200);
+  mpfr_init(q);
+  mpfr_set_prec(q, 200);
+  mpfr_init(r);
+  mpfr_set_prec(r, 200);
+  mpfr_set_d(p, .49999, MPFR_RNDN);
+  mpfr_set_d(q, 1, MPFR_RNDN);
+  /* r := q - p = 1 - p */
+  mpfr_sub(r, q, p, MPFR_RNDN);
+  /* q := p/r = p/(1 - p) */
+  mpfr_div(q, p, r, MPFR_RNDN);
+  /* r := log(q) = log(p/(1 - p)) */
+  mpfr_log(r, q, MPFR_RNDN);
+  mpfr_printf("mpfr 200-bit\t%.128Rg\n", r);
+
+  /*
+   * Print a double approximation to logit three different ways.  All
+   * three agree bit for bit on the libms I tried, with the nextafter
+   * adjustment (which is well within the 10 eps relative error bound
+   * advertised).  Apparently I must have used the Goldberg expression
+   * for what I wrote down in the test case.
+   */
+  printf("mpfr 53-bit\t%.17g\n", nextafter(mpfr_get_d(r, MPFR_RNDN), 0), 0);
+  volatile double p0 = .49999;
+  printf("log1p\t\t%.17g\n", nextafter(-log1p((1 - 2*p0)/p0), 0));
+  volatile double x = (1 - 2*p0)/p0;
+  volatile double xp1 = x + 1;
+  printf("Goldberg\t%.17g\n", -x*log(xp1)/(xp1 - 1));
+
+  /*
+   * Print a bad approximation, using the naive expression, to see a
+   * lot of wrong digits, far beyond the 10 eps relative error attained
+   * by -log1p((1 - 2*p)/p).
+   */
+  printf("naive\t\t%.17g\n", log(p0/(1 - p0)));
+
+  fflush(stdout);
+  return ferror(stdout);
+}
diff --git a/src/test/ptr_helpers.c b/src/test/ptr_helpers.c
new file mode 100644
index 0000000000..a55ab437fa
--- /dev/null
+++ b/src/test/ptr_helpers.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/ptr_helpers.h"
+
+/**
+ * Cast <b> (inptr_t value) to a void pointer.
+ */
+void *
+cast_intptr_to_voidstar(intptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to inptr_t value.
+ */
+intptr_t
+cast_voidstar_to_intptr(void *x)
+{
+  intptr_t r = (intptr_t)x;
+
+  return r;
+}
+
+/**
+ * Cast x (uinptr_t value) to void pointer.
+ */
+void *
+cast_uintptr_to_voidstar(uintptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to uinptr_t value.
+ */
+uintptr_t
+cast_voidstar_to_uintptr(void *x)
+{
+  uintptr_t r = (uintptr_t)x;
+
+  return r;
+}
diff --git a/src/test/ptr_helpers.h b/src/test/ptr_helpers.h
new file mode 100644
index 0000000000..7349bddd51
--- /dev/null
+++ b/src/test/ptr_helpers.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PTR_HELPERS_H
+#define TOR_PTR_HELPERS_H
+
+#include <stdint.h>
+
+void *
+cast_intptr_to_voidstar(intptr_t x);
+
+intptr_t
+cast_voidstar_to_intptr(void *x);
+
+void *
+cast_uintptr_to_voidstar(uintptr_t x);
+
+uintptr_t
+cast_voidstar_to_uintptr(void *x);
+
+#endif /* !defined(TOR_PTR_HELPERS_H) */
diff --git a/src/test/resolve_test_helpers.c b/src/test/resolve_test_helpers.c
new file mode 100644
index 0000000000..73ea730149
--- /dev/null
+++ b/src/test/resolve_test_helpers.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.c
+ * @brief Helper functions for mocking libc's blocking hostname lookup
+ *   facilities.
+ **/
+
+#define RESOLVE_PRIVATE
+#include "orconfig.h"
+#include "test/resolve_test_helpers.h"
+#include "lib/net/address.h"
+#include "lib/net/resolve.h"
+#include "test/test.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * Mock replacement for our getaddrinfo/gethostbyname wrapper.
+ **/
+static int
+replacement_host_lookup(const char *name, uint16_t family, tor_addr_t *addr)
+{
+  static const struct lookup_table_ent {
+    const char *name;
+    const char *ipv4;
+    const char *ipv6;
+  } entries[] = {
+    { "localhost", "127.0.0.1", "::1" },
+    { "torproject.org", "198.51.100.6", "2001:DB8::700" },
+    { NULL, NULL, NULL },
+  };
+
+  int r = -1;
+
+  for (unsigned i = 0; entries[i].name != NULL; ++i) {
+    if (!strcasecmp(name, entries[i].name)) {
+      if (family == AF_INET6) {
+        int s = tor_addr_parse(addr, entries[i].ipv6);
+        tt_int_op(s, OP_EQ, AF_INET6);
+      } else {
+        int s = tor_addr_parse(addr, entries[i].ipv4);
+        tt_int_op(s, OP_EQ, AF_INET);
+      }
+      r = 0;
+      break;
+    }
+  }
+
+  log_debug(LD_GENERAL, "resolve(%s,%d) => %s",
+            name, family, r == 0 ? fmt_addr(addr) : "-1");
+
+  return r;
+ done:
+  return -1;
+}
+
+/**
+ * Set up a mock replacement for our wrapper on libc's resolver code.
+ *
+ * According to our replacement, only "localhost" and "torproject.org"
+ * are real addresses; everything else doesn't exist.
+ *
+ * Use this function to avoid using the DNS resolver during unit tests;
+ * call unmock_hostname_resolver() when you're done.
+ **/
+void
+mock_hostname_resolver(void)
+{
+  MOCK(tor_addr_lookup_host_impl, replacement_host_lookup);
+}
+
+/**
+ * Unmock our wrappers for libc's blocking hostname resolver code.
+ **/
+void
+unmock_hostname_resolver(void)
+{
+  UNMOCK(tor_addr_lookup_host_impl);
+}
diff --git a/src/test/resolve_test_helpers.h b/src/test/resolve_test_helpers.h
new file mode 100644
index 0000000000..e7d2e29373
--- /dev/null
+++ b/src/test/resolve_test_helpers.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.h
+ * @brief Header for test/resolve_test_helpers.c
+ **/
+
+#ifndef TOR_TEST_RESOLVE_TEST_HELPERS_H
+#define TOR_TEST_RESOLVE_TEST_HELPERS_H
+
+void mock_hostname_resolver(void);
+void unmock_hostname_resolver(void);
+
+#endif /* !defined(TOR_TEST_RESOLVE_TEST_HELPERS_H) */
diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c
new file mode 100644
index 0000000000..7024fb5793
--- /dev/null
+++ b/src/test/rng_test_helpers.c
@@ -0,0 +1,259 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rng_test_helpers.c
+ * \brief Helpers for overriding PRNGs during unit tests.
+ *
+ * We define two PRNG overrides: a "reproducible PRNG" where the seed is
+ * chosen randomly but the stream can be replayed later on in case a bug is
+ * found, and a "deterministic PRNG" where the seed is fixed in the unit
+ * tests.
+ *
+ * Obviously, this code is testing-only.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "ext/tinytest.h"
+
+#include "test/rng_test_helpers.h"
+
+#ifndef TOR_UNIT_TESTS
+#error "No. Never link this code into Tor proper."
+#endif
+
+/**
+ * True iff the RNG is currently replaced.  Prevents double-replacement.
+ **/
+static bool rng_is_replaced = false;
+
+/**
+ * Mutex to protect deterministic prng.
+ *
+ * Note that if you actually _use_ the prng from two threads at the same time,
+ * the results will probably be nondeterministic anyway.
+ */
+static tor_mutex_t *rng_mutex = NULL;
+
+/**
+ * Cached old value for the thread prng.
+ **/
+static crypto_fast_rng_t *stored_fast_rng = NULL;
+
+/** replacement for crypto_strongest_rand that delegates to crypto_rand. */
+static void
+mock_crypto_strongest_rand(uint8_t *out, size_t len)
+{
+  crypto_rand((char *)out, len);
+}
+
+/* This is the seed of the deterministic randomness. */
+static uint8_t rng_seed[16];
+static crypto_xof_t *rng_xof = NULL;
+
+/**
+ * Print the seed for our PRNG to stdout.  We use this when we're failed
+ * test that had a reproducible RNG set.
+ **/
+void
+testing_dump_reproducible_rng_seed(void)
+{
+  printf("\n"
+         "Seed: %s\n",
+         hex_str((const char*)rng_seed, sizeof(rng_seed)));
+}
+
+/** Produce deterministic randomness for the stochastic tests using the global
+ * rng_xof output.
+ *
+ * This function produces deterministic data over multiple calls iff it's
+ * called in the same call order with the same 'n' parameter.
+ * If not, outputs will deviate. */
+static void
+crypto_rand_deterministic(char *out, size_t n)
+{
+  tor_assert(rng_xof);
+  tor_mutex_acquire(rng_mutex);
+  crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n);
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Implementation helper: override our crypto_rand() PRNG with a given seed of
+ * length <b>seed_len</b>.  Overlong seeds are truncated; short ones are
+ * padded.
+ **/
+static void
+enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len)
+{
+  tor_assert(!rng_is_replaced);
+  tor_assert(crypto_rand == crypto_rand__real);
+
+  memset(rng_seed, 0, sizeof(rng_seed));
+  memcpy(rng_seed, seed, MIN(seed_len, sizeof(rng_seed)));
+
+  rng_mutex = tor_mutex_new();
+
+  crypto_xof_free(rng_xof);
+  rng_xof = crypto_xof_new();
+  crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed));
+  MOCK(crypto_rand, crypto_rand_deterministic);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+
+  uint8_t fast_rng_seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(fast_rng_seed, 0xff, sizeof(fast_rng_seed));
+  memcpy(fast_rng_seed, rng_seed, MIN(sizeof(rng_seed),
+                                      sizeof(fast_rng_seed)));
+  crypto_fast_rng_t *fast_rng = crypto_fast_rng_new_from_seed(fast_rng_seed);
+  crypto_fast_rng_disable_reseed(fast_rng);
+  stored_fast_rng = crypto_replace_thread_fast_rng(fast_rng);
+
+  rng_is_replaced = true;
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a randomly chosen seed.  In the event of an
+ * error, you can log the seed later on with
+ * testing_dump_reproducible_rng_seed.
+ **/
+void
+testing_enable_reproducible_rng(void)
+{
+  const char *provided_seed = getenv("TOR_TEST_RNG_SEED");
+  if (provided_seed) {
+    size_t hexlen = strlen(provided_seed);
+    size_t seedlen = hexlen / 2;
+    uint8_t *seed = tor_malloc(hexlen / 2);
+    if (base16_decode((char*)seed, seedlen, provided_seed, hexlen) < 0) {
+      puts("Cannot decode value in TOR_TEST_RNG_SEED");
+      exit(1);
+    }
+    enable_deterministic_rng_impl(seed, seedlen);
+    tor_free(seed);
+  } else {
+    uint8_t seed[16];
+    crypto_rand((char*)seed, sizeof(seed));
+    enable_deterministic_rng_impl(seed, sizeof(seed));
+  }
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a fixed seed.  This variant is mainly useful
+ * for cases when we don't want coverage to change between runs.
+ *
+ * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of
+ * this "rng".  If you need a specific output, use
+ * testing_enable_prefilled_rng() instead.
+ **/
+void
+testing_enable_deterministic_rng(void)
+{
+  static const uint8_t quotation[] =
+    "What will it be? A tree? A weed? "
+    "Each one is started from a seed."; // -- Mary Ann Hoberman
+  enable_deterministic_rng_impl(quotation, sizeof(quotation));
+}
+
+static uint8_t *prefilled_rng_buffer = NULL;
+static size_t prefilled_rng_buflen;
+static size_t prefilled_rng_idx;
+
+/**
+ * crypto_rand() replacement that returns canned data.
+ **/
+static void
+crypto_rand_prefilled(char *out, size_t n)
+{
+  tor_mutex_acquire(rng_mutex);
+  while (n) {
+    size_t n_to_copy = MIN(prefilled_rng_buflen - prefilled_rng_idx, n);
+    memcpy(out, prefilled_rng_buffer + prefilled_rng_idx, n_to_copy);
+    out += n_to_copy;
+    n -= n_to_copy;
+    prefilled_rng_idx += n_to_copy;
+
+    if (prefilled_rng_idx == prefilled_rng_buflen) {
+      prefilled_rng_idx = 0;
+    }
+  }
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant
+ * that yields output from a buffer.  If it reaches the end of the buffer, it
+ * starts over.
+ *
+ * Note: the get_thread_fast_rng() prng is not replaced by this; we'll need
+ * more code to support that.
+ **/
+void
+testing_enable_prefilled_rng(const void *buffer, size_t buflen)
+{
+  tor_assert(buflen > 0);
+  tor_assert(!rng_mutex);
+  rng_mutex = tor_mutex_new();
+
+  tor_mutex_acquire(rng_mutex);
+
+  prefilled_rng_buffer = tor_memdup(buffer, buflen);
+  prefilled_rng_buflen = buflen;
+  prefilled_rng_idx = 0;
+
+  tor_mutex_release(rng_mutex);
+
+  MOCK(crypto_rand, crypto_rand_prefilled);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+}
+
+/**
+ * Reset the position in the prefilled RNG buffer to the start.
+ */
+void
+testing_prefilled_rng_reset(void)
+{
+  tor_mutex_acquire(rng_mutex);
+  prefilled_rng_idx = 0;
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Undo the overrides for our PRNG.  To be used at the end of testing.
+ *
+ * Note that this function should be safe to call even if the rng has not
+ * yet been replaced.
+ **/
+void
+testing_disable_rng_override(void)
+{
+  crypto_xof_free(rng_xof);
+  tor_free(prefilled_rng_buffer);
+  UNMOCK(crypto_rand);
+  UNMOCK(crypto_strongest_rand_);
+  tor_mutex_free(rng_mutex);
+
+  crypto_fast_rng_t *rng = crypto_replace_thread_fast_rng(stored_fast_rng);
+  crypto_fast_rng_free(rng);
+
+  rng_is_replaced = false;
+}
+
+/**
+ * As testing_disable_rng_override(), but dump the seed if the current
+ * test has failed.
+ */
+void
+testing_disable_reproducible_rng(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    testing_dump_reproducible_rng_seed();
+  }
+  testing_disable_rng_override();
+}
diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h
new file mode 100644
index 0000000000..d7925148ae
--- /dev/null
+++ b/src/test/rng_test_helpers.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_RNG_TEST_HELPERS_H
+#define TOR_RNG_TEST_HELPERS_H
+
+#include "core/or/or.h"
+
+void testing_enable_deterministic_rng(void);
+void testing_enable_reproducible_rng(void);
+void testing_enable_prefilled_rng(const void *buffer, size_t buflen);
+
+void testing_prefilled_rng_reset(void);
+
+void testing_disable_rng_override(void);
+
+void testing_disable_reproducible_rng(void);
+#define testing_disable_deterministic_rng() \
+  testing_disable_rng_override()
+#define testing_disable_prefilled_rng() \
+  testing_disable_rng_override()
+
+void testing_dump_reproducible_rng_seed(void);
+
+#endif /* !defined(TOR_RNG_TEST_HELPERS_H) */
diff --git a/src/test/test-child.c b/src/test/test-child.c
deleted file mode 100644
index 11a1695cad..0000000000
--- a/src/test/test-child.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#include <stdio.h>
-#ifdef _WIN32
-#define WINDOWS_LEAN_AND_MEAN
-#include <windows.h>
-#else
-#include <unistd.h>
-#endif /* defined(_WIN32) */
-#include <string.h>
-
-#ifdef _WIN32
-#define SLEEP(sec) Sleep((sec)*1000)
-#else
-#define SLEEP(sec) sleep(sec)
-#endif
-
-/** Trivial test program which prints out its command line arguments so we can
- * check if tor_spawn_background() works */
-int
-main(int argc, char **argv)
-{
-  int i;
-  int delay = 1;
-  int fast = 0;
-
-  if (argc > 1)  {
-    if (!strcmp(argv[1], "--hang")) {
-      delay = 60;
-    } else if (!strcmp(argv[1], "--fast")) {
-      fast = 1;
-      delay = 0;
-    }
-  }
-
-  fprintf(stdout, "OUT\n");
-  fprintf(stderr, "ERR\n");
-  for (i = 1; i < argc; i++)
-    fprintf(stdout, "%s\n", argv[i]);
-  if (!fast)
-    fprintf(stdout, "SLEEPING\n");
-  /* We need to flush stdout so that test_util_spawn_background_partial_read()
-     succeed. Otherwise ReadFile() will get the entire output in one */
-  // XXX: Can we make stdio flush on newline?
-  fflush(stdout);
-  if (!fast)
-    SLEEP(1);
-  fprintf(stdout, "DONE\n");
-  fflush(stdout);
-  if (fast)
-    return 0;
-
-  while (--delay) {
-    SLEEP(1);
-  }
-
-  return 0;
-}
-
diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c
index 43754ed1c2..3f952e484f 100644
--- a/src/test/test-memwipe.c
+++ b/src/test/test-memwipe.c
@@ -49,7 +49,7 @@ const char *s = NULL;
  * us do bad things, such as access freed buffers, without crashing. */
 extern const char *malloc_options;
 const char *malloc_options = "sufjj";
-#endif
+#endif /* defined(OpenBSD) */
 
 static unsigned
 fill_a_buffer_memset(void)
diff --git a/src/test/test-network.sh b/src/test/test-network.sh
index b7a9f1b3c0..5ef995f1a4 100755
--- a/src/test/test-network.sh
+++ b/src/test/test-network.sh
@@ -5,7 +5,7 @@
 # If we already know CHUTNEY_PATH, don't bother with argument parsing
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     # we can't produce any output, because we might be --quiet
     # this preserves arguments with spaces correctly
     exec "$TEST_NETWORK" "$@"
@@ -16,34 +16,16 @@ fi
 # Do we output anything at all?
 ECHO="${ECHO:-echo}"
 # Output is prefixed with the name of the script
-myname=$(basename $0)
-
-# Save the arguments before we destroy them
-# This might not preserve arguments with spaces in them
-ORIGINAL_ARGS="$@"
+myname=$(basename "$0")
 
 # We need to find CHUTNEY_PATH, so that we can call the version of this script
 # in chutney/tools with the same arguments. We also need to respect --quiet.
-until [ -z "$1" ]
-do
-  case "$1" in
-    --chutney-path)
-      CHUTNEY_PATH="$2"
-      shift
-    ;;
-    --tor-path)
-      TOR_DIR="$2"
-      shift
-    ;;
-    --quiet)
-      ECHO=true
-    ;;
-    *)
-      # maybe chutney's test-network.sh can handle it
-    ;;
-  esac
-  shift
-done
+CHUTNEY_PATH=$(echo "$@" | awk -F '--chutney-path ' '{sub(" .*","",$2); print $2}')
+TOR_DIR=$(echo "$@" | awk -F '--tor-dir ' '{sub(" .*","",$2); print $2}')
+
+if echo "$@" | grep -e "--quiet" > /dev/null; then
+  ECHO=true
+fi
 
 # optional: $TOR_DIR is the tor build directory
 # it's used to find the location of tor binaries
@@ -52,12 +34,12 @@ done
 #  - if $PWD looks like a tor build directory, set it to $PWD, or
 #  - unset $TOR_DIR, and let chutney fall back to finding tor binaries in $PATH
 if [ ! -d "$TOR_DIR" ]; then
-    if [ -d "$BUILDDIR/src/core/or" -a -d "$BUILDDIR/src/tools" ]; then
+    if [ -d "$BUILDDIR/src/core/or" ] && [ -d "$BUILDDIR/src/tools" ]; then
         # Choose the build directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$BUILDDIR"
         TOR_DIR="$BUILDDIR"
-    elif [ -d "$PWD/src/core/or" -a -d "$PWD/src/tools" ]; then
+    elif [ -d "$PWD/src/core/or" ] && [ -d "$PWD/src/tools" ]; then
         # Guess the tor directory is the current directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$PWD"
@@ -73,12 +55,12 @@ fi
 #  - if $PWD looks like a chutney directory, set it to $PWD, or
 #  - set it based on $TOR_DIR, expecting chutney to be next to tor, or
 #  - fail and tell the user how to clone the chutney repository
-if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then
+if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then
     if [ -x "$PWD/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$PWD"
         CHUTNEY_PATH="$PWD"
-    elif [ -d "$TOR_DIR" -a -d "$TOR_DIR/../chutney" -a \
-           -x "$TOR_DIR/../chutney/chutney" ]; then
+    elif [ -d "$TOR_DIR" ] && [ -d "$TOR_DIR/../chutney" ] && \
+          [ -x "$TOR_DIR/../chutney/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney"
         CHUTNEY_PATH="$TOR_DIR/../chutney"
     else
@@ -94,12 +76,12 @@ fi
 
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     $ECHO "$myname: Calling newer chutney script $TEST_NETWORK"
     # this may fail if some arguments have spaces in them
     # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces
     # will be handled correctly
-    exec "$TEST_NETWORK" $ORIGINAL_ARGS
+    exec "$TEST_NETWORK" "$@"
 else
     $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH."
     $ECHO "$myname: Please update your chutney using 'git pull'."
diff --git a/src/test/test-process.c b/src/test/test-process.c
new file mode 100644
index 0000000000..eb28ad90e9
--- /dev/null
+++ b/src/test/test-process.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include <stdio.h>
+#ifdef _WIN32
+#define WINDOWS_LEAN_AND_MEAN
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif /* defined(_WIN32) */
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define SLEEP(sec) Sleep((sec)*1000)
+#else
+#define SLEEP(sec) sleep(sec)
+#endif
+
+/* Trivial test program to test process_t. */
+int
+main(int argc, char **argv)
+{
+  /* Does our process get the right arguments? */
+  for (int i = 0; i < argc; ++i) {
+    fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]);
+    fflush(stdout);
+  }
+
+  /* Make sure our process got our environment variable. */
+  fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n",
+          getenv("TOR_TEST_ENV"));
+  fflush(stdout);
+
+  /* Test line handling on stdout and stderr. */
+  fprintf(stdout, "Output on stdout\nThis is a new line\n");
+  fflush(stdout);
+
+  fprintf(stderr, "Output on stderr\nThis is a new line\n");
+  fflush(stderr);
+
+  fprintf(stdout, "Partial line on stdout ...");
+  fflush(stdout);
+
+  fprintf(stderr, "Partial line on stderr ...");
+  fflush(stderr);
+
+  SLEEP(2);
+
+  fprintf(stdout, "end of partial line on stdout\n");
+  fflush(stdout);
+  fprintf(stderr, "end of partial line on stderr\n");
+  fflush(stderr);
+
+  /* Echo input from stdin. */
+  char buffer[1024];
+
+  int count = 0;
+
+  while (fgets(buffer, sizeof(buffer), stdin)) {
+    /* Strip the newline. */
+    size_t size = strlen(buffer);
+
+    if (size >= 1 && buffer[size - 1] == '\n') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    if (size >= 1 && buffer[size - 1] == '\r') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    fprintf(stdout, "Read line from stdin: '%s'\n", buffer);
+    fflush(stdout);
+
+    if (++count == 3)
+      break;
+  }
+
+  fprintf(stdout, "We are done for here, thank you!\n");
+
+  return 0;
+}
diff --git a/src/test/test.c b/src/test/test.c
index 58b468775c..6dbec26fa8 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -12,6 +12,7 @@
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "app/config/or_state_st.h"
+#include "test/rng_test_helpers.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -37,7 +38,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "lib/compress/compress.h"
@@ -283,7 +284,7 @@ test_fast_handshake(void *arg)
   /* First, test an entire handshake. */
   memset(client_handshake, 0, sizeof(client_handshake));
   tt_int_op(0, OP_EQ, fast_onionskin_create(&state, client_handshake));
-  tt_assert(! tor_mem_is_zero((char*)client_handshake,
+  tt_assert(! fast_mem_is_zero((char*)client_handshake,
                               sizeof(client_handshake)));
 
   tt_int_op(0, OP_EQ,
@@ -354,18 +355,6 @@ test_onion_queues(void *arg)
   tor_free(onionskin);
 }
 
-static crypto_cipher_t *crypto_rand_aes_cipher = NULL;
-
-// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR
-// cipher in <b>crypto_rand_aes_cipher</b>.
-static void
-crypto_rand_deterministic_aes(char *out, size_t n)
-{
-  tor_assert(crypto_rand_aes_cipher);
-  memset(out, 0, n);
-  crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n);
-}
-
 static void
 test_circuit_timeout(void *arg)
 {
@@ -397,8 +386,7 @@ test_circuit_timeout(void *arg)
 
   // Use a deterministic RNG here, or else we'll get nondeterministic
   // coverage in some of the circuitstats functions.
-  MOCK(crypto_rand, crypto_rand_deterministic_aes);
-  crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover");
+  testing_enable_deterministic_rng();
 
   circuitbuild_running_unit_tests();
 #define timeout0 (build_time_t)(30*1000.0)
@@ -534,8 +522,8 @@ test_circuit_timeout(void *arg)
   circuit_build_times_free_timeouts(&final);
   or_state_free(state);
   teardown_periodic_events();
-  UNMOCK(crypto_rand);
-  crypto_cipher_free(crypto_rand_aes_cipher);
+
+  testing_disable_deterministic_rng();
 }
 
 /** Test encoding and parsing of rendezvous service descriptors. */
@@ -845,18 +833,23 @@ struct testgroup_t testgroups[] = {
   { "channeltls/", channeltls_tests },
   { "checkdir/", checkdir_tests },
   { "circuitbuild/", circuitbuild_tests },
+  { "circuitpadding/", circuitpadding_tests },
   { "circuitlist/", circuitlist_tests },
   { "circuitmux/", circuitmux_tests },
-  { "circuituse/", circuituse_tests },
   { "circuitstats/", circuitstats_tests },
+  { "circuituse/", circuituse_tests },
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
+  { "config/mgr/", confmgr_tests },
+  { "config/parse/", confparse_tests },
   { "connection/", connection_tests },
   { "conscache/", conscache_tests },
   { "consdiff/", consdiff_tests },
   { "consdiffmgr/", consdiffmgr_tests },
   { "container/", container_tests },
+  { "container/namemap/", namemap_tests },
   { "control/", controller_tests },
+  { "control/btrack/", btrack_tests },
   { "control/event/", controller_event_tests },
   { "crypto/", crypto_tests },
   { "crypto/ope/", crypto_ope_tests },
@@ -864,42 +857,54 @@ struct testgroup_t testgroups[] = {
   { "crypto/openssl/", crypto_openssl_tests },
 #endif
   { "crypto/pem/", pem_tests },
+  { "crypto/rng/", crypto_rng_tests },
   { "dir/", dir_tests },
-  { "dir_handle_get/", dir_handle_get_tests },
+  { "dir/auth/process_descs/", process_descs_tests },
   { "dir/md/", microdesc_tests },
-  { "dir/voting-schedule/", voting_schedule_tests },
+  { "dir/voting/flags/", voting_flags_tests },
+  { "dir/voting/schedule/", voting_schedule_tests },
+  { "dir_handle_get/", dir_handle_get_tests },
+  { "dispatch/", dispatch_tests, },
+  { "dns/", dns_tests },
   { "dos/", dos_tests },
   { "entryconn/", entryconn_tests },
   { "entrynodes/", entrynodes_tests },
-  { "guardfraction/", guardfraction_tests },
   { "extorport/", extorport_tests },
   { "geoip/", geoip_tests },
-  { "legacy_hs/", hs_tests },
+  { "guardfraction/", guardfraction_tests },
   { "hs_cache/", hs_cache },
   { "hs_cell/", hs_cell_tests },
+  { "hs_client/", hs_client_tests },
   { "hs_common/", hs_common_tests },
   { "hs_config/", hs_config_tests },
   { "hs_control/", hs_control_tests },
   { "hs_descriptor/", hs_descriptor },
+  { "hs_dos/", hs_dos_tests },
+  { "hs_intropoint/", hs_intropoint_tests },
   { "hs_ntor/", hs_ntor_tests },
   { "hs_service/", hs_service_tests },
-  { "hs_client/", hs_client_tests },
-  { "hs_intropoint/", hs_intropoint_tests },
   { "introduce/", introduce_tests },
   { "keypin/", keypin_tests },
+  { "legacy_hs/", hs_tests },
   { "link-handshake/", link_handshake_tests },
   { "mainloop/", mainloop_tests },
+  { "netinfo/", netinfo_tests },
   { "nodelist/", nodelist_tests },
   { "oom/", oom_tests },
   { "oos/", oos_tests },
   { "options/", options_tests },
+  { "parsecommon/", parsecommon_tests },
   { "periodic-event/" , periodic_event_tests },
   { "policy/" , policy_tests },
+  { "prob_distr/", prob_distr_tests },
   { "procmon/", procmon_tests },
+  { "process/", process_tests },
   { "proto/http/", proto_http_tests },
   { "proto/misc/", proto_misc_tests },
   { "protover/", protover_tests },
   { "pt/", pt_tests },
+  { "pubsub/build/", pubsub_build_tests },
+  { "pubsub/msg/", pubsub_msg_tests },
   { "relay/" , relay_tests },
   { "relaycell/", relaycell_tests },
   { "relaycrypt/", relaycrypt_tests },
@@ -910,10 +915,12 @@ struct testgroup_t testgroups[] = {
   { "routerlist/", routerlist_tests },
   { "routerset/" , routerset_tests },
   { "scheduler/", scheduler_tests },
-  { "socks/", socks_tests },
+  { "sendme/", sendme_tests },
   { "shared-random/", sr_tests },
+  { "socks/", socks_tests },
   { "status/" , status_tests },
   { "storagedir/", storagedir_tests },
+  { "token_bucket/", token_bucket_tests },
   { "tortls/", tortls_tests },
 #ifndef ENABLE_NSS
   { "tortls/openssl/", tortls_openssl_tests },
@@ -921,10 +928,9 @@ struct testgroup_t testgroups[] = {
   { "tortls/x509/", x509_tests },
   { "util/", util_tests },
   { "util/format/", util_format_tests },
+  { "util/handle/", handle_tests },
   { "util/logging/", logging_tests },
   { "util/process/", util_process_tests },
   { "util/thread/", thread_tests },
-  { "util/handle/", handle_tests },
-  { "dns/", dns_tests },
   END_OF_GROUPS
 };
diff --git a/src/test/test.h b/src/test/test.h
index aacc9dba87..76c4c0ec75 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -177,71 +177,89 @@ extern const struct testcase_setup_t ed25519_test_setup;
 
 extern struct testcase_t accounting_tests[];
 extern struct testcase_t addr_tests[];
-extern struct testcase_t address_tests[];
 extern struct testcase_t address_set_tests[];
+extern struct testcase_t address_tests[];
 extern struct testcase_t bridges_tests[];
-extern struct testcase_t bwmgt_tests[];
+extern struct testcase_t btrack_tests[];
 extern struct testcase_t buffer_tests[];
+extern struct testcase_t bwmgt_tests[];
 extern struct testcase_t cell_format_tests[];
 extern struct testcase_t cell_queue_tests[];
 extern struct testcase_t channel_tests[];
 extern struct testcase_t channelpadding_tests[];
+extern struct testcase_t circuitpadding_tests[];
 extern struct testcase_t channeltls_tests[];
 extern struct testcase_t checkdir_tests[];
 extern struct testcase_t circuitbuild_tests[];
 extern struct testcase_t circuitlist_tests[];
 extern struct testcase_t circuitmux_tests[];
-extern struct testcase_t circuituse_tests[];
 extern struct testcase_t circuitstats_tests[];
+extern struct testcase_t circuituse_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
+extern struct testcase_t confmgr_tests[];
+extern struct testcase_t confparse_tests[];
 extern struct testcase_t connection_tests[];
 extern struct testcase_t conscache_tests[];
 extern struct testcase_t consdiff_tests[];
 extern struct testcase_t consdiffmgr_tests[];
 extern struct testcase_t container_tests[];
-extern struct testcase_t controller_tests[];
 extern struct testcase_t controller_event_tests[];
-extern struct testcase_t crypto_tests[];
+extern struct testcase_t controller_tests[];
 extern struct testcase_t crypto_ope_tests[];
 extern struct testcase_t crypto_openssl_tests[];
-extern struct testcase_t dir_tests[];
+extern struct testcase_t crypto_rng_tests[];
+extern struct testcase_t crypto_tests[];
 extern struct testcase_t dir_handle_get_tests[];
+extern struct testcase_t dir_tests[];
+extern struct testcase_t dispatch_tests[];
+extern struct testcase_t dns_tests[];
 extern struct testcase_t dos_tests[];
 extern struct testcase_t entryconn_tests[];
 extern struct testcase_t entrynodes_tests[];
-extern struct testcase_t guardfraction_tests[];
 extern struct testcase_t extorport_tests[];
 extern struct testcase_t geoip_tests[];
-extern struct testcase_t hs_tests[];
+extern struct testcase_t guardfraction_tests[];
+extern struct testcase_t handle_tests[];
 extern struct testcase_t hs_cache[];
 extern struct testcase_t hs_cell_tests[];
+extern struct testcase_t hs_client_tests[];
 extern struct testcase_t hs_common_tests[];
 extern struct testcase_t hs_config_tests[];
 extern struct testcase_t hs_control_tests[];
 extern struct testcase_t hs_descriptor[];
+extern struct testcase_t hs_dos_tests[];
+extern struct testcase_t hs_intropoint_tests[];
 extern struct testcase_t hs_ntor_tests[];
 extern struct testcase_t hs_service_tests[];
-extern struct testcase_t hs_client_tests[];
-extern struct testcase_t hs_intropoint_tests[];
+extern struct testcase_t hs_tests[];
 extern struct testcase_t introduce_tests[];
 extern struct testcase_t keypin_tests[];
 extern struct testcase_t link_handshake_tests[];
 extern struct testcase_t logging_tests[];
 extern struct testcase_t mainloop_tests[];
 extern struct testcase_t microdesc_tests[];
+extern struct testcase_t namemap_tests[];
+extern struct testcase_t netinfo_tests[];
 extern struct testcase_t nodelist_tests[];
 extern struct testcase_t oom_tests[];
 extern struct testcase_t oos_tests[];
 extern struct testcase_t options_tests[];
+extern struct testcase_t parsecommon_tests[];
 extern struct testcase_t pem_tests[];
 extern struct testcase_t periodic_event_tests[];
 extern struct testcase_t policy_tests[];
+extern struct testcase_t prob_distr_tests[];
+extern struct testcase_t slow_stochastic_prob_distr_tests[];
 extern struct testcase_t procmon_tests[];
+extern struct testcase_t process_tests[];
+extern struct testcase_t process_descs_tests[];
 extern struct testcase_t proto_http_tests[];
 extern struct testcase_t proto_misc_tests[];
 extern struct testcase_t protover_tests[];
 extern struct testcase_t pt_tests[];
+extern struct testcase_t pubsub_build_tests[];
+extern struct testcase_t pubsub_msg_tests[];
 extern struct testcase_t relay_tests[];
 extern struct testcase_t relaycell_tests[];
 extern struct testcase_t relaycrypt_tests[];
@@ -252,23 +270,25 @@ extern struct testcase_t routerkeys_tests[];
 extern struct testcase_t routerlist_tests[];
 extern struct testcase_t routerset_tests[];
 extern struct testcase_t scheduler_tests[];
-extern struct testcase_t storagedir_tests[];
+extern struct testcase_t sendme_tests[];
 extern struct testcase_t socks_tests[];
+extern struct testcase_t sr_tests[];
 extern struct testcase_t status_tests[];
+extern struct testcase_t storagedir_tests[];
 extern struct testcase_t thread_tests[];
-extern struct testcase_t tortls_tests[];
+extern struct testcase_t token_bucket_tests[];
 extern struct testcase_t tortls_openssl_tests[];
-extern struct testcase_t util_tests[];
+extern struct testcase_t tortls_tests[];
 extern struct testcase_t util_format_tests[];
 extern struct testcase_t util_process_tests[];
+extern struct testcase_t util_tests[];
+extern struct testcase_t voting_flags_tests[];
 extern struct testcase_t voting_schedule_tests[];
-extern struct testcase_t dns_tests[];
-extern struct testcase_t handle_tests[];
-extern struct testcase_t sr_tests[];
 extern struct testcase_t x509_tests[];
 
 extern struct testcase_t slow_crypto_tests[];
-extern struct testcase_t slow_util_tests[];
+extern struct testcase_t slow_process_tests[];
+extern struct testcase_t slow_ptr_tests[];
 
 extern struct testgroup_t testgroups[];
 
diff --git a/src/test/test_addr.c b/src/test/test_addr.c
index 8868edce25..c89c6e78d4 100644
--- a/src/test/test_addr.c
+++ b/src/test/test_addr.c
@@ -11,6 +11,8 @@
 #include "feature/client/addressmap.h"
 #include "test/log_test_helpers.h"
 #include "lib/net/resolve.h"
+#include "test/rng_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #ifdef HAVE_SYS_UN_H
 #include <sys/un.h>
@@ -239,7 +241,7 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(0,OP_EQ, tor_addr_lookup("9000::5", AF_UNSPEC, &t1));
   tt_int_op(AF_INET6,OP_EQ, tor_addr_family(&t1));
   tt_int_op(0x90,OP_EQ, tor_addr_to_in6_addr8(&t1)[0]);
-  tt_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
+  tt_assert(fast_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
   tt_int_op(0x05,OP_EQ, tor_addr_to_in6_addr8(&t1)[15]);
 
   /* === Test pton: valid af_inet6 */
@@ -696,7 +698,7 @@ test_addr_ip6_helpers(void *arg)
                               &t1,&mask,&port1,&port2);
   tt_int_op(r,OP_EQ,AF_INET6);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET6);
-  tt_assert(tor_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
+  tt_assert(fast_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
   tt_int_op(mask,OP_EQ,0);
   tt_int_op(port1,OP_EQ,1);
   tt_int_op(port2,OP_EQ,65535);
@@ -723,104 +725,559 @@ test_addr_ip6_helpers(void *arg)
   ;
 }
 
-/** Test tor_addr_port_parse(). */
+/* Test that addr_str successfully parses, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_PARSE_FMT(addr_str, expect_family, fmt_decorated, \
+                            expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that addr_str fails to parse, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_PARSE_XFAIL(addr_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str and default_port successfully parse, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, expect_family, \
+                                 fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that addr_port_str and default_port fail to parse, and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_PARSE_XFAIL(addr_port_str, default_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as an IPv4 address using
+ * tor_lookup_hostname(), and:
+ *  - the fmt_addr32() of the result is expect_str.
+ */
+#define TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, expect_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(addr_str, &addr32h); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_str_op(fmt_addr32(addr32h), OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse using tor_lookup_hostname(), with a
+ * permanent failure, and:
+ *  - the returned address is 0.
+ */
+#define TEST_ADDR_V4_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(bad_str, &addr32h); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_str as an IPv4 address using tor_lookup_hostname()
+ * does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is 0.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_V4_LOOKUP(host_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(host_str, &addr32h); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a require_family IP address using
+ * tor_addr_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_LOOKUP_FMT(addr_str, require_family, expect_family, \
+                             fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(addr_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse as a require_family IP address using
+ * tor_addr_lookup(), with a permanent failure, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_LOOKUP_XFAIL(bad_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(bad_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that looking up host_string as a require_family IP address using
+ * tor_addr_lookup(),  does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is null.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_LOOKUP(host_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(host_str, require_family, &addr); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, expect_family, \
+                                  fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(addr_port_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that bad_str fails to parse as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(bad_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_port_str as an IP address using
+ * tor_addr_port_lookup(),  does something sensible:
+ *  - the result is -1 or 0.
+ *  - if the result is a failure, the returned address is null, and the
+ *    returned port is zero,
+ *  - if the result is a success, the returned port is expect_success_port,
+ *    and the returned family is AF_INET or AF_INET6.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(host_port_str, &addr, &port); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 0); \
+    if (r == -1) { \
+      tt_assert(tor_addr_is_null(&addr)); \
+      tt_int_op(port, OP_EQ, 0); \
+    } else { \
+      tt_assert(tor_addr_family(&addr) == AF_INET || \
+                tor_addr_family(&addr) == AF_INET6); \
+      tt_int_op(port, OP_EQ, expect_success_port); \
+    } \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical IPv4 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup() with AF_INET,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_port_lookup(), because there is no port.
+ */
+#define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \
+                             addr_str, 111); \
+    TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET, 0, addr_str, 0); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET, AF_INET, 0, addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical fmt_decorated
+ * IPv6 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \
+                             addr_str, 222); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, addr_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_str successfully parses, and the fmt_decorated canonical
+ * IPv6 string is expect_str.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port.
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \
+                             expect_str, 333); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, expect_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical IPv4 address
+ * string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port.
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical undecorated
+ * IPv6 address string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port, and because it only
+ *    supports IPv4,
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET6, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+  STMT_END
+
+/* Test that bad_str fails to parse due to a bad address or port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup().
+ */
+#define TEST_ADDR_PARSE_XFAIL_MALFORMED(bad_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_XFAIL(bad_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str, 666); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(bad_str); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET6); \
+    TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str); \
+  STMT_END
+
+/* Test that host_str is treated as a hostname, and not an address.
+ * Check for success or failure using the network-dependent functions:
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup(), expecting a zero port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port.
+ */
+#define TEST_HOSTNAME(host_str) \
+  STMT_BEGIN \
+    TEST_HOST_V4_LOOKUP(host_str); \
+    TEST_HOST_LOOKUP(host_str, AF_UNSPEC); \
+    TEST_HOST_LOOKUP(host_str, AF_INET); \
+    TEST_HOST_LOOKUP(host_str, AF_INET6); \
+    TEST_HOST_PORT_LOOKUP(host_str, 0); \
+    TEST_ADDR_PARSE_XFAIL(host_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \
+  STMT_END
+
+/* Test that host_port_str is treated as a hostname and port, and not a
+ * hostname or an address.
+ * Check for success or failure using the network-dependent function:
+ *  - tor_addr_port_lookup(), expecting expect_success_port if the lookup is
+ *    successful.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(), because it doesn't support ports,
+ *  - tor_addr_lookup(), regardless of the address family, because it doesn't
+ *    support ports.
+ */
+#define TEST_HOSTNAME_PORT(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port); \
+    TEST_ADDR_PARSE_XFAIL(host_port_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET6); \
+  STMT_END
+
+static void
+test_addr_parse_canonical(void *arg)
+{
+  int r;
+  tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
+  char buf[TOR_ADDR_BUF_LEN];
+
+  (void)arg;
+
+  /* Correct calls. */
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.1");
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2");
+
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::33:44]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[2::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1);
+
+  /* Allow IPv6 without square brackets, when there is no port, but only if
+   * there is a default port */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::1", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("2::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0);
+ done:
+  ;
+}
+
+/** Test tor_addr_parse() and tor_addr_port_parse(). */
 static void
 test_addr_parse(void *arg)
 {
+
   int r;
   tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
   char buf[TOR_ADDR_BUF_LEN];
-  uint16_t port = 0;
 
-  /* Correct call. */
   (void)arg;
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.1:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "192.0.2.1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "::1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  /* Domain name. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
 
-  /* Only IP. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  mock_hostname_resolver();
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,200);
+  /* IPv6-mapped IPv4 addresses. Tor doesn't really use these. */
+  TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0,
+                     "11:22:33:44:55:66:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_ADDR_V6_PARSE("11:22::33:44:1.2.3.4", 0,
+                     "11:22::33:44:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, 400);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,400);
+  /* Ports. */
+  TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234);
+  TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234);
 
-  /* Bad port. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  /* Host names. */
+  TEST_HOSTNAME("localhost");
+  TEST_HOSTNAME_PORT("localhost:1234", 1234);
+  TEST_HOSTNAME_PORT("localhost:0", 0);
 
-  /* Only domain name */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("torproject.org");
+  TEST_HOSTNAME_PORT("torproject.org:56", 56);
 
-  /* Bad IP address */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("probably-not-a-valid-dns.name-tld");
+  TEST_HOSTNAME_PORT("probably-not-a-valid-dns.name-tld:789", 789);
+
+  /* Malformed addresses. */
+  /* Empty string. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("");
+
+  /* Square brackets around IPv4 address. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.1]");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345");
 
-  /* Make sure that the default port has lower priority than the real
-     one */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:1337",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1337);
+  /* Only left square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[11:22::33:44");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1369",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1369);
+  /* Only right square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44]");
+
+  /* Leading colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED(":11:22::33:44");
+
+  /* Trailing colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:");
+
+  /* Bad port. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:77777");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:88888");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:99999");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:-2");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:-3");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:-4");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:1 bad");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:bad-port-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port-1");
+
+  /* Bad hostname */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid:22222");
+
+  /* Ambiguous cases */
+  /* Too many hex words in IPv4-mapped IPv6 address.
+   * But some OS host lookup routines accept it as a hostname, or
+   * as an IP address?? (I assume they discard unused characters). */
+  TEST_HOSTNAME("11:22:33:44:55:66:77:88:1.2.3.4");
+
+  /* IPv6 address with port and no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address:port ? */
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+  /* Is it a port, or are there too many hex words?
+   * We reject it either way, but some OS host lookup routines accept it as an
+   * IPv6 address:port */
+  TEST_HOSTNAME_PORT("11:22:33:44:55:66:77:88:99", 99);
+  /* But we accept it if it has square brackets. */
+  TEST_ADDR_V6_PORT_PARSE("[11:22:33:44:55:66:77:88]:99",
+                           "11:22:33:44:55:66:77:88",99);
+
+  /* Bad IPv4 address
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv4 address[:port], with a zero last octet */
+  TEST_HOSTNAME("192.0.1");
+  TEST_HOSTNAME_PORT("192.0.2:1234", 1234);
+
+  /* More bad IPv6 addresses and ports: no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address[:port] */
+  TEST_HOSTNAME_PORT("::1:12345", 12345);
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+
+  /* And this is an ambiguous case, which is interpreted as an IPv6 address. */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::88:99", 0);
+  /* Use square brackets to resolve the ambiguity */
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::88:99]", 1);
+  TEST_ADDR_V6_PORT_PARSE("[11:22::88]:99",
+                           "11:22::88",99);
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -891,27 +1348,6 @@ test_virtaddrmap(void *data)
   ;
 }
 
-static const char *canned_data = NULL;
-static size_t canned_data_len = 0;
-
-/* Mock replacement for crypto_rand() that returns canned data from
- * canned_data above. */
-static void
-crypto_canned(char *ptr, size_t n)
-{
-  if (canned_data_len) {
-    size_t to_copy = MIN(n, canned_data_len);
-    memcpy(ptr, canned_data, to_copy);
-    canned_data += to_copy;
-    canned_data_len -= to_copy;
-    n -= to_copy;
-    ptr += to_copy;
-  }
-  if (n) {
-    crypto_rand_unmocked(ptr, n);
-  }
-}
-
 static void
 test_virtaddrmap_persist(void *data)
 {
@@ -919,6 +1355,8 @@ test_virtaddrmap_persist(void *data)
   const char *a, *b, *c;
   tor_addr_t addr;
   char *ones = NULL;
+  const char *canned_data;
+  size_t canned_data_len;
 
   addressmap_init();
 
@@ -937,7 +1375,7 @@ test_virtaddrmap_persist(void *data)
                 "1234567890" // the second call returns this.
                 "abcdefghij"; // the third call returns this.
   canned_data_len = 30;
-  MOCK(crypto_rand, crypto_canned);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
 
   a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
                                           tor_strdup("quuxit.baz"));
@@ -947,9 +1385,9 @@ test_virtaddrmap_persist(void *data)
   tt_assert(b);
   tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual");
   tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual");
+  testing_disable_prefilled_rng();
 
   // Now try something to get us an ipv4 address
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16",
                                                 AF_INET, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -966,22 +1404,23 @@ test_virtaddrmap_persist(void *data)
 
   // Try some canned entropy and verify all the we discard duplicates,
   // addresses that end with 0, and addresses that end with 255.
-  MOCK(crypto_rand, crypto_canned);
   canned_data = "\x01\x02\x03\x04" // okay
                 "\x01\x02\x03\x04" // duplicate
                 "\x03\x04\x00\x00" // bad ending 1
                 "\x05\x05\x00\xff" // bad ending 2
                 "\x05\x06\x07\xf0"; // okay
   canned_data_len = 20;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumble.onion"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumpus.onion"));
   tt_str_op(a, OP_EQ, "192.168.3.4");
   tt_str_op(b, OP_EQ, "192.168.7.240");
+  testing_disable_prefilled_rng();
 
   // Now try IPv6!
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20",
                                                 AF_INET6, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -997,7 +1436,7 @@ test_virtaddrmap_persist(void *data)
   tt_assert(!strcmpstart(b, "[1010:f"));
 
   // Try IPv6 with canned entropy, to make sure we detect duplicates.
-  MOCK(crypto_rand, crypto_canned);
+
   canned_data = "acanthopterygian" // okay
                 "cinematographist" // okay
                 "acanthopterygian" // duplicate
@@ -1006,6 +1445,8 @@ test_virtaddrmap_persist(void *data)
                 "cinematographist" // duplicate
                 "coadministration"; // okay
   canned_data_len = 16 * 7;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
                                           tor_strdup("wuffle.baz"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -1018,9 +1459,11 @@ test_virtaddrmap_persist(void *data)
 
   // Try address exhaustion: make sure we can actually fail if we
   // get too many already-existing addresses.
+  testing_disable_prefilled_rng();
   canned_data_len = 128*1024;
   canned_data = ones = tor_malloc(canned_data_len);
   memset(ones, 1, canned_data_len);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
   // There is some chance this one will fail if a previous random
   // allocation gave out the address already.
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -1037,7 +1480,7 @@ test_virtaddrmap_persist(void *data)
   expect_single_log_msg_containing("Ran out of virtual addresses!");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(ones);
   addressmap_free_all();
   teardown_capture_of_logs();
@@ -1213,6 +1656,7 @@ struct testcase_t addr_tests[] = {
   ADDR_LEGACY(basic),
   ADDR_LEGACY(ip6_helpers),
   ADDR_LEGACY(parse),
+  ADDR_LEGACY(parse_canonical),
   { "virtaddr", test_virtaddrmap, 0, NULL, NULL },
   { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL },
   { "localname", test_addr_localname, 0, NULL, NULL },
diff --git a/src/test/test_address.c b/src/test/test_address.c
index c33c30aee5..32fb4aa232 100644
--- a/src/test/test_address.c
+++ b/src/test/test_address.c
@@ -24,6 +24,10 @@
 #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/node_st.h"
 #include "feature/nodelist/nodelist.h"
 #include "lib/net/address.h"
 #include "test/test.h"
@@ -1170,8 +1174,167 @@ test_address_tor_addr_in_same_network_family(void *ignored)
   return;
 }
 
+static node_t *
+helper_create_mock_node(char id_char)
+{
+  node_t *node = tor_malloc_zero(sizeof(node_t));
+  routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t));
+  tor_addr_make_null(&ri->ipv6_addr, AF_INET6);
+  node->ri = ri;
+  memset(node->identity, id_char, sizeof(node->identity));
+  return node;
+}
+
+static void
+helper_free_mock_node(node_t *node)
+{
+  if (!node)
+    return;
+  tor_free(node->ri);
+  tor_free(node);
+}
+
+#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, ipv4_addr); \
+    node->ri->addr = tor_addr_to_ipv4h(&addr); \
+    node->ri->or_port = ipv4_port; \
+  }
+
+#define NODE_CLEAR_IPV4(node) { \
+    node->ri->addr = 0; \
+    node->ri->or_port = 0; \
+  }
+
+#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \
+    tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \
+    node->ri->ipv6_orport = ipv6_port; \
+  }
+
+static void
+test_address_tor_node_in_same_network_family(void *ignored)
+{
+  (void)ignored;
+  node_t *node_a = helper_create_mock_node('a');
+  node_t *node_b = helper_create_mock_node('b');
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "8.8.4.4", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "1.1.1.1", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_a);
+  NODE_SET_IPV6(node_a, "2001:470:20::2", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_b);
+  NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1);
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+ done:
+  helper_free_mock_node(node_a);
+  helper_free_mock_node(node_b);
+}
+
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+/* Test dirserv_router_has_valid_address() on a stub routerinfo, with only its
+ * address fields set. Use IPv4 ipv4_addr_str and IPv6 ipv6_addr_str.
+ * Fail if it does not return rv. */
+#define TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, ipv6_addr_str, rv) \
+  STMT_BEGIN \
+    ri = tor_malloc_zero(sizeof(routerinfo_t)); \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, (ipv4_addr_str));   \
+    ri->addr = tor_addr_to_ipv4h(&addr); \
+    tor_addr_parse(&ri->ipv6_addr, (ipv6_addr_str)); \
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, (rv)); \
+    tor_free(ri); \
+  STMT_END
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a null
+ * IPv6 address. */
+#define CHECK_RI_ADDR(ipv4_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, "::", rv)
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a non-internal
+ * IPv4 address, so that the IPv6 check is reached. */
+#define CHECK_RI_ADDR6(ipv6_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER("1.0.0.1", ipv6_addr_str, rv)
+
+static void
+test_address_dirserv_router_addr_private(void *opt_dir_allow_private)
+{
+  /* A stub routerinfo structure, with only its address fields set. */
+  routerinfo_t *ri = NULL;
+  /* The expected return value for private addresses.
+   * Modified if DirAllowPrivateAddresses is 1. */
+  int private_rv = -1;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(get_options, mock_get_options);
+
+  if (opt_dir_allow_private) {
+    mock_options.DirAllowPrivateAddresses = 1;
+    private_rv = 0;
+  }
+
+  CHECK_RI_ADDR("1.0.0.1", 0);
+  CHECK_RI_ADDR("10.0.0.1", private_rv);
+
+  CHECK_RI_ADDR6("2600::1", 0);
+  CHECK_RI_ADDR6("fe80::1", private_rv);
+
+  /* Null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  CHECK_RI_ADDR("0.0.0.0", private_rv);
+  TEST_ROUTER_VALID_ADDRESS_HELPER("0.0.0.0", "::", private_rv);
+
+  /* IPv6 null succeeds, because IPv4 is not null */
+  CHECK_RI_ADDR6("::", 0);
+
+  /* Byte-zeroed null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, private_rv);
+    tor_free(ri);
+  }
+
+  /* IPv6 null succeeds, because IPv4 is not internal */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    ri->addr = 16777217; /* 1.0.0.1 */
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, 0);
+    tor_free(ri);
+  }
+
+ done:
+  tor_free(ri);
+  UNMOCK(get_options);
+}
+
 #define ADDRESS_TEST(name, flags) \
   { #name, test_address_ ## name, flags, NULL, NULL }
+#define ADDRESS_TEST_STR_ARG(name, flags, str_arg) \
+  { #name "/" str_arg, test_address_ ## name, flags, &passthrough_setup, \
+    (void *)(str_arg) }
 
 struct testcase_t address_tests[] = {
   ADDRESS_TEST(udp_socket_trick_whitebox, TT_FORK),
@@ -1202,5 +1365,8 @@ struct testcase_t address_tests[] = {
   ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0),
   ADDRESS_TEST(tor_addr_eq_ipv4h, 0),
   ADDRESS_TEST(tor_addr_in_same_network_family, 0),
+  ADDRESS_TEST(tor_node_in_same_network_family, 0),
+  ADDRESS_TEST(dirserv_router_addr_private, 0),
+  ADDRESS_TEST_STR_ARG(dirserv_router_addr_private, 0, "allow_private"),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c
index fb8408b3c3..6e299d779e 100644
--- a/src/test/test_address_set.c
+++ b/src/test/test_address_set.c
@@ -4,6 +4,7 @@
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/address_set.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -31,6 +32,12 @@ mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
   return dummy_ns;
 }
 
+static void
+mock_dirlist_add_trusted_dir_addresses(void)
+{
+  return;
+}
+
 /* Number of address a single node_t can have. Default to the production
  * value. This is to control the size of the bloom filter. */
 static int addr_per_node = 2;
@@ -98,6 +105,8 @@ test_nodelist(void *arg)
        mock_networkstatus_get_latest_consensus_by_flavor);
   MOCK(get_estimated_address_per_node,
        mock_get_estimated_address_per_node);
+  MOCK(dirlist_add_trusted_dir_addresses,
+       mock_dirlist_add_trusted_dir_addresses);
 
   dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
   dummy_ns->flavor = FLAV_MICRODESC;
@@ -113,7 +122,10 @@ test_nodelist(void *arg)
    * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
   addr_per_node = 1024;
 
-  /* No node no nothing. The lookups should be empty. */
+  /* No node no nothing. The lookups should be empty. We've mocked the
+   * dirlist_add_trusted_dir_addresses in order for _no_ authorities to be
+   * added to the filter else it makes this test to trigger many false
+   * positive. */
   nodelist_set_consensus(dummy_ns);
 
   /* The address set should be empty. */
@@ -167,6 +179,7 @@ test_nodelist(void *arg)
   UNMOCK(networkstatus_get_latest_consensus);
   UNMOCK(networkstatus_get_latest_consensus_by_flavor);
   UNMOCK(get_estimated_address_per_node);
+  UNMOCK(dirlist_add_trusted_dir_addresses);
 }
 
 struct testcase_t address_set_tests[] = {
diff --git a/src/test/test_bt.sh b/src/test/test_bt.sh
index df8bcb8eda..312905a4e2 100755
--- a/src/test/test_bt.sh
+++ b/src/test/test_bt.sh
@@ -3,8 +3,6 @@
 
 exitcode=0
 
-ulimit -c 0
-
 export ASAN_OPTIONS="handle_segv=0:allow_user_segv_handler=1"
 "${builddir:-.}/src/test/test-bt-cl" backtraces || exit $?
 "${builddir:-.}/src/test/test-bt-cl" assert 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?"
diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c
index 0c15a02ee4..b29c2c6cbc 100644
--- a/src/test/test_bt_cl.c
+++ b/src/test/test_bt_cl.c
@@ -4,6 +4,9 @@
 #include "orconfig.h"
 #include <stdio.h>
 #include <stdlib.h>
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
 
 /* To prevent 'assert' from going away. */
 #undef TOR_COVERAGE
@@ -43,7 +46,7 @@ crash(int x)
     *(volatile int *)0 = 0;
 #endif /* defined(__clang_analyzer__) || defined(__COVERITY__) */
   } else if (crashtype == 1) {
-    tor_assert(1 == 0);
+    tor_assertf(1 == 0, "%d != %d", 1, 0);
   } else if (crashtype == -1) {
     ;
   }
@@ -88,6 +91,11 @@ main(int argc, char **argv)
     return 1;
   }
 
+#ifdef HAVE_SYS_RESOURCE_H
+  struct rlimit rlim = { .rlim_cur = 0, .rlim_max = 0 };
+  setrlimit(RLIMIT_CORE, &rlim);
+#endif
+
 #if !(defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
    defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION))
     puts("Backtrace reporting is not supported on this platform");
diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c
new file mode 100644
index 0000000000..80da7829ae
--- /dev/null
+++ b/src/test/test_btrack.c
@@ -0,0 +1,129 @@
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+
+static void
+send_state(const orconn_state_msg_t *msg_in)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_state_publish(msg);
+}
+
+static void
+send_status(const orconn_status_msg_t *msg_in)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_status_publish(msg);
+}
+
+static void
+send_chan(const ocirc_chan_msg_t *msg_in)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_btrack_launch(void *arg)
+{
+  orconn_state_msg_t conn;
+  ocirc_chan_msg_t circ;
+  memset(&conn, 0, sizeof(conn));
+  memset(&circ, 0, sizeof(circ));
+
+  (void)arg;
+  conn.gid = 1;
+  conn.chan = 1;
+  conn.proxy_type = PROXY_NONE;
+  conn.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 1;
+  circ.onehop = true;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1");
+  expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1");
+  teardown_capture_of_logs();
+
+  conn.gid = 2;
+  conn.chan = 2;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 2;
+  circ.onehop = false;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0");
+  expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+static void
+test_btrack_delete(void *arg)
+{
+  orconn_state_msg_t state;
+  orconn_status_msg_t status;
+  memset(&state, 0, sizeof(state));
+  memset(&status, 0, sizeof(status));
+
+  (void)arg;
+  state.gid = 1;
+  state.chan = 1;
+  state.proxy_type = PROXY_NONE;
+  state.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&state);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0");
+  teardown_capture_of_logs();
+
+  status.gid = 1;
+  status.status = OR_CONN_EVENT_CLOSED;
+  status.reason = 0;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_status(&status);
+  expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+struct testcase_t btrack_tests[] = {
+  { "launch", test_btrack_launch, TT_FORK, &helper_pubsub_setup, NULL },
+  { "delete", test_btrack_delete, TT_FORK, &helper_pubsub_setup, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c
index 3e7364a5c8..3084c19d74 100644
--- a/src/test/test_buffers.c
+++ b/src/test/test_buffers.c
@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/tls/tortls.h"
 #include "lib/compress/compress.h"
diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c
index 5a013aa268..e6f028ed74 100644
--- a/src/test/test_bwmgt.c
+++ b/src/test/test_bwmgt.c
@@ -6,18 +6,67 @@
  * \brief tests for bandwidth management / token bucket functions
  */
 
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
 #define TOKEN_BUCKET_PRIVATE
 
 #include "core/or/or.h"
-#include "test/test.h"
 
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/evloop/token_bucket.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+
+#include "app/config/or_options_st.h"
+#include "core/or/connection_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 // an imaginary time, in timestamp units. Chosen so it will roll over.
 static const uint32_t START_TS = UINT32_MAX-10;
 static const int32_t KB = 1024;
 static const uint32_t GB = (UINT64_C(1) << 30);
 
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+static networkstatus_t *dummy_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return dummy_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  tor_assert(f == FLAV_MICRODESC);
+  return dummy_ns;
+}
+
+/* Number of address a single node_t can have. Default to the production
+ * value. This is to control the size of the bloom filter. */
+static int addr_per_node = 2;
+static int
+mock_get_estimated_address_per_node(void)
+{
+  return addr_per_node;
+}
+
 static void
 test_bwmgt_token_buf_init(void *arg)
 {
@@ -220,8 +269,162 @@ test_bwmgt_token_buf_helpers(void *arg)
   ;
 }
 
+static void
+test_bwmgt_dir_conn_global_write_low(void *arg)
+{
+  bool ret;
+  int addr_family;
+  connection_t *conn = NULL;
+  routerstatus_t *rs = NULL; microdesc_t *md = NULL; routerinfo_t *ri = NULL;
+  tor_addr_t relay_addr;
+
+  (void) arg;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+  MOCK(get_estimated_address_per_node,
+       mock_get_estimated_address_per_node);
+
+  /*
+   * The following is rather complex but that is what it takes to add a dummy
+   * consensus with a valid routerlist which will populate our node address
+   * set that we need to lookup to test the known relay code path.
+   *
+   * We MUST do that before we MOCK(get_options) else it is another world of
+   * complexity.
+   */
+
+  /* This will be the address of our relay. */
+  tor_addr_parse(&relay_addr, "1.2.3.4");
+
+  /* We'll now add a relay into our routerlist and see if we let it. */
+  dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
+  dummy_ns->flavor = FLAV_MICRODESC;
+  dummy_ns->routerstatus_list = smartlist_new();
+
+  md = tor_malloc_zero(sizeof(*md));
+  ri = tor_malloc_zero(sizeof(*ri));
+  rs = tor_malloc_zero(sizeof(*rs));
+  crypto_rand(rs->identity_digest, sizeof(rs->identity_digest));
+  crypto_rand(md->digest, sizeof(md->digest));
+  memcpy(rs->descriptor_digest, md->digest, DIGEST256_LEN);
+
+  /* Set IP address. */
+  rs->addr = tor_addr_to_ipv4h(&relay_addr);
+  ri->addr = rs->addr;
+  /* Add the rs to the consensus becoming a node_t. */
+  smartlist_add(dummy_ns->routerstatus_list, rs);
+
+  /* Add all configured authorities (hardcoded) before we set the consensus so
+   * the address set exists. */
+  ret = consider_adding_dir_servers(&mock_options, &mock_options);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* This will make the nodelist bloom filter very large
+   * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
+  addr_per_node = 1024;
+
+  nodelist_set_consensus(dummy_ns);
+
+  /* Ok, now time to control which options we use. */
+  MOCK(get_options, mock_get_options);
+
+  /* Set ourselves as an authoritative dir. */
+  mock_options.AuthoritativeDir = 1;
+  mock_options.V3AuthoritativeDir = 1;
+  mock_options.UseDefaultFallbackDirs = 0;
+
+  /* This will set our global bucket to 1 byte and thus we will hit the
+   * banwdith limit in our test. */
+  mock_options.BandwidthRate = 1;
+  mock_options.BandwidthBurst = 1;
+
+  /* Else an IPv4 address screams. */
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Initialize the global buckets. */
+  connection_bucket_init();
+
+  /* The address "127.0.0.1" is set with this helper. */
+  conn = test_conn_get_connection(DIR_CONN_STATE_MIN_, CONN_TYPE_DIR,
+                                  DIR_PURPOSE_MIN_);
+  tt_assert(conn);
+
+  /* First try a non authority non relay IP thus a client but we are not
+   * configured to reject requests under load so we should get a false value
+   * that our limit is _not_ low. */
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Now, we will reject requests under load so try again a non authority non
+   * relay IP thus a client. We should get a warning that our limit is too
+   * low. */
+  mock_options.AuthDirRejectRequestsUnderLoad = 1;
+
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Now, lets try with a connection address from moria1. It should always
+   * pass even though our limit is too low. */
+  addr_family = tor_addr_parse(&conn->addr, "128.31.0.39");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* IPv6 testing of gabelmoo. */
+  addr_family = tor_addr_parse(&conn->addr, "[2001:638:a000:4140::ffff:189]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a known relay address. It should pass. Possible due to
+   * our consensus setting above. */
+  memcpy(&conn->addr, &relay_addr, sizeof(tor_addr_t));
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a random IP that is not an authority nor a relay. */
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Finally, just make sure it still denies an IP if we are _not_ a v3
+   * directory authority. */
+  mock_options.V3AuthoritativeDir = 0;
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Random IPv6 should not be allowed. */
+  addr_family = tor_addr_parse(&conn->addr, "[CAFE::ACAB]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  connection_free_minimal(conn);
+  routerstatus_free(rs); routerinfo_free(ri); microdesc_free(md);
+  smartlist_clear(dummy_ns->routerstatus_list);
+  networkstatus_vote_free(dummy_ns);
+
+  UNMOCK(get_estimated_address_per_node);
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+  UNMOCK(get_options);
+}
+
 #define BWMGT(name)                                          \
-  { #name, test_bwmgt_ ## name , 0, NULL, NULL }
+  { #name, test_bwmgt_ ## name , TT_FORK, NULL, NULL }
 
 struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_init),
@@ -229,5 +432,7 @@ struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_dec),
   BWMGT(token_buf_refill),
   BWMGT(token_buf_helpers),
+
+  BWMGT(dir_conn_global_write_low),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_channel.c b/src/test/test_channel.c
index e55b9b0750..6a6bc9d810 100644
--- a/src/test/test_channel.c
+++ b/src/test/test_channel.c
@@ -598,7 +598,6 @@ test_channel_outbound_cell(void *arg)
   circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan);
   tt_int_op(channel_num_circuits(chan), OP_EQ, 1);
   /* Test the cmux state. */
-  tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux);
   tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
             OP_EQ, 1);
 
@@ -1541,6 +1540,10 @@ test_channel_listener(void *arg)
   channel_listener_dump_statistics(chan, LOG_INFO);
 
  done:
+  if (chan) {
+    channel_listener_unregister(chan);
+    tor_free(chan);
+  }
   channel_free_all();
 }
 
@@ -1567,4 +1570,3 @@ struct testcase_t channel_tests[] = {
     NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c
index 5da2d81377..885246628e 100644
--- a/src/test/test_channelpadding.c
+++ b/src/test/test_channelpadding.c
@@ -21,7 +21,7 @@
 #include "test/log_test_helpers.h"
 #include "lib/tls/tortls.h"
 #include "lib/evloop/timers.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "core/or/cell_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -289,8 +289,6 @@ test_channelpadding_timers(void *arg)
   channel_t *chans[CHANNELS_TO_TEST];
   (void)arg;
 
-  tor_libevent_postfork();
-
   if (!connection_array)
     connection_array = smartlist_new();
 
@@ -393,7 +391,6 @@ test_channelpadding_killonehop(void *arg)
   channelpadding_decision_t decision;
   int64_t new_time;
   (void)arg;
-  tor_libevent_postfork();
 
   routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
   monotime_init();
@@ -502,8 +499,6 @@ test_channelpadding_consensus(void *arg)
   int64_t new_time;
   (void)arg;
 
-  tor_libevent_postfork();
-
   /*
    * Params tested:
    *   nf_pad_before_usage
@@ -898,8 +893,6 @@ test_channelpadding_decide_to_pad_channel(void *arg)
     connection_array = smartlist_new();
   (void)arg;
 
-  tor_libevent_postfork();
-
   monotime_init();
   monotime_enable_test_mocking();
   monotime_set_mock_time_nsec(1);
diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c
index 10513e451d..054d3910e4 100644
--- a/src/test/test_channeltls.c
+++ b/src/test/test_channeltls.c
@@ -8,7 +8,7 @@
 #define TOR_CHANNEL_INTERNAL_
 #include "core/or/or.h"
 #include "lib/net/address.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/mainloop/connection.h"
diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c
index 538f20781f..7291e04d6a 100644
--- a/src/test/test_circuitbuild.c
+++ b/src/test/test_circuitbuild.c
@@ -27,11 +27,11 @@ static smartlist_t dummy_nodes;
 static extend_info_t dummy_ei;
 
 static int
-mock_count_acceptable_nodes(smartlist_t *nodes)
+mock_count_acceptable_nodes(const smartlist_t *nodes, int direct)
 {
   (void)nodes;
 
-  return DEFAULT_ROUTE_LEN + 1;
+  return direct ? 1 : DEFAULT_ROUTE_LEN + 1;
 }
 
 /* Test route lengths when the caller of new_route_len() doesn't
@@ -167,6 +167,7 @@ test_upgrade_from_guard_wait(void *arg)
   tt_assert(!list);
 
  done:
+  smartlist_free(list);
   circuit_free(circ);
   entry_guard_free_(guard);
 }
@@ -177,6 +178,6 @@ struct testcase_t circuitbuild_tests[] = {
   { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL },
   { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL },
   { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK,
-    NULL, NULL },
+    &helper_pubsub_setup, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c
new file mode 100644
index 0000000000..934ddb0208
--- /dev/null
+++ b/src/test/test_circuitpadding.c
@@ -0,0 +1,3205 @@
+#define TOR_CHANNEL_INTERNAL_
+#define TOR_TIMERS_PRIVATE
+#define CIRCUITPADDING_PRIVATE
+#define CIRCUITPADDING_MACHINES_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define CRYPT_PATH_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "lib/testsupport/testsupport.h"
+#include "core/or/connection_or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/crypt_path.h"
+#include <event.h>
+#include "lib/evloop/compat_libevent.h"
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "core/or/relay.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/mainloop/netstatus.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/or/protover.h"
+#include "feature/nodelist/nodelist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "test/rng_test_helpers.h"
+
+/* Start our monotime mocking at 1 second past whatever monotime_init()
+ * thought the actual wall clock time was, for platforms with bad resolution
+ * and weird timevalues during monotime_init() before mocking. */
+#define MONOTIME_MOCK_START   (monotime_absolute_nsec()+\
+                               TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC)
+
+extern smartlist_t *connection_array;
+void circuit_expire_old_circuits_clientside(void);
+
+circid_t get_unique_circ_id_by_chan(channel_t *chan);
+void helper_create_basic_machine(void);
+static void helper_create_conditional_machines(void);
+
+static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan);
+channel_t *new_fake_channel(void);
+void test_circuitpadding_negotiation(void *arg);
+void test_circuitpadding_wronghop(void *arg);
+void test_circuitpadding_conditions(void *arg);
+
+void test_circuitpadding_serialize(void *arg);
+void test_circuitpadding_rtt(void *arg);
+void test_circuitpadding_tokens(void *arg);
+void test_circuitpadding_state_length(void *arg);
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding);
+void free_fake_orcirc(circuit_t *circ);
+void free_fake_origin_circuit(origin_circuit_t *circ);
+
+static int deliver_negotiated = 1;
+static int64_t curr_mocked_time;
+
+static node_t padding_node;
+static node_t non_padding_node;
+
+static channel_t dummy_channel;
+static circpad_machine_spec_t circ_client_machine;
+
+static void
+timers_advance_and_run(int64_t msec_update)
+{
+  curr_mocked_time += msec_update*TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+  timers_run_pending();
+}
+
+static void
+nodes_init(void)
+{
+  padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  padding_node.rs->pv.supports_hs_setup_padding = 1;
+
+  non_padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  non_padding_node.rs->pv.supports_hs_setup_padding = 0;
+}
+
+static void
+nodes_free(void)
+{
+  tor_free(padding_node.rs);
+
+  tor_free(non_padding_node.rs);
+}
+
+static const node_t *
+node_get_by_id_mock(const char *identity_digest)
+{
+  if (identity_digest[0] == 1) {
+    return &padding_node;
+  } else if (identity_digest[0] == 0) {
+    return &non_padding_node;
+  }
+
+  return NULL;
+}
+
+static const node_t *
+circuit_get_nth_node_mock(origin_circuit_t *circ, int hop)
+{
+  (void) circ;
+  (void) hop;
+
+  return &padding_node;
+}
+
+static or_circuit_t *
+new_fake_orcirc(channel_t *nchan, channel_t *pchan)
+{
+  or_circuit_t *orcirc = NULL;
+  circuit_t *circ = NULL;
+  crypt_path_t tmp_cpath;
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+
+  orcirc = tor_malloc_zero(sizeof(*orcirc));
+  circ = &(orcirc->base_);
+  circ->magic = OR_CIRCUIT_MAGIC;
+
+  //circ->n_chan = nchan;
+  circ->n_circ_id = get_unique_circ_id_by_chan(nchan);
+  cell_queue_init(&(circ->n_chan_cells));
+  circ->n_hop = NULL;
+  circ->streams_blocked_on_n_chan = 0;
+  circ->streams_blocked_on_p_chan = 0;
+  circ->n_delete_pending = 0;
+  circ->p_delete_pending = 0;
+  circ->received_destroy = 0;
+  circ->state = CIRCUIT_STATE_OPEN;
+  circ->purpose = CIRCUIT_PURPOSE_OR;
+  circ->package_window = CIRCWINDOW_START_MAX;
+  circ->deliver_window = CIRCWINDOW_START_MAX;
+  circ->n_chan_create_cell = NULL;
+
+  //orcirc->p_chan = pchan;
+  orcirc->p_circ_id = get_unique_circ_id_by_chan(pchan);
+  cell_queue_init(&(orcirc->p_chan_cells));
+
+  circuit_set_p_circid_chan(orcirc, orcirc->p_circ_id, pchan);
+  circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan);
+
+  memset(&tmp_cpath, 0, sizeof(tmp_cpath));
+  if (cpath_init_circuit_crypto(&tmp_cpath, whatevs_key,
+                                sizeof(whatevs_key), 0, 0)<0) {
+    log_warn(LD_BUG,"Circuit initialization failed");
+    return NULL;
+  }
+  orcirc->crypto = tmp_cpath.pvt_crypto;
+
+  return orcirc;
+}
+
+void
+free_fake_orcirc(circuit_t *circ)
+{
+  or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
+
+  relay_crypto_clear(&orcirc->crypto);
+
+  circpad_circuit_free_all_machineinfos(circ);
+  tor_free(circ);
+}
+
+void
+free_fake_origin_circuit(origin_circuit_t *circ)
+{
+  circpad_circuit_free_all_machineinfos(TO_CIRCUIT(circ));
+  circuit_clear_cpath(circ);
+  tor_free(circ);
+}
+
+void dummy_nop_timer(void);
+
+//static int dont_stop_libevent = 0;
+
+static circuit_t *client_side;
+static circuit_t *relay_side;
+
+static int n_client_cells = 0;
+static int n_relay_cells = 0;
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction)
+{
+  (void)cmux;
+  (void)circ;
+  (void)direction;
+
+  return;
+}
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno)
+{
+  (void)cell; (void)on_stream; (void)filename; (void)lineno;
+
+  if (circ == client_side) {
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      // Deliver to relay
+      circpad_handle_padding_negotiate(relay_side, cell);
+    } else {
+
+      int is_target_hop = circpad_padding_is_from_expected_hop(circ,
+                                                             layer_hint);
+      tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_OUT);
+      tt_int_op(is_target_hop, OP_EQ, 1);
+
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(client_side);
+
+      // Receive padding cell at middle
+      circpad_deliver_recognized_relay_cell_events(relay_side,
+              cell->payload[0], NULL);
+    }
+    n_client_cells++;
+  } else if (circ == relay_side) {
+    tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_IN);
+
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATED) {
+      // XXX: blah need right layer_hint..
+      if (deliver_negotiated)
+        circpad_handle_padding_negotiated(client_side, cell,
+                                          TO_ORIGIN_CIRCUIT(client_side)
+                                             ->cpath->next);
+    } else if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      circpad_handle_padding_negotiate(client_side, cell);
+    } else {
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(relay_side);
+
+      // Receive padding cell at client
+      circpad_deliver_recognized_relay_cell_events(client_side,
+              cell->payload[0],
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+    }
+
+    n_relay_cells++;
+  }
+
+ done:
+  timers_advance_and_run(1);
+  return 0;
+}
+
+// Test reading and writing padding to strings (or options_t + consensus)
+void
+test_circuitpadding_serialize(void *arg)
+{
+  (void)arg;
+}
+
+static signed_error_t
+circpad_send_command_to_hop_mock(origin_circuit_t *circ, uint8_t hopnum,
+                                 uint8_t relay_command, const uint8_t *payload,
+                                 ssize_t payload_len)
+{
+  (void) circ;
+  (void) hopnum;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  return 0;
+}
+
+void
+test_circuitpadding_rtt(void *arg)
+{
+  /* Test Plan:
+   *
+   * 1. Test RTT measurement server side
+   *    a. test usage of measured RTT
+   * 2. Test termination of RTT measurement
+   *    a. test non-update of RTT
+   * 3. Test client side circuit and non-application of RTT..
+   */
+  circpad_delay_t rtt_estimate;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+  helper_create_basic_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0);
+
+  /* Test 1: Test measuring RTT */
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+
+  timers_advance_and_run(20);
+
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 2: Termination of RTT measurement (from the previous test) */
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec;
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  timers_advance_and_run(4);
+  circpad_cell_event_nonpadding_sent(relay_side);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ,
+            rtt_estimate);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 3: Make sure client side machine properly ignores RTT */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_NE, client_side->padding_info[0]->rtt_estimate_usec);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_EQ,
+            circpad_machine_current_state(
+                client_side->padding_info[0])->histogram_edges[0]);
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+
+  return;
+}
+
+void
+helper_create_basic_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.name = "basic";
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_START].use_rtt_estimate = 1;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 2500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 5000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[4] = 20000;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+
+  return;
+}
+
+#define BIG_HISTOGRAM_LEN 10
+
+/** Setup a machine with a big histogram */
+static void
+helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy)
+{
+  const int tokens_per_bin = 2;
+
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circpad_state_t *burst_state =
+    &circ_client_machine.states[CIRCPAD_STATE_BURST];
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_RECV] =CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_SENT] =CIRCPAD_STATE_CANCEL;
+
+  burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  burst_state->histogram_len = BIG_HISTOGRAM_LEN;
+
+  int n_tokens = 0;
+  int i;
+  for (i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    burst_state->histogram[i] = tokens_per_bin;
+    n_tokens += tokens_per_bin;
+  }
+
+  burst_state->histogram_edges[0] = 0;
+  burst_state->histogram_edges[1] = 1;
+  burst_state->histogram_edges[2] = 7;
+  burst_state->histogram_edges[3] = 15;
+  burst_state->histogram_edges[4] = 31;
+  burst_state->histogram_edges[5] = 62;
+  burst_state->histogram_edges[6] = 125;
+  burst_state->histogram_edges[7] = 250;
+  burst_state->histogram_edges[8] = 500;
+  burst_state->histogram_edges[9] = 1000;
+
+  burst_state->histogram_total_tokens = n_tokens;
+  burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM;
+  burst_state->length_dist.param1 = n_tokens;
+  burst_state->length_dist.param2 = n_tokens;
+  burst_state->max_length = n_tokens;
+  burst_state->length_includes_nonpadding = 1;
+  burst_state->use_rtt_estimate = 0;
+  burst_state->token_removal = removal_strategy;
+}
+
+static circpad_decision_t
+circpad_machine_schedule_padding_mock(circpad_machine_runtime_t *mi)
+{
+  (void)mi;
+  return 0;
+}
+
+static uint64_t
+mock_monotime_absolute_usec(void)
+{
+  return 100;
+}
+
+/** Test higher token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_higher(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_HIGHER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Test right boundaries of each histogram bin: */
+  const circpad_delay_t bin_right_bounds[] =
+    {0,    6,  14,  30,  61,  124, 249, 499, 999, CIRCPAD_DELAY_INFINITE-1};
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_right_bounds[i], OP_EQ,
+               histogram_get_bin_upper_bound(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 5, 6, 7, 8};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all lower bins are not touched */
+  for (i=0; i < 4 ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  mi->padding_scheduled_at_usec = current_time;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test lower token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_lower(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_LOWER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 2, 1, 0};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all higher bins are untouched */
+  for (i = 5; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy by bin  */
+static void
+test_circuitpadding_closest_token_removal(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 6, 1, 7, 0, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_closest_token_removal_usec(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* XXX we want to test remove_token_exact and
+     circpad_machine_remove_closest_token() with usec */
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 1, 0, 6, 7, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_token_removal_exact(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_EXACT);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /**********************************************************************/
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Ensure that we will clear out bin #4 with this usec */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_cell_event_nonpadding_sent(client_side);
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 1);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[4], OP_EQ, 0);
+
+  /* Ensure that we will not remove any other tokens even tho we try to, since
+   * this is what the exact strategy dictates */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  circpad_cell_event_nonpadding_sent(client_side);
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    if (i != 4) {
+      tt_int_op(mi->histogram[i], OP_EQ, 2);
+    }
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+#undef BIG_HISTOGRAM_LEN
+
+void
+test_circuitpadding_tokens(void *arg)
+{
+  const circpad_state_t *state;
+  circpad_machine_runtime_t *mi;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  testing_enable_reproducible_rng();
+
+  /** Test plan:
+   *
+   * 1. Test symmetry between bin_to_usec and usec_to_bin
+   *    a. Test conversion
+   *    b. Test edge transitions (lower, upper)
+   * 2. Test remove higher on an empty bin
+   *    a. Normal bin
+   *    b. Infinity bin
+   *    c. Bin 0
+   *    d. No higher
+   * 3. Test remove lower
+   *    a. Normal bin
+   *    b. Bin 0
+   *    c. No lower
+   * 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  mi = client_side->padding_info[0];
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_sent(client_side);
+  /* We have to save the infinity bin because one inf delay
+   * could have been chosen when we transition to burst */
+  circpad_hist_token_t inf_bin = mi->histogram[4];
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  state = circpad_machine_current_state(client_side->padding_info[0]);
+
+  // Test 0: convert bin->usec->bin
+  // Bin 0+1 have different semantics
+  for (int bin = 0; bin < 2; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+  }
+  for (int bin = 2; bin < state->histogram_len-1; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    /* Verify we round down */
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec+3);
+    tt_int_op(bin, OP_EQ, bin2);
+
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec-1);
+    tt_int_op(bin, OP_EQ, bin2+1);
+  }
+
+  // Test 1: converting usec->bin->usec->bin
+  // Bin 0+1 have different semantics.
+  for (circpad_delay_t i = 0; i <= state->histogram_edges[0]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_LE, usec);
+  }
+  for (circpad_delay_t i = state->histogram_edges[0]+1;
+       i <= state->histogram_edges[0] +
+         state->histogram_edges[state->histogram_len-2]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_GE, usec);
+  }
+
+  /* 2.a. Normal higher bin */
+  {
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+  }
+
+  /* 2.b. Higher Infinity bin */
+  {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+
+    /* Test past the infinity bin */
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 5)+1000000);
+
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+  }
+
+  /* 2.c. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+    mi->histogram[0] = 1;
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* Drain the infinity bin and cause a refill */
+  while (inf_bin != 0) {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_cell_event_nonpadding_received(client_side);
+    inf_bin--;
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+
+  // We should have refilled here.
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 3.a. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 1);
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* 3.b. Test remove lower normal bin */
+  {
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    /* 3.c. No lower */
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+  }
+
+  /* 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 5. Test remove closest usec
+   *    a. Closest 0
+   *    b. Closest lower (below midpoint)
+   *    c. Closest higher (above midpoint)
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  // XXX: Need more coverage of the actual usec branches
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_wronghop(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Padding sent from hop 1 and 3 to client
+   * 2. Send negotiated from hop 1 and 3 to client
+   * 3. Garbled negotiated cell
+   * 4. Padding negotiate sent to client
+   * 5. Send negotiate stop command for unknown machine
+   * 6. Send negotiated to relay
+   * 7. Garbled padding negotiate cell
+   */
+  (void)arg;
+  uint32_t read_bw = 0, overhead_bw = 0;
+  cell_t cell;
+  signed_error_t ret;
+  origin_circuit_t *orig_client;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  /* Mock this function so that our cell counting tests don't get confused by
+   * padding that gets sent by scheduled timers. */
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  orig_client = TO_ORIGIN_CIRCUIT(client_side);
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build three hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+
+  /* 1. Test padding from first and third hop */
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_LT,
+            orig_client->n_overhead_read_circ_bw);
+
+  /* 2. Test padding negotiated not handled from hops 1,3 */
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 3. Garbled negotiated cell */
+  memset(&cell, 255, sizeof(cell));
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 4. Test that negotiate is dropped at origin */
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+  relay_send_command_from_edge(0, relay_side,
+                               RELAY_COMMAND_PADDING_NEGOTIATE,
+                               (void*)cell.payload,
+                               (size_t)3, NULL);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 5. Test that asking to stop the wrong machine does nothing */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            255, 2, CIRCPAD_COMMAND_STOP);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* 6. Sending negotiated command to relay does nothing */
+  ret = circpad_handle_padding_negotiated(relay_side, &cell, NULL);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 7. Test garbled negotated cell (bad command 255) */
+  memset(&cell, 0, sizeof(cell));
+  ret = circpad_handle_padding_negotiate(relay_side, &cell);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Spoof padding negotiated on circuit with no padding */
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_OK);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_ERR);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_negotiation(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Test circuit where padding is supported by middle
+   *    a. Make sure padding negotiation is sent
+   *    b. Test padding negotiation delivery and parsing
+   * 2. Test circuit where padding is unsupported by middle
+   *    a. Make sure padding negotiation is not sent
+   * 3. Test failure to negotiate a machine due to desync.
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Force negotiate padding. */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            CIRCPAD_MACHINE_CIRC_SETUP,
+                            2, CIRCPAD_COMMAND_START);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 3. Test failure to negotiate a machine due to desync */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  SMARTLIST_FOREACH(relay_padding_machines,
+          circpad_machine_spec_t *,
+          m, tor_free(m->states); tor_free(m));
+  smartlist_free(relay_padding_machines);
+  relay_padding_machines = smartlist_new();
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify echo was sent */
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+}
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding)
+{
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+  char digest[DIGEST_LEN];
+  tor_addr_t addr;
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_sent(client);
+
+  // Receive extend cell at middle
+  circpad_cell_event_nonpadding_received(mid_relay);
+
+  // Advance time a tiny bit so we can calculate an RTT
+  curr_mocked_time += 10 * TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+
+  // Receive extended cell at middle
+  circpad_cell_event_nonpadding_sent(mid_relay);
+
+  // Receive extended cell at first hop
+  circpad_cell_event_nonpadding_received(client);
+
+  // Add a hop to cpath
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+  cpath_extend_linked_list(&TO_ORIGIN_CIRCUIT(client)->cpath, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_OPEN;
+
+  // add an extend info to indicate if this node supports padding or not.
+  // (set the first byte of the digest for our mocked node_get_by_id)
+  digest[0] = padding;
+
+  hop->extend_info = extend_info_new(
+          padding ? "padding" : "non-padding",
+          digest, NULL, NULL, NULL,
+          &addr, padding);
+
+  cpath_init_circuit_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  // Signal that the hop was added
+  circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client));
+}
+
+static circpad_machine_spec_t *
+helper_create_length_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  /* No token removal.. end via state_length only */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Let's have this one end after 12 packets */
+  ret->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param1 = 12;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param2 = 13;
+  ret->states[CIRCPAD_STATE_BURST].max_length = 12;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 4;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 6;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 0;
+
+  return ret;
+}
+
+static circpad_machine_spec_t *
+helper_create_conditional_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Use EXACT removal strategy, otherwise setup_tokens() does not work */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 3;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 0;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  return ret;
+}
+
+static void
+helper_create_conditional_machines(void)
+{
+  circpad_machine_spec_t *add = helper_create_conditional_machine();
+
+  if (!origin_padding_machines)
+    origin_padding_machines = smartlist_new();
+  if (!relay_padding_machines)
+    relay_padding_machines = smartlist_new();
+
+  add->machine_num = 2;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 0;
+  add->conditions.min_hops = 2;
+  add->conditions.state_mask = CIRCPAD_CIRC_BUILDING|
+           CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 1;
+  add->conditions.min_hops = 3;
+  add->conditions.state_mask = CIRCPAD_CIRC_OPENED|
+           CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 2;
+  circpad_register_padding_machine(add, relay_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  circpad_register_padding_machine(add, relay_padding_machines);
+}
+
+void
+test_circuitpadding_state_length(void *arg)
+{
+  /**
+   * Test plan:
+   *  * Explicitly test that with no token removal enabled, we hit
+   *    the state length limit due to either padding, or non-padding.
+   *  * Repeat test with an arbitrary token removal strategy, and
+   *    verify that if we run out of tokens due to padding before we
+   *    hit the state length, we still go to state end (all our
+   *    token removal tests only test nonpadding token removal).
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  circpad_machine_spec_t *client_machine =
+      helper_create_length_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  client_side->padding_machine[0] = client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  circpad_machine_runtime_t *mi = client_side->padding_info[0];
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_EQ, NULL);
+
+  /* Verify that non-padding does not change our state length */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* verify that sending padding changes our state length */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+  circpad_send_padding_cell_for_callback(mi);
+
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+
+  /* Now, count nonpadding as part of the state length */
+  client_machine->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* Verify that non-padding does change our state length now */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_cell_event_nonpadding_sent(client_side);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Now, just test token removal when we send padding */
+  client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_NE, NULL);
+  tt_int_op(mi->chosen_bin, OP_EQ, 2);
+
+  /* verify that sending padding changes our state length and
+   * our histogram now */
+  for (uint32_t i = mi->histogram[2]-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_int_op(mi->chosen_bin, OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, i);
+  }
+
+  tt_i64_op(mi->state_length, OP_EQ, 7);
+  tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+  circpad_send_padding_cell_for_callback(mi);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+ done:
+  tor_free(client_machine->states);
+  tor_free(client_machine);
+
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(node_get_by_id);
+
+  return;
+}
+
+void
+test_circuitpadding_conditions(void *arg)
+{
+  /**
+   * Test plan:
+   *  0. Make a few origin and client machines with diff conditions
+   *     * vanguards, purposes, has_opened circs, no relay early
+   *     * Client side should_negotiate_end
+   *     * Length limits
+   *  1. Test STATE_END transitions
+   *  2. Test new machine after end with same conditions
+   *  3. Test new machine due to changed conditions
+   *     * Esp: built event, no relay early, no streams
+   * XXX: Diff test:
+   *  1. Test STATE_END with pending timers
+   *  2. Test marking a circuit before padding callback fires
+   *  3. Test freeing a circuit before padding callback fires
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  helper_create_conditional_machines();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Send another event.. verify machine gets re-added properly
+   * (test race with shutdown) */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = 0;
+  circpad_machine_event_circ_has_no_streams(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Now make the circuit opened and send built event */
+  TO_ORIGIN_CIRCUIT(client_side)->has_opened = 1;
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->remaining_relay_early_cells = 0;
+  circpad_machine_event_circ_has_no_relay_early(
+          TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  get_options_mutable()->HSLayer2Nodes = (void*)1;
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = (void*)1;
+  circpad_machine_event_circ_has_streams(TO_ORIGIN_CIRCUIT(client_side));
+
+  /* Verify different machine is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  /* Hold off on negotiated */
+  deliver_negotiated = 0;
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client
+   * but not the response for the padding machine */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify machine is gone from the relay (but negotiated not back yet */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Add another hop and verify it's back */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+ done:
+  /* XXX: Free everything */
+  testing_disable_reproducible_rng();
+  return;
+}
+
+/** Disabled unstable test until #29298 is implemented (see #29122) */
+#if 0
+void
+test_circuitpadding_circuitsetup_machine(void *arg)
+{
+  int64_t actual_mocked_monotime_start;
+  /**
+   * Test case plan:
+   *
+   * 1. Simulate a normal circuit setup pattern
+   *    a. Application traffic
+   *
+   * FIXME: This should focus more on exercising the machine
+   * features rather than actual traffic patterns. For example,
+   * test cancellation and bins empty/refill
+   */
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  dummy_channel.cmux = circuitmux_alloc();
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Test case #1: Build a 3 hop circuit, then wait and let pad */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(n_client_cells, OP_EQ, 1);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_BURST);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 6);
+
+  tt_int_op(client_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_END);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_GAP);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+
+  /* Verify we can't schedule padding in END state */
+  circpad_decision_t ret =
+      circpad_machine_schedule_padding(client_side->padding_info[0]);
+  tt_int_op(ret, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Simulate application traffic */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 7);
+
+  // Test timer cancellation
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 8);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Test timer cancel due to state rules */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  circpad_cell_event_padding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Simulate application traffic to cancel timer */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* No cells sent, except negotiate end from relay */
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 9);
+
+  /* Test mark for close and free */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE);
+  free_fake_orcirc(relay_side);
+  timers_advance_and_run(5000);
+
+  /* No cells sent */
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+
+  return;
+}
+#endif /* 0 */
+
+/** Helper function: Initializes a padding machine where every state uses the
+ *  uniform probability distribution.  */
+static void
+helper_circpad_circ_distribution_machine_setup(int min, int max)
+{
+  circpad_machine_states_init(&circ_client_machine, 7);
+
+  circpad_state_t *zero_st = &circ_client_machine.states[0];
+  zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1;
+  zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM;
+  /* param2 is upper bound, param1 is lower */
+  zero_st->iat_dist.param1 = min;
+  zero_st->iat_dist.param2 = max;
+  zero_st->dist_added_shift_usec = min;
+  zero_st->dist_max_sample_usec = max;
+
+  circpad_state_t *first_st = &circ_client_machine.states[1];
+  first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2;
+  first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC;
+  /* param1 is Mu, param2 is sigma. */
+  first_st->iat_dist.param1 = 9;
+  first_st->iat_dist.param2 = 3;
+  first_st->dist_added_shift_usec = min;
+  first_st->dist_max_sample_usec = max;
+
+  circpad_state_t *second_st = &circ_client_machine.states[2];
+  second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3;
+  second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC;
+  /* param1 is Alpha, param2 is 1.0/Beta */
+  second_st->iat_dist.param1 = 1;
+  second_st->iat_dist.param2 = 0.5;
+  second_st->dist_added_shift_usec = min;
+  second_st->dist_max_sample_usec = max;
+
+  circpad_state_t *third_st = &circ_client_machine.states[3];
+  third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4;
+  third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC;
+  /* param1 is 'p' (success probability) */
+  third_st->iat_dist.param1 = 0.2;
+  third_st->dist_added_shift_usec = min;
+  third_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fourth_st = &circ_client_machine.states[4];
+  fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5;
+  fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL;
+  /* param1 is k, param2 is Lambda */
+  fourth_st->iat_dist.param1 = 1.5;
+  fourth_st->iat_dist.param2 = 1;
+  fourth_st->dist_added_shift_usec = min;
+  fourth_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fifth_st = &circ_client_machine.states[5];
+  fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6;
+  fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO;
+  /* param1 is sigma, param2 is xi */
+  fifth_st->iat_dist.param1 = 1;
+  fifth_st->iat_dist.param2 = 5;
+  fifth_st->dist_added_shift_usec = min;
+  fifth_st->dist_max_sample_usec = max;
+}
+
+/** Simple test that the padding delays sampled from a uniform distribution
+ *  actually faill within the uniform distribution range. */
+static void
+test_circuitpadding_sample_distribution(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  int n_samples;
+  int n_states;
+
+  (void) arg;
+
+  /* mock this function so that we dont actually schedule any padding */
+  MOCK(circpad_machine_schedule_padding,
+       circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Initialize a machine with multiple probability distributions */
+  circpad_machines_init();
+  helper_circpad_circ_distribution_machine_setup(0, 10);
+
+  /* Initialize machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* For every state, sample a bunch of values from the distribution and ensure
+   * they fall within range. */
+  for (n_states = 0 ; n_states < 6; n_states++) {
+    /* Make sure we in the right state */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, n_states);
+
+    for (n_samples = 0; n_samples < 100; n_samples++) {
+      circpad_delay_t delay = circpad_machine_sample_delay(mi);
+      tt_int_op(delay, OP_GE, 0);
+      tt_int_op(delay, OP_LE, 10);
+    }
+
+    /* send a non-padding cell to move to the next machine state */
+    circpad_cell_event_nonpadding_received(client_side);
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  UNMOCK(circpad_machine_schedule_padding);
+  testing_disable_reproducible_rng();
+}
+
+static circpad_decision_t
+circpad_machine_spec_transition_mock(circpad_machine_runtime_t *mi,
+                                circpad_event_t event)
+{
+  (void) mi;
+  (void) event;
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/* Test per-machine padding rate limits */
+static void
+test_circuitpadding_machine_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  /* First we are going to test the per-machine rate limits */
+  circ_client_machine.max_padding_percent = 50;
+  circ_client_machine.allowed_padding_count = 100;
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Cover wrap */
+  for (;i<UINT16_MAX;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/2+1);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, mi);
+  for (i=0;i<UINT16_MAX;i++) {
+    circpad_cell_event_nonpadding_sent(client_side);
+  }
+
+  tt_int_op(mi->nonpadding_sent, OP_EQ, UINT16_MAX/2);
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/4+1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  testing_disable_reproducible_rng();
+}
+
+/* Test global padding rate limits */
+static void
+test_circuitpadding_global_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+  int64_t actual_mocked_monotime_start;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  testing_enable_reproducible_rng();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  dummy_channel.cmux = circuitmux_alloc();
+
+  /* Setup machine and circuits */
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  helper_create_basic_machine();
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(relay_side, 0);
+  mi = relay_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Now test the global limits by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_global_allowed_cells=100 circpad_global_max_padding_pct=50",
+                         NULL, 0, 0);
+  /* Register global limits with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Now send 92 non-padding cells to get near the
+   * circpad_global_max_padding_pct=50 limit; in particular with 96 non-padding
+   * cells, the padding traffic is still 51% of total traffic so limit should
+   * trigger */
+  for (i=0;i<92;i++) {
+    circpad_cell_event_nonpadding_sent(relay_side);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Send another non-padding cell to bring the padding traffic to 50% of total
+   * traffic and get past the limit */
+  circpad_cell_event_nonpadding_sent(relay_side);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  testing_disable_reproducible_rng();
+}
+
+/* Test reduced and disabled padding */
+static void
+test_circuitpadding_reduce_disable(void *arg)
+{
+  (void) arg;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel);
+  client_side = (circuit_t *)origin_circuit_new();
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  circpad_machines_init();
+  helper_create_conditional_machines();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Now test the reduced padding machine by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_reduced=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  /* Test that machines get torn down when padding is disabled */
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_disabled=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = NULL;
+  circpad_new_consensus_params(&vote1);
+
+  get_options_mutable()->ReducedCircuitPadding = 1;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  get_options_mutable()->CircuitPadding = 0;
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  testing_disable_reproducible_rng();
+}
+
+/** Just a basic machine whose whole purpose is to reach the END state */
+static void
+helper_create_ender_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_END;
+
+  circ_client_machine.conditions.state_mask = CIRCPAD_STATE_ALL;
+  circ_client_machine.conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+}
+
+static time_t mocked_timeofday;
+/** Set timeval to a mock date and time. This is necessary
+ * to make tor_gettimeofday() mockable. */
+static void
+mock_tor_gettimeofday(struct timeval *timeval)
+{
+  timeval->tv_sec = mocked_timeofday;
+  timeval->tv_usec = 0;
+}
+
+/** Test manual managing of circuit lifetimes by the circuitpadding
+ *  subsystem. In particular this test goes through all the cases of the
+ *  circpad_marked_circuit_for_padding() function, via
+ *  circuit_mark_for_close() as well as
+ *  circuit_expire_old_circuits_clientside(). */
+static void
+test_circuitpadding_manage_circuit_lifetime(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+  MOCK(tor_gettimeofday, mock_tor_gettimeofday);
+  mocked_timeofday = 23;
+
+  helper_create_ender_machine();
+
+  /* Enable manual circuit lifetime manage for this test */
+  circ_client_machine.manage_circ_lifetime = 1;
+
+  /* Test setup */
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Check that the circuit is not marked for close */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+
+  /* Mark this circuit for close due to a remote reason */
+  circuit_mark_for_close(client_side,
+                         END_CIRC_REASON_FLAG_REMOTE|END_CIRC_REASON_NONE);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close due to a protocol issue */
+  circuit_mark_for_close(client_side, END_CIRC_REASON_TORPROTOCOL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark a measurement circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  client_side->marked_for_close = 0;
+
+  /* Mark a general circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+
+  /* Check that this circuit is still not marked for close since we are
+   * managing the lifetime manually, but the circuit was tagged as such by the
+   * circpadding subsystem */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* We just tested case (1) from the comments of
+   * circpad_circuit_should_be_marked_for_close() */
+
+  /* Transition the machine to the END state but did not delete its machine */
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* We just tested case (3) from the comments of
+   * circpad_circuit_should_be_marked_for_close().
+   * Now let's go for case (2). */
+
+  /* Reset the close mark */
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close */
+  circuit_mark_for_close(client_side, 0);
+
+  /* See that the circ got closed since we are already in END state */
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* We just tested case (2). Now let's see that case (4) is unreachable as
+     that comment claims */
+
+  /* First, reset all close marks and tags */
+  client_side->marked_for_close = 0;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  /* Now re-create the ender machine so that we can transition to END again */
+  /* Free up some stuff first */
+  circpad_circuit_free_all_machineinfos(client_side);
+  tor_free(circ_client_machine.states);
+  helper_create_ender_machine();
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* Check we are in START. */
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Test that we don't expire this circuit yet */
+  client_side->timestamp_dirty = 0;
+  client_side->state = CIRCUIT_STATE_OPEN;
+  tor_gettimeofday(&client_side->timestamp_began);
+  TO_ORIGIN_CIRCUIT(client_side)->circuit_idle_timeout = 23;
+  mocked_timeofday += 24;
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->timestamp_dirty, OP_NE, 0);
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Runaway circpad test: if the machine does not transition to end,
+   * test that after CIRCPAD_DELAY_MAX_SECS, we get marked anyway */
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  client_side->padding_info[0]->last_cell_time_sec =
+      approx_time()-(CIRCPAD_DELAY_MAX_SECS+10);
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* Test back to normal: if we had activity, we won't close */
+  client_side->padding_info[0]->last_cell_time_sec = approx_time();
+  client_side->marked_for_close = 0;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Transition to END, but before we're past the dirty timer */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Verify that the circuit was not closed. */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Now that we are in END state, we can be closed by expiry, but via
+   * the timestamp_dirty path, not the idle path. So first test not dirty
+   * enough. */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  tor_free(circ_client_machine.states);
+  monotime_disable_test_mocking();
+  UNMOCK(tor_gettimeofday);
+}
+
+/** Helper for the test_circuitpadding_hs_machines test:
+ *
+ *  - Create a client and relay circuit.
+ *  - Setup right circuit purpose and attach a machine to the client circuit.
+ *  - Verify that state transitions work as intended and state length gets
+ *    enforced.
+ *
+ *  This function is able to do this test both for intro and rend circuits
+ *  depending on the value of <b>test_intro_circs</b>.
+ */
+static void
+helper_test_hs_machines(bool test_intro_circs)
+{
+  /* Setup the circuits */
+  origin_circuit_t *origin_client_side = origin_circuit_new();
+  client_side = TO_CIRCUIT(origin_client_side);
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+
+  /* extend the client circ to two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* machines only apply on opened circuits */
+  origin_client_side->has_opened = 1;
+
+  /************************************/
+
+  /* Attaching the client machine now won't work here because of a wrong
+   * purpose */
+  tt_assert(!client_side->padding_machine[0]);
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_assert(!client_side->padding_machine[0]);
+
+  /* Change the purpose, see the machine getting attached */
+  client_side->purpose = test_intro_circs ?
+    CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT : CIRCUIT_PURPOSE_C_REND_JOINED;
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify that the right machine is attached */
+  tt_str_op(client_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "client_ip_circ" : "client_rp_circ");
+  tt_str_op(relay_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "relay_ip_circ": "relay_rp_circ");
+
+  /***********************************/
+
+  /* Intro machines are at START state, but rend machines have already skipped
+   * to OBFUSCATE_CIRC_SETUP because of the sent PADDING_NEGOTIATE. */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /*Send non-padding to move the machines from START to OBFUSCATE_CIRC_SETUP */
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /* Check that the state lengths have been sampled and are within range */
+  circpad_machine_runtime_t *client_machine_runtime =
+    client_side->padding_info[0];
+  circpad_machine_runtime_t *relay_machine_runtime =
+    relay_side->padding_info[0];
+
+  if (test_intro_circs) {
+    /* on the client side, we don't send any padding so
+     * state length is not set */
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, -1);
+    /* relay side has state limits. check them */
+    tt_i64_op(relay_machine_runtime->state_length, OP_GE,
+              INTRO_MACHINE_MINIMUM_PADDING);
+    tt_i64_op(relay_machine_runtime->state_length, OP_LT,
+              INTRO_MACHINE_MAXIMUM_PADDING);
+  } else {
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, 1);
+    tt_i64_op(relay_machine_runtime->state_length, OP_EQ, 1);
+  }
+
+  if (test_intro_circs) {
+    int i;
+    /* Send state_length worth of padding from the relay and see that the
+     * client state goes to END */
+    for (i = (int) relay_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(relay_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted (the padding info should now be null on both sides) */
+    tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+    tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  } else {
+    int i;
+    /* Send state_length worth of padding and see that the state goes to END */
+    for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(client_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted. */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_END);
+  }
+
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+}
+
+/** Test that the HS circuit padding machines work as intended. */
+static void
+test_circuitpadding_hs_machines(void *arg)
+{
+  (void)arg;
+
+  /* Test logic:
+   *
+   * 1) Register the HS machines, which aim to hide the presense of
+   *    onion service traffic on the client-side
+   *
+   * 2) Call helper_test_hs_machines() to perform tests for the intro circuit
+   *    machines and for the rend circuit machines.
+  */
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock);
+  MOCK(circuit_get_nth_node, circuit_get_nth_node_mock);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  curr_mocked_time = 1*TOR_NSEC_PER_USEC;
+
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  /************************************/
+
+  /* Register the HS machines */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  /***********************************/
+
+  /* Do the tests for the intro circuit machines */
+  helper_test_hs_machines(true);
+  /* Do the tests for the rend circuit machines */
+  helper_test_hs_machines(false);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+
+  SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  smartlist_free(origin_padding_machines);
+  smartlist_free(relay_padding_machines);
+
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuit_get_nth_node);
+  UNMOCK(circpad_machine_schedule_padding);
+}
+
+/** Test that we effectively ignore non-padding cells in padding circuits. */
+static void
+test_circuitpadding_ignore_non_padding_cells(void *arg)
+{
+  int retval;
+  relay_header_t rh;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_CIRCUIT_PADDING;
+
+  rh.command = RELAY_COMMAND_BEGIN;
+
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = handle_relay_cell_command(NULL, client_side, NULL, NULL, &rh, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  expect_log_msg_containing("Ignored cell");
+
+ done:
+  ;
+}
+
+#define TEST_CIRCUITPADDING(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t circuitpadding_tests[] = {
+  TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_state_length, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK),
+  /** Disabled unstable test until #29298 is implemented (see #29122) */
+  //  TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_reduce_disable, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_manage_circuit_lifetime, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_hs_machines, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_ignore_non_padding_cells, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c
index 1cbcb14f2b..9bfaabeb2f 100644
--- a/src/test/test_circuitstats.c
+++ b/src/test/test_circuitstats.c
@@ -28,7 +28,7 @@ origin_circuit_t *subtest_fourhop_circuit(struct timeval, int);
 origin_circuit_t *add_opened_threehop(void);
 origin_circuit_t *build_unopened_fourhop(struct timeval);
 
-int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 
 static int marked_for_close;
 /* Mock function because we are not trying to test the close circuit that does
@@ -57,9 +57,9 @@ add_opened_threehop(void)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = DEFAULT_ROUTE_LEN;
 
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
 
   or_circ->has_opened = 1;
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -82,10 +82,10 @@ build_unopened_fourhop(struct timeval circ_start_time)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = 4;
 
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
 
   tor_free(fakehop);
 
@@ -197,7 +197,7 @@ test_circuitstats_hoplen(void *arg)
 }
 
 #define TEST_CIRCUITSTATS(name, flags) \
-    { #name, test_##name, (flags), NULL, NULL }
+    { #name, test_##name, (flags), &helper_pubsub_setup, NULL }
 
 struct testcase_t circuitstats_tests[] = {
   TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK),
diff --git a/src/test/test_cmdline.sh b/src/test/test_cmdline.sh
new file mode 100755
index 0000000000..cf758c3851
--- /dev/null
+++ b/src/test/test_cmdline.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+umask 077
+set -e
+
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+die() { echo "$1" >&2 ; exit 5; }
+
+echo "A"
+
+DATA_DIR=$(mktemp -d -t tor_cmdline_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# 1. Test list-torrc-options.
+OUT="${DATA_DIR}/output"
+
+echo "B"
+"${TOR_BINARY}" --list-torrc-options > "$OUT"
+
+echo "C"
+
+# regular options are given.
+grep -i "SocksPort" "$OUT" >/dev/null || die "Did not find SocksPort"
+
+
+echo "D"
+
+# unlisted options are given, since they do not have the NOSET flag.
+grep -i "__SocksPort" "$OUT" > /dev/null || die "Did not find __SocksPort"
+
+echo "E"
+
+# unsettable options are not given.
+if grep -i "DisableIOCP" "$OUT"  /dev/null; then
+    die "Found DisableIOCP"
+fi
+if grep -i "HiddenServiceOptions" "$OUT" /dev/null ; then
+    die "Found HiddenServiceOptions"
+fi
+echo "OK"
diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c
index 2f8646e897..ecd97e3474 100644
--- a/src/test/test_compat_libevent.c
+++ b/src/test/test_compat_libevent.c
@@ -151,8 +151,6 @@ test_compat_libevent_postloop_events(void *arg)
   mainloop_event_t *a = NULL, *b = NULL;
   periodic_timer_t *timed = NULL;
 
-  tor_libevent_postfork();
-
   /* If postloop events don't work, then these events will activate one
    * another ad infinitum and, and the periodic event will never occur. */
   b = mainloop_event_postloop_new(activate_event_cb, &a);
@@ -187,4 +185,3 @@ struct testcase_t compat_libevent_tests[] = {
     TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_config.c b/src/test/test_config.c
index 855725411a..ebc0624fb2 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -16,7 +16,7 @@
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "test/test.h"
@@ -45,6 +45,7 @@
 #include "app/config/statefile.h"
 
 #include "test/test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "feature/dirclient/dir_server_st.h"
 #include "core/or/port_cfg_st.h"
@@ -54,6 +55,7 @@
 #include "lib/meminfo/meminfo.h"
 #include "lib/net/gethostname.h"
 #include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -1754,6 +1756,18 @@ add_default_fallback_dir_servers_known_default(void)
   n_add_default_fallback_dir_servers_known_default++;
 }
 
+/* Helper for test_config_adding_dir_servers(), which should be
+ * refactored: clear the fields in the options which the options object
+ * does not really own. */
+static void
+ads_clear_helper(or_options_t *options)
+{
+  options->DirAuthorities = NULL;
+  options->AlternateBridgeAuthority = NULL;
+  options->AlternateDirAuthority = NULL;
+  options->FallbackDir = NULL;
+}
+
 /* Test all the different combinations of adding dir servers */
 static void
 test_config_adding_dir_servers(void *arg)
@@ -1761,7 +1775,7 @@ test_config_adding_dir_servers(void *arg)
   (void)arg;
 
   /* allocate options */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
 
   /* Allocate and populate configuration lines:
    *
@@ -1884,7 +1898,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -1966,7 +1982,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2107,7 +2125,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2248,7 +2268,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2390,7 +2412,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2542,7 +2566,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2696,7 +2722,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2859,7 +2887,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3016,7 +3046,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3182,7 +3214,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3345,7 +3379,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3514,10 +3550,7 @@ test_config_adding_dir_servers(void *arg)
   tor_free(test_fallback_directory->value);
   tor_free(test_fallback_directory);
 
-  options->DirAuthorities = NULL;
-  options->AlternateBridgeAuthority = NULL;
-  options->AlternateDirAuthority = NULL;
-  options->FallbackDir = NULL;
+  ads_clear_helper(options);
   or_options_free(options);
 
   UNMOCK(add_default_fallback_dir_servers);
@@ -3532,7 +3565,7 @@ test_config_default_dir_servers(void *arg)
   int fallback_count = 0;
 
   /* new set of options should stop fallback parsing */
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 0;
   /* set old_options to NULL to force dir update */
   consider_adding_dir_servers(opts, NULL);
@@ -3546,7 +3579,7 @@ test_config_default_dir_servers(void *arg)
   /* if we disable the default fallbacks, there must not be any extra */
   tt_assert(fallback_count == trusted_count);
 
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 1;
   consider_adding_dir_servers(opts, opts);
   trusted_count = smartlist_len(router_get_trusted_dir_servers());
@@ -3606,7 +3639,7 @@ test_config_directory_fetch(void *arg)
   (void)arg;
 
   /* Test Setup */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
   routerinfo_t routerinfo;
   memset(&routerinfo, 0, sizeof(routerinfo));
   mock_router_pick_published_address_result = -1;
@@ -3618,9 +3651,10 @@ test_config_directory_fetch(void *arg)
        mock_router_my_exit_policy_is_reject_star);
   MOCK(advertised_server_mode, mock_advertised_server_mode);
   MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo);
+  or_options_free(options);
+  options = options_new();
 
   /* Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
   options->ClientOnly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
@@ -3629,7 +3663,8 @@ test_config_directory_fetch(void *arg)
             OP_EQ, 1);
 
   /* Bridge Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->UseBridges = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
@@ -3639,7 +3674,8 @@ test_config_directory_fetch(void *arg)
 
   /* Bridge Relays (Bridges) must act like clients, and use multiple
    * directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->BridgeRelay = 1;
   options->ORPort_set = 1;
   tt_assert(server_mode(options) == 1);
@@ -3650,7 +3686,8 @@ test_config_directory_fetch(void *arg)
 
   /* Clients set to FetchDirInfoEarly must fetch it from the authorities,
    * but can use multiple authorities for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->FetchDirInfoEarly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
@@ -3661,7 +3698,8 @@ test_config_directory_fetch(void *arg)
   /* OR servers only fetch the consensus from the authorities when they don't
    * know their own address, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
 
   mock_router_pick_published_address_result = -1;
@@ -3681,7 +3719,8 @@ test_config_directory_fetch(void *arg)
   /* Exit OR servers only fetch the consensus from the authorities when they
    * refuse unknown exits, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
   options->ExitRelay = 1;
   mock_router_pick_published_address_result = 0;
@@ -3711,7 +3750,8 @@ test_config_directory_fetch(void *arg)
    * advertising their dirport, and never use multiple directories for
    * bootstrap. This only applies if they are also OR servers.
    * (We don't care much about the behaviour of non-OR directory servers.) */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->DirPort_set = 1;
   options->ORPort_set = 1;
   options->DirCache = 1;
@@ -3765,7 +3805,7 @@ test_config_directory_fetch(void *arg)
             OP_EQ, 0);
 
  done:
-  tor_free(options);
+  or_options_free(options);
   UNMOCK(router_pick_published_address);
   UNMOCK(router_get_my_routerinfo);
   UNMOCK(advertised_server_mode);
@@ -4029,6 +4069,8 @@ test_config_parse_port_config__ports__ports_given(void *data)
 
   slout = smartlist_new();
 
+  mock_hostname_resolver();
+
   // Test error when encounters an invalid Port specification
   config_port_invalid = mock_config_line("DNSPort", "");
   ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
@@ -4725,6 +4767,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
 #endif /* defined(_WIN32) */
 
  done:
+  unmock_hostname_resolver();
   if (slout)
     SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_free(slout);
@@ -5063,7 +5106,7 @@ test_config_include_no_permission(void *data)
     chmod(dir, 0700);
   tor_free(dir);
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 static void
 test_config_include_recursion_before_after(void *data)
@@ -5778,7 +5821,7 @@ test_config_compute_max_mem_in_queues(void *data)
 #else
   /* We are on a 32-bit system. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1));
-#endif
+#endif /* SIZEOF_VOID_P >= 8 */
 
   /* We are able to detect the amount of RAM on the system. */
   total_system_memory_return = 0;
@@ -5819,7 +5862,7 @@ test_config_compute_max_mem_in_queues(void *data)
   /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
             MAX_DEFAULT_MEMORY_QUEUE_SIZE);
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
  done:
   UNMOCK(get_total_system_memory);
@@ -5863,6 +5906,7 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "is back here");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
   lp = lp->next;
+  tt_assert(!lp);
   config_free_lines(lines);
 
   /* Try with the "extended" flag enabled. */
@@ -5889,9 +5933,166 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR);
   lp = lp->next;
+  tt_assert(!lp);
+
+ done:
+  config_free_lines(lines);
+}
+
+static void
+test_config_kvline_parse(void *arg)
+{
+  (void)arg;
+
+  config_line_t *lines = NULL;
+  char *enc = NULL;
+
+  lines = kvline_parse("A=B CD=EF", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "A");
+  tt_str_op(lines->value, OP_EQ, "B");
+  tt_str_op(lines->next->key, OP_EQ, "CD");
+  tt_str_op(lines->next->value, OP_EQ, "EF");
+  enc = kvline_encode(lines, 0);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB CDE=F", 0);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "");
+  tt_str_op(lines->value, OP_EQ, "AB");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F");
+  tt_assert(lines);
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB CDE=F");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\"", 0);
+  tt_assert(!lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "C");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F G");
+  tt_str_op(lines->next->next->key, OP_EQ, "");
+  tt_str_op(lines->next->next->value, OP_EQ, "GHI");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB=", KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse(" AB ", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD DE FGH=I", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD DE FGH=I");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=\"CD E\" DE FGH=\"I\"", KV_OMIT_VALS|KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD E");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS|KV_QUOTED);
+  tt_str_op(enc, OP_EQ, "AB=\"CD E\" DE FGH=I");
 
  done:
   config_free_lines(lines);
+  tor_free(enc);
+}
+
+static void
+test_config_getinfo_config_names(void *arg)
+{
+  (void)arg;
+  char *answer = NULL;
+  const char *error = NULL;
+  int rv;
+
+  rv = getinfo_helper_config(NULL, "config/names", &answer, &error);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(error, OP_EQ, NULL);
+
+  // ContactInfo should be listed.
+  tt_assert(strstr(answer, "\nContactInfo String\n"));
+
+  // V1AuthoritativeDirectory should not be listed, since it is obsolete.
+  tt_assert(! strstr(answer, "V1AuthoritativeDirectory"));
+
+  // ___UsingTestNetworkDefaults should not be listed, since it is invisible.
+  tt_assert(! strstr(answer, "UsingTestNetworkDefaults"));
+
+ done:
+  tor_free(answer);
 }
 
 #define CONFIG_TEST(name, flags)                          \
@@ -5946,5 +6147,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(include_opened_file_list, 0),
   CONFIG_TEST(compute_max_mem_in_queues, 0),
   CONFIG_TEST(extended_fmt, 0),
+  CONFIG_TEST(kvline_parse, 0),
+  CONFIG_TEST(getinfo_config_names, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_confmgr.c b/src/test/test_confmgr.c
new file mode 100644
index 0000000000..d5c73b48e4
--- /dev/null
+++ b/src/test/test_confmgr.c
@@ -0,0 +1,325 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confparse.c's features that support multiple configuration
+ * formats and configuration objects.
+ */
+
+#define CONFPARSE_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confparse.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+/*
+ * Set up a few objects: a pasture_cfg is toplevel; it has a llama_cfg and an
+ * alpaca_cfg.
+ */
+
+typedef struct {
+  uint32_t magic;
+  char *address;
+  int opentopublic;
+  config_suite_t *subobjs;
+} pasture_cfg_t;
+
+typedef struct {
+  char *llamaname;
+  int cuteness;
+  uint32_t magic;
+  int eats_meat; /* deprecated; llamas are never carnivorous. */
+
+  char *description; // derived from other fields.
+} llama_cfg_t;
+
+typedef struct {
+  uint32_t magic;
+  int fuzziness;
+  char *alpacaname;
+  int n_wings; /* deprecated; alpacas don't have wings. */
+} alpaca_cfg_t;
+
+/*
+ * Make the above into configuration objects.
+ */
+
+static pasture_cfg_t pasture_cfg_t_dummy;
+static llama_cfg_t llama_cfg_t_dummy;
+static alpaca_cfg_t alpaca_cfg_t_dummy;
+
+#define PV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(pasture_cfg_t, #name, type, name, 0, dflt)
+#define LV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(llama_cfg_t, #name, type, name, 0, dflt)
+#define AV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(alpaca_cfg_t, #name, type, name, 0, dflt)
+static const config_var_t pasture_vars[] = {
+  PV(address, STRING, NULL),
+  PV(opentopublic, BOOL, "1"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t llama_vars[] =
+{
+  LV(llamaname, STRING, NULL),
+  LV(eats_meat, BOOL, NULL),
+  LV(cuteness, POSINT, "100"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t alpaca_vars[] =
+{
+  AV(alpacaname, STRING, NULL),
+  AV(fuzziness, POSINT, "50"),
+  AV(n_wings, POSINT, "0"),
+  END_OF_CONFIG_VARS
+};
+
+static config_deprecation_t llama_deprecations[] = {
+  { "eats_meat", "Llamas are herbivores." },
+  {NULL,NULL}
+};
+
+static config_deprecation_t alpaca_deprecations[] = {
+  { "n_wings", "Alpacas are quadrupeds." },
+  {NULL,NULL}
+};
+
+static int clear_llama_cfg_called = 0;
+static void
+clear_llama_cfg(const config_mgr_t *mgr, void *llamacfg)
+{
+  (void)mgr;
+  llama_cfg_t *lc = llamacfg;
+  tor_free(lc->description);
+  ++clear_llama_cfg_called;
+}
+
+static config_abbrev_t llama_abbrevs[] = {
+  { "gracia", "cuteness", 0, 0 },
+  { "gentillesse", "cuteness", 0, 0 },
+  { NULL, NULL, 0, 0 },
+};
+
+static const config_format_t pasture_fmt = {
+  sizeof(pasture_cfg_t),
+  {
+    "pasture_cfg_t",
+    8989,
+    offsetof(pasture_cfg_t, magic)
+  },
+  .vars = pasture_vars,
+  .config_suite_offset = offsetof(pasture_cfg_t, subobjs),
+};
+
+static const config_format_t llama_fmt = {
+  sizeof(llama_cfg_t),
+  {
+    "llama_cfg_t",
+    0x11aa11,
+    offsetof(llama_cfg_t, magic)
+  },
+  .vars = llama_vars,
+  .config_suite_offset = -1,
+  .deprecations = llama_deprecations,
+  .abbrevs = llama_abbrevs,
+  .clear_fn = clear_llama_cfg,
+};
+
+static const config_format_t alpaca_fmt = {
+  sizeof(alpaca_cfg_t),
+  {
+    "alpaca_cfg_t",
+    0xa15aca,
+    offsetof(alpaca_cfg_t, magic)
+  },
+  .vars = alpaca_vars,
+  .config_suite_offset = -1,
+  .deprecations = alpaca_deprecations,
+};
+
+#define LLAMA_IDX 0
+#define ALPACA_IDX 1
+
+static config_mgr_t *
+get_mgr(bool freeze)
+{
+  config_mgr_t *mgr = config_mgr_new(&pasture_fmt);
+  tt_int_op(LLAMA_IDX, OP_EQ, config_mgr_add_format(mgr, &llama_fmt));
+  tt_int_op(ALPACA_IDX, OP_EQ, config_mgr_add_format(mgr, &alpaca_fmt));
+  if (freeze)
+    config_mgr_freeze(mgr);
+  return mgr;
+
+ done:
+  config_mgr_free(mgr);
+  return NULL;
+}
+
+static void
+test_confmgr_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  smartlist_t *vars = NULL;
+  tt_ptr_op(mgr, OP_NE, NULL);
+
+  vars = config_mgr_list_vars(mgr);
+  tt_int_op(smartlist_len(vars), OP_EQ, 8); // 8 vars total.
+
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "CUTENESS"));
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "GRACIA"));
+  smartlist_free(vars);
+
+  vars = config_mgr_list_deprecated_vars(mgr); // 2 deprecated vars.
+  tt_int_op(smartlist_len(vars), OP_EQ, 2);
+  tt_assert(smartlist_contains_string(vars, "eats_meat"));
+  tt_assert(smartlist_contains_string(vars, "n_wings"));
+
+  tt_str_op("Llamas are herbivores.", OP_EQ,
+            config_find_deprecation(mgr, "EATS_MEAT"));
+  tt_str_op("Alpacas are quadrupeds.", OP_EQ,
+            config_find_deprecation(mgr, "N_WINGS"));
+
+ done:
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confmgr_magic(void *args)
+{
+  (void)args;
+  // Every time we build a manager, it is supposed to get a different magic
+  // number.  Let's test that.
+  config_mgr_t *mgr1 = get_mgr(true);
+  config_mgr_t *mgr2 = get_mgr(true);
+  config_mgr_t *mgr3 = get_mgr(true);
+
+  pasture_cfg_t *p1 = NULL, *p2 = NULL, *p3 = NULL;
+
+  tt_assert(mgr1);
+  tt_assert(mgr2);
+  tt_assert(mgr3);
+
+  p1 = config_new(mgr1);
+  p2 = config_new(mgr2);
+  p3 = config_new(mgr3);
+
+  tt_assert(p1);
+  tt_assert(p2);
+  tt_assert(p3);
+
+  // By chance, two managers get the same magic with P=2^-32.  Let's
+  // make sure that at least two of them are different, so that our
+  // odds of a false positive are 1/2^-64.
+  tt_assert((p1->magic != p2->magic) || (p2->magic != p3->magic));
+
+ done:
+  config_free(mgr1, p1);
+  config_free(mgr2, p2);
+  config_free(mgr3, p3);
+
+  config_mgr_free(mgr1);
+  config_mgr_free(mgr2);
+  config_mgr_free(mgr3);
+}
+
+static const char *simple_pasture =
+  "LLamaname hugo\n"
+  "Alpacaname daphne\n"
+  "gentillesse 42\n"
+  "address 123 Camelid ave\n";
+
+static void
+test_confmgr_parse(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, p); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_int_op(p->opentopublic, OP_EQ, 1);
+  tt_str_op(p->address, OP_EQ, "123 Camelid ave");
+
+  // We are using this API directly; modules outside confparse will, in the
+  // future, not.
+  const alpaca_cfg_t *ac = config_mgr_get_obj(mgr, p, ALPACA_IDX);
+  const llama_cfg_t *lc = config_mgr_get_obj(mgr, p, LLAMA_IDX);
+  tt_str_op(lc->llamaname, OP_EQ, "hugo");
+  tt_str_op(ac->alpacaname, OP_EQ, "daphne");
+  tt_int_op(lc->cuteness, OP_EQ, 42);
+  tt_int_op(ac->fuzziness, OP_EQ, 50);
+
+  // We set the description for the llama here, so that the clear function
+  // can clear it.  (Later we can do this in a verification function.)
+  clear_llama_cfg_called = 0;
+  llama_cfg_t *mut_lc = config_mgr_get_obj_mutable(mgr, p, LLAMA_IDX);
+  mut_lc->description = tor_strdup("A llama named Hugo.");
+  config_free(mgr, p);
+  tt_int_op(clear_llama_cfg_called, OP_EQ, 1);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_mgr_free(mgr);
+  tor_free(msg);
+}
+
+static void
+test_confmgr_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  pasture_cfg_t *defaults = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+  char *s = NULL;
+
+  config_init(mgr, p); // set defaults.
+  config_init(mgr, defaults); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  s = config_dump(mgr, defaults, p, 1, 0);
+  tt_str_op("address 123 Camelid ave\n"
+            "alpacaname daphne\n"
+            "cuteness 42\n"
+            "llamaname hugo\n", OP_EQ, s);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_free(mgr, defaults);
+  config_mgr_free(mgr);
+
+  tor_free(msg);
+  tor_free(s);
+}
+
+#define CONFMGR_TEST(name, flags)                       \
+  { #name, test_confmgr_ ## name, flags, NULL, NULL }
+
+struct testcase_t confmgr_tests[] = {
+  CONFMGR_TEST(init, 0),
+  CONFMGR_TEST(magic, 0),
+  CONFMGR_TEST(parse, 0),
+  CONFMGR_TEST(dump, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c
new file mode 100644
index 0000000000..5f29a22c10
--- /dev/null
+++ b/src/test/test_confparse.c
@@ -0,0 +1,1086 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confparse.c module that we use to parse various
+ * configuration/state file types.
+ */
+
+#define CONFPARSE_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "feature/nodelist/routerset.h"
+#include "lib/confmgt/confparse.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "lib/confmgt/unitparse.h"
+
+typedef struct test_struct_t {
+  uint32_t magic;
+  char *s;
+  char *fn;
+  int pos;
+  int i;
+  int deprecated_int;
+  uint64_t u64;
+  int interval;
+  int msec_interval;
+  uint64_t mem;
+  double dbl;
+  int boolean;
+  int autobool;
+  time_t time;
+  smartlist_t *csv;
+  int csv_interval;
+  config_line_t *lines;
+  config_line_t *mixed_lines;
+  routerset_t *routerset;
+  int hidden_int;
+  config_line_t *mixed_hidden_lines;
+
+  config_line_t *extra_lines;
+} test_struct_t;
+
+static test_struct_t test_struct_t_dummy;
+
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
+  VAR(#member, conftype, member, initvalue)
+#define OBSOLETE(varname)                       \
+  CONFIG_VAR_OBSOLETE(varname)
+
+static const config_var_t test_vars[] = {
+  V(s, STRING, "hello"),
+  V(fn, FILENAME, NULL),
+  V(pos, POSINT, NULL),
+  V(i, INT, "-10"),
+  V(deprecated_int, INT, "3"),
+  V(u64, UINT64, NULL),
+  V(interval, INTERVAL, "10 seconds"),
+  V(msec_interval, MSEC_INTERVAL, "150 msec"),
+  V(mem, MEMUNIT, "10 MB"),
+  V(dbl, DOUBLE, NULL),
+  V(boolean, BOOL, "0"),
+  V(autobool, AUTOBOOL, "auto"),
+  V(time, ISOTIME, NULL),
+  V(csv, CSV, NULL),
+  V(csv_interval, CSV_INTERVAL, "5 seconds"),
+  V(lines, LINELIST, NULL),
+  VAR("MixedLines", LINELIST_V, mixed_lines, NULL),
+  VAR("LineTypeA", LINELIST_S, mixed_lines, NULL),
+  VAR("LineTypeB", LINELIST_S, mixed_lines, NULL),
+  OBSOLETE("obsolete"),
+  {
+   .member = { .name = "routerset",
+               .type = CONFIG_TYPE_EXTENDED,
+               .type_def = &ROUTERSET_type_defn,
+               .offset = offsetof(test_struct_t, routerset),
+             },
+  },
+  VAR("__HiddenInt", POSINT, hidden_int, "0"),
+  VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL),
+  VAR("__HiddenLineA", LINELIST_S, mixed_hidden_lines, NULL),
+  VAR("VisibleLineB", LINELIST_S, mixed_hidden_lines, NULL),
+
+  END_OF_CONFIG_VARS,
+};
+
+static config_abbrev_t test_abbrevs[] = {
+  { "uint", "pos", 0, 0 },
+  { "float", "dbl", 0, 1 },
+  { NULL, NULL, 0, 0 }
+};
+
+static config_deprecation_t test_deprecation_notes[] = {
+  { "deprecated_int", "This integer is deprecated." },
+  { NULL, NULL }
+};
+
+static int
+test_validate_cb(void *old_options, void *options, void *default_options,
+                 int from_setconf, char **msg)
+{
+  (void)old_options;
+  (void)default_options;
+  (void)from_setconf;
+  (void)msg;
+  test_struct_t *ts = options;
+
+  if (ts->i == 0xbad) {
+    *msg = tor_strdup("bad value for i");
+    return -1;
+  }
+  return 0;
+}
+
+#define TEST_MAGIC 0x1337
+
+static const config_format_t test_fmt = {
+  sizeof(test_struct_t),
+  {
+   "test_struct_t",
+   TEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  test_abbrevs,
+  test_deprecation_notes,
+  test_vars,
+  test_validate_cb,
+  NULL,
+  NULL,
+  -1,
+};
+
+/* Make sure that config_init sets everything to the right defaults. */
+static void
+test_confparse_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_init(mgr, tst);
+
+  // Make sure that options are initialized right. */
+  tt_str_op(tst->s, OP_EQ, "hello");
+  tt_ptr_op(tst->fn, OP_EQ, NULL);
+  tt_int_op(tst->pos, OP_EQ, 0);
+  tt_int_op(tst->i, OP_EQ, -10);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, 0);
+  tt_int_op(tst->interval, OP_EQ, 10);
+  tt_int_op(tst->msec_interval, OP_EQ, 150);
+  tt_u64_op(tst->mem, OP_EQ, 10 * 1024 * 1024);
+  tt_double_op(tst->dbl, OP_LT, .0000000001);
+  tt_double_op(tst->dbl, OP_GT, -0.0000000001);
+  tt_int_op(tst->boolean, OP_EQ, 0);
+  tt_int_op(tst->autobool, OP_EQ, -1);
+  tt_i64_op(tst->time, OP_EQ, 0);
+  tt_ptr_op(tst->csv, OP_EQ, NULL);
+  tt_int_op(tst->csv_interval, OP_EQ, 5);
+  tt_ptr_op(tst->lines, OP_EQ, NULL);
+  tt_ptr_op(tst->mixed_lines, OP_EQ, NULL);
+  tt_int_op(tst->hidden_int, OP_EQ, 0);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static const char simple_settings[] =
+      "s this is a \n"
+      "fn /simple/test of the\n"
+      "uint 77\n" // this is an abbrev
+      "i 3\n"
+      "u64   1000000000000  \n"
+      "interval 5 minutes \n"
+      "msec_interval 5 minutes \n"
+      "mem 10\n"
+      "dbl 6.060842\n"
+      "BOOLEAN 1\n"
+      "aUtObOOl 0\n"
+      "time 2019-06-14 13:58:51\n"
+      "csv configuration, parsing  , system  \n"
+      "csv_interval 10 seconds, 5 seconds, 10 hours\n"
+      "lines hello\n"
+      "LINES world\n"
+      "linetypea i d\n"
+      "linetypeb i c\n"
+      "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+      "__hiddenint 11\n"
+      "__hiddenlineA XYZ\n"
+      "visiblelineB ABC\n";
+
+/* Return a configuration object set up from simple_settings above. */
+static test_struct_t *
+get_simple_config(const config_mgr_t *mgr)
+{
+  test_struct_t *result = NULL;
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(simple_settings, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  result = tst;
+  tst = NULL; // prevent free
+ done:
+  tor_free(msg);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  return result;
+}
+
+/* Make sure that config_assign can parse things. */
+static void
+test_confparse_assign_simple(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  tt_str_op(tst->s, OP_EQ, "this is a");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the");
+  tt_int_op(tst->pos, OP_EQ, 77);
+  tt_int_op(tst->i, OP_EQ, 3);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, UINT64_C(1000000000000));
+  tt_int_op(tst->interval, OP_EQ, 5 * 60);
+  tt_int_op(tst->msec_interval, OP_EQ, 5 * 60 * 1000);
+  tt_u64_op(tst->mem, OP_EQ, 10);
+  tt_double_op(tst->dbl, OP_LT, 6.060843);
+  tt_double_op(tst->dbl, OP_GT, 6.060841);
+  tt_int_op(tst->boolean, OP_EQ, 1);
+  tt_int_op(tst->autobool, OP_EQ, 0);
+  tt_i64_op(tst->time, OP_EQ, 1560520731);
+  tt_ptr_op(tst->csv, OP_NE, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 3);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "configuration");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "parsing");
+  tt_str_op(smartlist_get(tst->csv, 2), OP_EQ, "system");
+  tt_int_op(tst->csv_interval, OP_EQ, 10);
+  tt_int_op(tst->hidden_int, OP_EQ, 11);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(!tst->lines->next->next);
+
+  tt_assert(tst->mixed_lines);
+  tt_str_op(tst->mixed_lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(tst->mixed_lines->value, OP_EQ, "i d");
+  tt_assert(tst->mixed_lines->next);
+  tt_str_op(tst->mixed_lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(tst->mixed_lines->next->value, OP_EQ, "i c");
+  tt_assert(!tst->mixed_lines->next->next);
+
+  tt_assert(tst->mixed_hidden_lines);
+  tt_str_op(tst->mixed_hidden_lines->key, OP_EQ, "__HiddenLineA");
+  tt_str_op(tst->mixed_hidden_lines->value, OP_EQ, "XYZ");
+  tt_assert(tst->mixed_hidden_lines->next);
+  tt_str_op(tst->mixed_hidden_lines->next->key, OP_EQ, "VisibleLineB");
+  tt_str_op(tst->mixed_hidden_lines->next->value, OP_EQ, "ABC");
+  tt_assert(!tst->mixed_hidden_lines->next->next);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an obsolete option, and make sure we get a warning. */
+static void
+test_confparse_assign_obsolete(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("obsolete option here",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("Skipping obsolete configuration option");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an deprecated option, and make sure we get a warning
+ * but the assignment works anyway. */
+static void
+test_confparse_assign_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("deprecated_int 7",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("This integer is deprecated.");
+
+  tt_int_op(tst->deprecated_int, OP_EQ, 7);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to re-assign an option name that has been depreacted in favor of
+ * another. */
+static void
+test_confparse_assign_replaced(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("float 1000\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("use 'dbl' instead.");
+
+  tt_double_op(tst->dbl, OP_GT, 999.999);
+  tt_double_op(tst->dbl, OP_LT, 1000.001);
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a linelist value with no option. */
+static void
+test_confparse_assign_emptystring(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("lines\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("has no value");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a the same option twice; make sure we get a warning. */
+static void
+test_confparse_assign_twice(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("pos 10\n"
+                           "pos 99\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("used more than once");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+typedef struct badval_test_t {
+  const char *cfg;
+  const char *expect_msg;
+} badval_test_t;
+
+/* Try to set an option and make sure that we get a failure and an expected
+ * warning. */
+static void
+test_confparse_assign_badval(void *arg)
+{
+  const badval_test_t *bt = arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(bt->cfg, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  if (! strstr(msg, bt->expect_msg)) {
+    TT_DIE(("'%s' did not contain '%s'" , msg, bt->expect_msg));
+  }
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Various arguments for badval test.
+ *
+ * Note that the expected warnings here are _very_ truncated, since we
+ * are writing these tests before a refactoring that we expect will
+ * change them.
+ */
+static const badval_test_t bv_notint = { "pos X\n", "malformed" };
+static const badval_test_t bv_negint = { "pos -10\n", "out of bounds" };
+static const badval_test_t bv_badu64 = { "u64 u64\n", "malformed" };
+static const badval_test_t bv_dbl1 = { "dbl xxx\n", "Could not convert" };
+static const badval_test_t bv_dbl2 = { "dbl 1.0 xx\n", "Could not convert" };
+static const badval_test_t bv_dbl3 = {
+   "dbl 1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl4 = {
+   "dbl 1e1000\n", "too large to express" };
+static const badval_test_t bv_dbl5 = {
+   "dbl -1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl6 = {
+   "dbl -1e1000\n", "too large to express" };
+static const badval_test_t bv_badcsvi1 =
+  { "csv_interval 10 wl\n", "malformed" };
+static const badval_test_t bv_badcsvi2 =
+  { "csv_interval cl,10\n", "malformed" };
+static const badval_test_t bv_nonoption = { "fnord 10\n", "Unknown option" };
+static const badval_test_t bv_badmem = { "mem 3 trits\n", "malformed" };
+static const badval_test_t bv_badbool = { "boolean 7\n", "Unrecognized value"};
+static const badval_test_t bv_badabool =
+  { "autobool 7\n", "Unrecognized value" };
+static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" };
+static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" };
+static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" };
+static const badval_test_t bv_big_interval =
+  { "interval 1000 months", "too large" };
+
+/* Try config_dump(), and make sure it behaves correctly */
+static void
+test_confparse_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  char *dumped = NULL;
+
+  /* Minimal version. */
+  dumped = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  /* commented */
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 1);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "# deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+ done:
+  config_free(mgr, tst);
+  tor_free(dumped);
+  config_mgr_free(mgr);
+}
+
+/* Try confparse_reset_line(), and make sure it behaves correctly */
+static void
+test_confparse_reset(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  config_reset_line(mgr, tst, "interval", 0);
+  tt_int_op(tst->interval, OP_EQ, 0);
+
+  config_reset_line(mgr, tst, "interval", 1);
+  tt_int_op(tst->interval, OP_EQ, 10);
+
+  tt_ptr_op(tst->routerset, OP_NE, NULL);
+  config_reset_line(mgr, tst, "routerset", 0);
+  tt_ptr_op(tst->routerset, OP_EQ, NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, and make sure
+ * it behaves correctly. */
+static void
+test_confparse_reassign(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *rs = NULL;
+
+  int r = config_get_lines(
+         "s eleven\n"
+         "i 12\n"
+         "lines 13\n"
+         "csv 14,15\n"
+         "routerset 127.0.0.1\n",
+         &lines, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); // unchanged
+  tt_int_op(tst->pos, OP_EQ, 77); // unchanged
+  tt_int_op(tst->i, OP_EQ, 12);
+  tt_ptr_op(tst->lines, OP_NE, NULL);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "13");
+  tt_ptr_op(tst->lines->next, OP_EQ, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 2);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "14");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "15");
+
+  rs = routerset_to_string(tst->routerset);
+  tt_str_op(rs, OP_EQ, "127.0.0.1");
+
+  // Try again with the CLEAR_FIRST and USE_DEFAULTS flags
+  r = config_assign(mgr, tst, lines,
+                    CAL_CLEAR_FIRST|CAL_USE_DEFAULTS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  // tt_ptr_op(tst->fn, OP_EQ, NULL); //XXXX why is this not cleared?
+  // tt_int_op(tst->pos, OP_EQ, 0); //XXXX why is this not cleared?
+  tt_int_op(tst->i, OP_EQ, 12);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  tor_free(rs);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, using the +foo
+ * linelist-extending syntax. */
+static void
+test_confparse_reassign_extend(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  int r = config_get_lines(
+         "+lines 13\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(tst->lines->next->next);
+  tt_str_op(tst->lines->next->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->next->value, OP_EQ, "13");
+  tt_assert(tst->lines->next->next->next == NULL);
+  config_free_lines(lines);
+
+  r = config_get_lines(
+         "/lines\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+  config_free_lines(lines);
+
+  config_free(mgr, tst);
+  tst = get_simple_config(mgr);
+  r = config_get_lines(
+         "/lines away!\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Test out confparse_get_assigned(). */
+static void
+test_confparse_get_assigned(void *arg)
+{
+  (void)arg;
+
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+
+  lines = config_get_assigned_option(mgr, tst, "I", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "i");
+  tt_str_op(lines->value, OP_EQ, "3");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "this is a");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "obsolete", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "nonesuch", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "mixedlines", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(lines->value, OP_EQ, "i d");
+  tt_assert(lines->next);
+  tt_str_op(lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->next->value, OP_EQ, "i c");
+  tt_assert(lines->next->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "linetypeb", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->value, OP_EQ, "i c");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  tor_free(tst->s);
+  tst->s = tor_strdup("Hello\nWorld");
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "\"Hello\\nWorld\"");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  config_mgr_free(mgr);
+}
+
+/* Another variant, which accepts and stores unrecognized lines.*/
+#define ETEST_MAGIC 13371337
+
+static struct_member_t extra = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(test_struct_t, extra_lines),
+};
+
+static config_format_t etest_fmt = {
+  sizeof(test_struct_t),
+  {
+   "test_struct_t (with extra lines)",
+   ETEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  test_abbrevs,
+  test_deprecation_notes,
+  test_vars,
+  test_validate_cb,
+  NULL,
+  &extra,
+  -1,
+};
+
+/* Try out the feature where we can store unrecognized lines and dump them
+ * again.  (State files use this.) */
+static void
+test_confparse_extra_lines(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&etest_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *dump = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(
+      "unknotty addita\n"
+      "pos 99\n"
+      "wombat knish\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->extra_lines);
+
+  dump = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dump, OP_EQ,
+      "pos 99\n"
+      "unknotty addita\n"
+      "wombat knish\n");
+
+ done:
+  tor_free(msg);
+  tor_free(dump);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_unitparse(void *args)
+{
+  (void)args;
+  /* spot-check a few memunit values. */
+  int ok = 3;
+  tt_u64_op(config_parse_memunit("100 MB", &ok), OP_EQ, 100<<20);
+  tt_assert(ok);
+  tt_u64_op(config_parse_memunit("100 TB", &ok), OP_EQ, UINT64_C(100)<<40);
+  tt_assert(ok);
+  // This is a floating-point value, but note that 1.5 can be represented
+  // precisely.
+  tt_u64_op(config_parse_memunit("1.5 MB", &ok), OP_EQ, 3<<19);
+  tt_assert(ok);
+
+  /* Try some good intervals and msec intervals */
+  tt_int_op(config_parse_interval("2 days", &ok), OP_EQ, 48*3600);
+  tt_assert(ok);
+  tt_int_op(config_parse_interval("1.5 hour", &ok), OP_EQ, 5400);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("1 minute", &ok), OP_EQ, 60);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("2 days", &ok), OP_EQ, 48*3600*1000);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("10 msec", &ok), OP_EQ, 10);
+  tt_assert(ok);
+
+  /* Try a couple of unitless values. */
+  tt_int_op(config_parse_interval("10", &ok), OP_EQ, 10);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("15.0", &ok), OP_EQ, 15);
+  tt_assert(ok);
+
+  /* u64 overflow */
+  /* XXXX our implementation does not currently detect this. See bug 30920. */
+  /*
+  tt_u64_op(config_parse_memunit("20000000 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  */
+
+  /* i32 overflow */
+  tt_int_op(config_parse_interval("1000 months", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("4 weeks", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+
+  /* bad units */
+  tt_u64_op(config_parse_memunit("7 nybbles", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // XXXX these next two should return -1 according to the documentation.
+  tt_int_op(config_parse_interval("7 cowznofski", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("1 kalpa", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+ done:
+  ;
+}
+
+static void
+test_confparse_check_ok_fail(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  tst->pos = -10;
+  tt_assert(! config_check_ok(mgr, tst, LOG_INFO));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_vars(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_vars(mgr);
+  smartlist_t *varnames = smartlist_new();
+  char *joined = NULL;
+
+  tt_assert(vars);
+  SMARTLIST_FOREACH(vars, config_var_t *, cv,
+                    smartlist_add(varnames, (void*)cv->member.name));
+  smartlist_sort_strings(varnames);
+  joined = smartlist_join_strings(varnames, "::", 0, NULL);
+  tt_str_op(joined, OP_EQ,
+            "LineTypeA::"
+            "LineTypeB::"
+            "MixedHiddenLines::"
+            "MixedLines::"
+            "VisibleLineB::"
+            "__HiddenInt::"
+            "__HiddenLineA::"
+            "autobool::"
+            "boolean::"
+            "csv::"
+            "csv_interval::"
+            "dbl::"
+            "deprecated_int::"
+            "fn::"
+            "i::"
+            "interval::"
+            "lines::"
+            "mem::"
+            "msec_interval::"
+            "obsolete::"
+            "pos::"
+            "routerset::"
+            "s::"
+            "time::"
+            "u64");
+
+ done:
+  tor_free(joined);
+  smartlist_free(varnames);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_deprecated_vars(mgr);
+  char *joined = NULL;
+
+  tt_assert(vars);
+  smartlist_sort_strings(vars);
+  joined = smartlist_join_strings(vars, "::", 0, NULL);
+
+  tt_str_op(joined, OP_EQ, "deprecated_int");
+
+ done:
+  tor_free(joined);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_find_option_name(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+
+  // exact match
+  tt_str_op(config_find_option_name(mgr, "u64"), OP_EQ, "u64");
+  // case-insensitive match
+  tt_str_op(config_find_option_name(mgr, "S"), OP_EQ, "s");
+  tt_str_op(config_find_option_name(mgr, "linetypea"), OP_EQ, "LineTypeA");
+  // prefix match
+  tt_str_op(config_find_option_name(mgr, "deprec"), OP_EQ, "deprecated_int");
+  // explicit abbreviation
+  tt_str_op(config_find_option_name(mgr, "uint"), OP_EQ, "pos");
+  tt_str_op(config_find_option_name(mgr, "UINT"), OP_EQ, "pos");
+  // no match
+  tt_ptr_op(config_find_option_name(mgr, "absent"), OP_EQ, NULL);
+
+ done:
+  config_mgr_free(mgr);
+}
+
+#define CONFPARSE_TEST(name, flags)                          \
+  { #name, test_confparse_ ## name, flags, NULL, NULL }
+
+#define BADVAL_TEST(name)                               \
+  { "badval_" #name, test_confparse_assign_badval, 0,   \
+      &passthrough_setup, (void*)&bv_ ## name }
+
+struct testcase_t confparse_tests[] = {
+  CONFPARSE_TEST(init, 0),
+  CONFPARSE_TEST(assign_simple, 0),
+  CONFPARSE_TEST(assign_obsolete, 0),
+  CONFPARSE_TEST(assign_deprecated, 0),
+  CONFPARSE_TEST(assign_replaced, 0),
+  CONFPARSE_TEST(assign_emptystring, 0),
+  CONFPARSE_TEST(assign_twice, 0),
+  BADVAL_TEST(notint),
+  BADVAL_TEST(negint),
+  BADVAL_TEST(badu64),
+  BADVAL_TEST(dbl1),
+  BADVAL_TEST(dbl2),
+  BADVAL_TEST(dbl3),
+  BADVAL_TEST(dbl4),
+  BADVAL_TEST(dbl5),
+  BADVAL_TEST(dbl6),
+  BADVAL_TEST(badcsvi1),
+  BADVAL_TEST(badcsvi2),
+  BADVAL_TEST(nonoption),
+  BADVAL_TEST(badmem),
+  BADVAL_TEST(badbool),
+  BADVAL_TEST(badabool),
+  BADVAL_TEST(badtime),
+  BADVAL_TEST(virt),
+  BADVAL_TEST(rs),
+  BADVAL_TEST(big_interval),
+  CONFPARSE_TEST(dump, 0),
+  CONFPARSE_TEST(reset, 0),
+  CONFPARSE_TEST(reassign, 0),
+  CONFPARSE_TEST(reassign_extend, 0),
+  CONFPARSE_TEST(get_assigned, 0),
+  CONFPARSE_TEST(extra_lines, 0),
+  CONFPARSE_TEST(unitparse, 0),
+  CONFPARSE_TEST(check_ok_fail, 0),
+  CONFPARSE_TEST(list_vars, 0),
+  CONFPARSE_TEST(list_deprecated, 0),
+  CONFPARSE_TEST(find_option_name, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_connection.h b/src/test/test_connection.h
index 47a5599e5f..40121e6d38 100644
--- a/src/test/test_connection.h
+++ b/src/test/test_connection.h
@@ -1,6 +1,9 @@
 /* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_CONNECTION_H
+#define TOR_TEST_CONNECTION_H
+
 /** Some constants used by test_connection and helpers */
 #define TEST_CONN_FAMILY        (AF_INET)
 #define TEST_CONN_ADDRESS       "127.0.0.1"
@@ -11,3 +14,4 @@
 void test_conn_lookup_addr_helper(const char *address,
                                   int family, tor_addr_t *addr);
 
+#endif /* !defined(TOR_TEST_CONNECTION_H) */
diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c
index 682ba5b970..7c4c92ea42 100644
--- a/src/test/test_consdiff.c
+++ b/src/test/test_consdiff.c
@@ -14,6 +14,39 @@
 #define tt_str_eq_line(a,b) \
   tt_assert(line_str_eq((b),(a)))
 
+static int
+consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area)
+{
+  size_t len = strlen(s);
+  return consensus_split_lines(out, s, len, area);
+}
+
+static int
+consensus_compute_digest_(const char *cons,
+                          consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
+static int
+consensus_compute_digest_as_signed_(const char *cons,
+                                    consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest_as_signed(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
 static void
 test_consdiff_smartlist_slice(void *arg)
 {
@@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg)
 
   /* Create a regular smartlist. */
   (void)arg;
-  consensus_split_lines(sl, "a\nd\nc\na\nb\n", area);
+  consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area);
 
   /* See that smartlist_slice_string_pos respects the bounds of the slice. */
   sls = smartlist_slice(sl, 2, 5);
@@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg)
   int e_lengths2[] = { 0, 1, 1, 2, 3, 4 };
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area);
-  consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area);
+  consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area);
+  consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area);
-  consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area);
-  consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area);
-  consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area);
+  consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area);
+  consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area);
+  consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area);
+  consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area);
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
   sls3 = smartlist_slice(sl3, 0, -1);
@@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\nb\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   /* Length of sls1 is 0. */
   sls1 = smartlist_slice(sl1, 0, 0);
@@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\na\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\na\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg)
   tt_assert(!bitarray_is_set(changed2, 3));
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "a\nb\na\nb\n", area);
+  consensus_split_lines_(sl2, "a\nb\na\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg)
   bitarray_clear(changed1, 3);
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "b\nb\nb\nb\n", area);
+  consensus_split_lines_(sl2, "b\nb\nb\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg)
   /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
-  consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(7, OP_EQ, smartlist_len(diff));
@@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg)
 
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "B\n", area);
-  consensus_split_lines(cons2, "A\nB\n", area);
+  consensus_split_lines_(cons1, "B\n", area);
+  consensus_split_lines_(cons2, "A\nB\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(3, OP_EQ, smartlist_len(diff));
@@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg)
   diff = smartlist_new();
   setup_capture_of_logs(LOG_WARN);
 
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
 
   /* Command without range. */
   smartlist_add_linecpy(diff, area, "a");
@@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_clear(diff);
 
   /* Test appending text, 'a'. */
-  consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
+  consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(8, OP_EQ, smartlist_len(cons2));
@@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test deleting text, 'd'. */
-  consensus_split_lines(diff, "4d\n1,2d\n", area);
+  consensus_split_lines_(diff, "4d\n1,2d\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(2, OP_EQ, smartlist_len(cons2));
@@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test changing text, 'c'. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(5, OP_EQ, smartlist_len(cons2));
@@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test 'a', 'd' and 'c' together. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(6, OP_EQ, smartlist_len(cons2));
@@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg)
       );
 
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons2_str, &digests2));
+      consensus_compute_digest_(cons2_str, &digests2));
 
-  consensus_split_lines(cons1, cons1_str, area);
-  consensus_split_lines(cons2, cons2_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
+  consensus_split_lines_(cons2, cons2_str, area);
 
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_EQ, diff);
@@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   smartlist_clear(cons1);
-  consensus_split_lines(cons1, cons1_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(11, OP_EQ, smartlist_len(diff));
@@ -991,8 +1024,8 @@ test_consdiff_apply_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons1_str, &digests1));
-  consensus_split_lines(cons1, cons1_str, area);
+      consensus_compute_digest_(cons1_str, &digests1));
+  consensus_split_lines_(cons1, cons1_str, area);
 
   /* diff doesn't have enough lines. */
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
@@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = {
   CONSDIFF_LEGACY(apply_diff),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c
index 254a5ba5d0..74226b8c52 100644
--- a/src/test/test_consdiffmgr.c
+++ b/src/test/test_consdiffmgr.c
@@ -21,6 +21,23 @@
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
+static char *
+consensus_diff_apply_(const char *c, const char *d)
+{
+  size_t c_len = strlen(c);
+  size_t d_len = strlen(d);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  char *c_tmp = tor_memdup(c, c_len);
+  char *d_tmp = tor_memdup(d, d_len);
+  char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len);
+  tor_free(c_tmp);
+  tor_free(d_tmp);
+  return result;
+}
+
 // ============================== Setup/teardown the consdiffmgr
 // These functions get run before/after each test in this module
 
@@ -153,7 +170,8 @@ lookup_diff_from(consensus_cache_entry_t **out,
                  const char *str1)
 {
   uint8_t digest[DIGEST256_LEN];
-  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+  if (router_get_networkstatus_v3_sha3_as_signed(digest,
+                                                 str1, strlen(str1))<0) {
     TT_FAIL(("Unable to compute sha3-as-signed"));
     return CONSDIFF_NOT_FOUND;
   }
@@ -175,14 +193,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav,
 
   consensus_cache_entry_incref(ent);
   size_t size;
-  char *diff_string = NULL;
-  int r = uncompress_or_copy(&diff_string, &size, ent);
+  const char *diff_string = NULL;
+  char *diff_owned = NULL;
+  int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent);
   consensus_cache_entry_decref(ent);
   if (diff_string == NULL || r < 0)
     return -1;
 
-  char *applied = consensus_diff_apply(str1, diff_string);
-  tor_free(diff_string);
+  char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size);
+  tor_free(diff_owned);
   if (applied == NULL)
     return -1;
 
@@ -282,7 +301,8 @@ test_consdiffmgr_add(void *arg)
   (void) arg;
   time_t now = approx_time();
 
-  char *body = NULL;
+  const char *body = NULL;
+  char *body_owned = NULL;
 
   consensus_cache_entry_t *ent = NULL;
   networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
@@ -324,7 +344,7 @@ test_consdiffmgr_add(void *arg)
   tt_assert(ent);
   consensus_cache_entry_incref(ent);
   size_t s;
-  r = uncompress_or_copy(&body, &s, ent);
+  r = uncompress_or_set_ptr(&body, &s, &body_owned, ent);
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(s, OP_EQ, 4);
   tt_mem_op(body, OP_EQ, "quux", 4);
@@ -337,7 +357,7 @@ test_consdiffmgr_add(void *arg)
   networkstatus_vote_free(ns_tmp);
   teardown_capture_of_logs();
   consensus_cache_entry_decref(ent);
-  tor_free(body);
+  tor_free(body_owned);
 }
 
 static void
@@ -370,7 +390,8 @@ test_consdiffmgr_make_diffs(void *arg)
   ns = fake_ns_new(FLAV_MICRODESC, now-3600);
   md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
   r = consdiffmgr_add_consensus(md_ns_body, ns);
-  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body,
+                                             strlen(md_ns_body));
   networkstatus_vote_free(ns);
   tt_int_op(r, OP_EQ, 0);
 
@@ -414,7 +435,7 @@ test_consdiffmgr_make_diffs(void *arg)
   r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
   tt_int_op(r, OP_EQ, 0);
   diff_text = tor_memdup_nulterm(diff_body, diff_size);
-  applied = consensus_diff_apply(md_ns_body, diff_text);
+  applied = consensus_diff_apply_(md_ns_body, diff_text);
   tt_assert(applied);
   tt_str_op(applied, OP_EQ, md_ns_body_2);
 
diff --git a/src/test/test_containers.c b/src/test/test_containers.c
index aedd2f7a89..67ba457975 100644
--- a/src/test/test_containers.c
+++ b/src/test/test_containers.c
@@ -96,6 +96,30 @@ test_container_smartlist_basic(void *arg)
   tor_free(v555);
 }
 
+/** Test SMARTLIST_FOREACH_REVERSE_BEGIN loop macro */
+static void
+test_container_smartlist_foreach_reverse(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  int i;
+
+  (void) arg;
+
+  /* Add integers to smartlist in increasing order */
+  for (i=0;i<100;i++) {
+    smartlist_add(sl, (void*)(uintptr_t)i);
+  }
+
+  /* Pop them out in reverse and test their value */
+  SMARTLIST_FOREACH_REVERSE_BEGIN(sl, void*, k) {
+    i--;
+    tt_ptr_op(k, OP_EQ, (void*)(uintptr_t)i);
+  } SMARTLIST_FOREACH_END(k);
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for smartlist-of-strings functionality. */
 static void
 test_container_smartlist_strings(void *arg)
@@ -582,6 +606,66 @@ test_container_smartlist_ints_eq(void *arg)
   smartlist_free(sl2);
 }
 
+static void
+test_container_smartlist_grow(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  int i;
+  const char *s[] = { "first", "2nd", "3rd" };
+
+  /* case 1: starting from empty. */
+  smartlist_grow(sl, 10);
+  tt_int_op(10, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 10; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 2: starting with a few elements, probably not reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 5);
+  tt_int_op(5, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  tt_ptr_op(smartlist_get(sl, 3), OP_EQ, NULL);
+  tt_ptr_op(smartlist_get(sl, 4), OP_EQ, NULL);
+
+  /* case 3: starting with a few elements, but reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 100);
+  tt_int_op(100, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  for (i = 3; i < 100; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 4: shrinking doesn't happen. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 1);
+  tt_int_op(3, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for bitarray code */
 static void
 test_container_bitarray(void *arg)
@@ -922,6 +1006,10 @@ test_container_smartlist_remove(void *arg)
   tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]);
   tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]);
   tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]);
+  /* Ordinary code should never look at this pointer; we're doing it here
+   * to make sure that we really cleared the pointer we removed.
+   */
+  tt_ptr_op(sl->list[4], OP_EQ, NULL);
 
  done:
   smartlist_free(sl);
@@ -1281,12 +1369,14 @@ test_container_smartlist_strings_eq(void *arg)
 struct testcase_t container_tests[] = {
   CONTAINER_LEGACY(smartlist_basic),
   CONTAINER_LEGACY(smartlist_strings),
+  CONTAINER_LEGACY(smartlist_foreach_reverse),
   CONTAINER_LEGACY(smartlist_overlap),
   CONTAINER_LEGACY(smartlist_digests),
   CONTAINER_LEGACY(smartlist_join),
   CONTAINER_LEGACY(smartlist_pos),
   CONTAINER(smartlist_remove, 0),
   CONTAINER(smartlist_ints_eq, 0),
+  CONTAINER(smartlist_grow, 0),
   CONTAINER_LEGACY(bitarray),
   CONTAINER_LEGACY(digestset),
   CONTAINER_LEGACY(strmap),
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index 5b406e159b..55eb79e448 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -1,11 +1,15 @@
 /* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define CONTROL_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "feature/client/bridges.h"
 #include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hs/hs_common.h"
 #include "feature/nodelist/networkstatus.h"
@@ -15,48 +19,255 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "lib/net/resolve.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
 #include "feature/dirclient/download_status_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/node_st.h"
 
+typedef struct {
+  const char *input;
+  const char *expected_parse;
+  const char *expected_error;
+} parser_testcase_t;
+
+typedef struct {
+  const control_cmd_syntax_t *syntax;
+  size_t n_testcases;
+  const parser_testcase_t *testcases;
+} parse_test_params_t;
+
+static char *
+control_cmd_dump_args(const control_cmd_args_t *result)
+{
+  buf_t *buf = buf_new();
+  buf_add_string(buf, "{ args=[");
+  if (result->args) {
+    if (smartlist_len(result->args)) {
+        buf_add_string(buf, " ");
+    }
+    SMARTLIST_FOREACH_BEGIN(result->args, const char *, s) {
+      const bool last = (s_sl_idx == smartlist_len(result->args)-1);
+      buf_add_printf(buf, "%s%s ",
+                     escaped(s),
+                     last ? "" : ",");
+    } SMARTLIST_FOREACH_END(s);
+  }
+  buf_add_string(buf, "]");
+  if (result->cmddata) {
+    buf_add_string(buf, ", obj=");
+    buf_add_string(buf, escaped(result->cmddata));
+  }
+  if (result->kwargs) {
+    buf_add_string(buf, ", { ");
+    const config_line_t *line;
+    for (line = result->kwargs; line; line = line->next) {
+      const bool last = (line->next == NULL);
+      buf_add_printf(buf, "%s=%s%s ", line->key, escaped(line->value),
+                     last ? "" : ",");
+    }
+    buf_add_string(buf, "}");
+  }
+  buf_add_string(buf, " }");
+
+  char *encoded = buf_extract(buf, NULL);
+  buf_free(buf);
+  return encoded;
+}
+
+static void
+test_controller_parse_cmd(void *arg)
+{
+  const parse_test_params_t *params = arg;
+  control_cmd_args_t *result = NULL;
+  char *error = NULL;
+  char *encoded = NULL;
+
+  for (size_t i = 0; i < params->n_testcases; ++i) {
+    const parser_testcase_t *t = &params->testcases[i];
+    result = control_cmd_parse_args("EXAMPLE",
+                                    params->syntax,
+                                    strlen(t->input),
+                                    t->input,
+                                    &error);
+    // A valid test should expect exactly one parse or error.
+    tt_int_op((t->expected_parse == NULL), OP_NE,
+              (t->expected_error == NULL));
+    // We get a result or an error, not both.
+    tt_int_op((result == NULL), OP_EQ, (error != NULL));
+    // We got the one we expected.
+    tt_int_op((result == NULL), OP_EQ, (t->expected_parse == NULL));
+
+    if (result) {
+      encoded = control_cmd_dump_args(result);
+      tt_str_op(encoded, OP_EQ, t->expected_parse);
+    } else {
+      tt_str_op(error, OP_EQ, t->expected_error);
+    }
+
+    tor_free(error);
+    tor_free(encoded);
+    control_cmd_args_free(result);
+  }
+
+ done:
+  tor_free(error);
+  tor_free(encoded);
+  control_cmd_args_free(result);
+}
+
+#define OK(inp, out) \
+  { inp "\r\n", out, NULL }
+#define ERR(inp, err) \
+  { inp "\r\n", NULL, err }
+
+#define TESTPARAMS(syntax, array)                \
+  { &syntax,                                     \
+      ARRAY_LENGTH(array),                       \
+      array }
+
+static const parser_testcase_t one_to_three_tests[] = {
+   ERR("", "Need at least 1 argument(s)"),
+   ERR("   \t", "Need at least 1 argument(s)"),
+   OK("hello", "{ args=[ \"hello\" ] }"),
+   OK("hello world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world      ", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello there world", "{ args=[ \"hello\", \"there\", \"world\" ] }"),
+   ERR("why hello there world", "Cannot accept more than 3 argument(s)"),
+   ERR("hello\r\nworld.\r\n.", "Unexpected body"),
+};
+
+static const control_cmd_syntax_t one_to_three_syntax = {
+   .min_args=1, .max_args=3
+};
+
+static const parse_test_params_t parse_one_to_three_params =
+  TESTPARAMS( one_to_three_syntax, one_to_three_tests );
+
+// =
+static const parser_testcase_t no_args_one_obj_tests[] = {
+  ERR("Hi there!\r\n.", "Cannot accept more than 0 argument(s)"),
+  ERR("", "Empty body"),
+  OK("\r\n", "{ args=[], obj=\"\\n\" }"),
+  OK("\r\nHello world\r\n", "{ args=[], obj=\"Hello world\\n\\n\" }"),
+  OK("\r\nHello\r\nworld\r\n", "{ args=[], obj=\"Hello\\nworld\\n\\n\" }"),
+  OK("\r\nHello\r\n..\r\nworld\r\n",
+     "{ args=[], obj=\"Hello\\n.\\nworld\\n\\n\" }"),
+};
+static const control_cmd_syntax_t no_args_one_obj_syntax = {
+   .min_args=0, .max_args=0,
+   .want_cmddata=true,
+};
+static const parse_test_params_t parse_no_args_one_obj_params =
+  TESTPARAMS( no_args_one_obj_syntax, no_args_one_obj_tests );
+
+static const parser_testcase_t no_args_kwargs_tests[] = {
+  OK("", "{ args=[] }"),
+  OK(" ", "{ args=[] }"),
+  OK("hello there=world", "{ args=[], { hello=\"\", there=\"world\" } }"),
+  OK("hello there=world today",
+     "{ args=[], { hello=\"\", there=\"world\", today=\"\" } }"),
+  ERR("=Foo", "Cannot parse keyword argument(s)"),
+};
+static const control_cmd_syntax_t no_args_kwargs_syntax = {
+   .min_args=0, .max_args=0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_VALS
+};
+static const parse_test_params_t parse_no_args_kwargs_params =
+  TESTPARAMS( no_args_kwargs_syntax, no_args_kwargs_tests );
+
+static const char *one_arg_kwargs_allow_keywords[] = {
+  "Hello", "world", NULL
+};
+static const parser_testcase_t one_arg_kwargs_tests[] = {
+  ERR("", "Need at least 1 argument(s)"),
+  OK("Hi", "{ args=[ \"Hi\" ] }"),
+  ERR("hello there=world", "Unrecognized keyword argument \"there\""),
+  OK("Hi HELLO=foo", "{ args=[ \"Hi\" ], { HELLO=\"foo\" } }"),
+  OK("Hi world=\"bar baz\" hello  ",
+     "{ args=[ \"Hi\" ], { world=\"bar baz\", hello=\"\" } }"),
+};
+static const control_cmd_syntax_t one_arg_kwargs_syntax = {
+   .min_args=1, .max_args=1,
+   .accept_keywords=true,
+   .allowed_keywords=one_arg_kwargs_allow_keywords,
+   .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+static const parse_test_params_t parse_one_arg_kwargs_params =
+  TESTPARAMS( one_arg_kwargs_syntax, one_arg_kwargs_tests );
+
+static char *reply_str = NULL;
+/* Mock for control_write_reply that copies the string for inspection
+ * by tests */
+static void
+mock_control_write_reply(control_connection_t *conn, int code, int c,
+                                const char *s)
+{
+  (void)conn;
+  (void)code;
+  (void)c;
+  tor_free(reply_str);
+  reply_str = tor_strdup(s);
+}
+
 static void
 test_add_onion_helper_keyarg_v3(void *arg)
 {
   int ret, hs_version;
   add_onion_secret_key_t pk;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   const char *key_new_alg = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit ED25519-V3 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
+  /* Test "BEST" key generation (Assumes BEST = ED25519-V3). */
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, NULL);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
+  tt_assert(key_new_blob);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
+
   /* Test discarding the private key. */
+  tor_free(reply_str);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
@@ -76,9 +287,10 @@ test_add_onion_helper_keyarg_v3(void *arg)
 
     tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk);
     tt_assert(key_blob);
+    tor_free(reply_str);
     ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg,
                                   &key_new_blob, &pk, &hs_version,
-                                  &err_msg);
+                                  NULL);
     tor_free(key_blob);
     tt_int_op(ret, OP_EQ, 0);
     tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
@@ -86,7 +298,7 @@ test_add_onion_helper_keyarg_v3(void *arg)
     tt_mem_op(pk.v3, OP_EQ, hex_sk, 64);
     tt_ptr_op(key_new_alg, OP_EQ, NULL);
     tt_ptr_op(key_new_blob, OP_EQ, NULL);
-    tt_ptr_op(err_msg, OP_EQ, NULL);
+    tt_ptr_op(reply_str, OP_EQ, NULL);
     tor_free(pk.v3); pk.v3 = NULL;
     tor_free(key_new_blob);
   }
@@ -94,7 +306,8 @@ test_add_onion_helper_keyarg_v3(void *arg)
  done:
   tor_free(pk.v3);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -105,72 +318,61 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_t *pk1 = NULL;
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   char *encoded = NULL;
   char *arg_str = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit RSA1024 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
-
-  /* Test "BEST" key generation (Assumes BEST = RSA1024). */
-  crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
-  tt_assert(pk.v2);
-  tt_str_op(key_new_alg, OP_EQ, "RSA1024");
-  tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test discarding the private key. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+  ret = add_onion_helper_keyarg("NEW:RSA1024", 1, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test generating a invalid key type. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a RSA1024 key. */
-  tor_free(err_msg);
+  tor_free(reply_str);
   pk1 = pk_generate(0);
   tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded));
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0);
 
   /* Test loading a invalid key type. */
@@ -179,36 +381,37 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_asprintf(&arg_str, "RSA512:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a invalid key. */
   tor_free(arg_str);
   crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(err_msg);
+  tor_free(reply_str);
   encoded[strlen(encoded)/2] = '\0';
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
  done:
   crypto_pk_free(pk1);
   crypto_pk_free(pk.v2);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
   tor_free(encoded);
   tor_free(arg_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -362,49 +565,52 @@ static void
 test_add_onion_helper_clientauth(void *arg)
 {
   rend_authorized_client_t *client = NULL;
-  char *err_msg = NULL;
   int created = 0;
 
   (void)arg;
 
+  MOCK(control_write_reply, mock_control_write_reply);
   /* Test "ClientName" only. */
-  client = add_onion_helper_clientauth("alice", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice", &created, NULL);
   tt_assert(client);
   tt_assert(created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test "ClientName:Blob" */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B",
-                                       &created, &err_msg);
+                                       &created, NULL);
   tt_assert(client);
   tt_assert(!created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test invalid client names */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("no*asterisks*allowed", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid auth cookie */
-  client = add_onion_helper_clientauth("alice:12345", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice:12345", &created, NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid syntax */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
  done:
   rend_authorized_client_free(client);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 /* Mocks and data/variables used for GETINFO download status tests */
@@ -1543,7 +1749,7 @@ test_current_time(void *arg)
 static size_t n_nodelist_get_list = 0;
 static smartlist_t *nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 mock_nodelist_get_list(void)
 {
   n_nodelist_get_list++;
@@ -1614,7 +1820,15 @@ test_getinfo_md_all(void *arg)
   return;
 }
 
+#define PARSER_TEST(type)                                             \
+  { "parse/" #type, test_controller_parse_cmd, 0, &passthrough_setup, \
+      (void*)&parse_ ## type ## _params }
+
 struct testcase_t controller_tests[] = {
+  PARSER_TEST(one_to_three),
+  PARSER_TEST(no_args_one_obj),
+  PARSER_TEST(no_args_kwargs),
+  PARSER_TEST(one_arg_kwargs),
   { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0,
     NULL, NULL },
   { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0,
diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c
index 70d36e53d4..0d276ef8a6 100644
--- a/src/test/test_controller_events.c
+++ b/src/test/test_controller_events.c
@@ -4,13 +4,21 @@
 #define CONNECTION_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "app/main/subsysmgr.h"
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitlist.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
 
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -351,10 +359,10 @@ test_cntev_dirboot_defer_desc(void *arg)
   /* This event should get deferred */
   control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
   assert_bootmsg("45 TAG=requesting_descriptors");
@@ -374,29 +382,215 @@ test_cntev_dirboot_defer_orconn(void *arg)
   control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
   assert_bootmsg("0 TAG=starting");
   /* This event should get deferred */
-  control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+  control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
-  assert_bootmsg("80 TAG=conn_or");
+  assert_bootmsg("75 TAG=enough_dirinfo");
  done:
   tor_free(saved_event_str);
   UNMOCK(queue_control_event_string);
 }
 
-#define TEST(name, flags)                                               \
+static void
+test_cntev_signal(void *arg)
+{
+  (void)arg;
+  int rv;
+
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+
+  /* Nothing is listening for signals, so no event should be queued. */
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+
+  /* Now try with signals included in the event mask. */
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_GOT_SIGNAL));
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL RELOAD\r\n");
+
+  rv = control_event_signal(SIGACTIVE);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL ACTIVE\r\n");
+
+  /* Try a signal that doesn't exist. */
+  setup_full_capture_of_logs(LOG_WARN);
+  tor_free(saved_event_str);
+  rv = control_event_signal(99999);
+  tt_int_op(-1, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+  expect_single_log_msg_containing("Unrecognized signal 99999");
+
+ done:
+  tor_free(saved_event_str);
+ teardown_capture_of_logs();
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+setup_orconn_state(orconn_state_msg_t *msg, uint64_t gid, uint64_t chan,
+                   int proxy_type)
+{
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->proxy_type = proxy_type;
+}
+
+static void
+send_orconn_state(const orconn_state_msg_t *msg_in, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  msg->state = state;
+  orconn_state_publish(msg);
+}
+
+static void
+send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->onehop = onehop;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_cntev_orconn_state(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_NONE);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  send_ocirc_chan(1, 1, true);
+  assert_bootmsg("5 TAG=conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  /* It doesn't know it's an origin circuit yet */
+  assert_bootmsg("15 TAG=handshake_done");
+  send_ocirc_chan(2, 2, false);
+  assert_bootmsg("80 TAG=ap_conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("85 TAG=ap_conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("89 TAG=ap_handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("90 TAG=ap_handshake_done");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_pt(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("1 TAG=conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("2 TAG=conn_done_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("76 TAG=ap_conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("77 TAG=ap_conn_done_pt");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_proxy(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_CONNECT);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("3 TAG=conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("4 TAG=conn_done_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("78 TAG=ap_conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("79 TAG=ap_conn_done_proxy");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+#define TEST(name, flags)                               \
   { #name, test_cntev_ ## name, flags, 0, NULL }
 
+#define T_PUBSUB(name, setup)                                           \
+  { #name, test_cntev_ ## name, TT_FORK, &helper_pubsub_setup, NULL }
+
 struct testcase_t controller_event_tests[] = {
   TEST(sum_up_cell_stats, TT_FORK),
   TEST(append_cell_stats, TT_FORK),
   TEST(format_cell_stats, TT_FORK),
   TEST(event_mask, TT_FORK),
-  TEST(dirboot_defer_desc, TT_FORK),
-  TEST(dirboot_defer_orconn, TT_FORK),
+  T_PUBSUB(dirboot_defer_desc, TT_FORK),
+  T_PUBSUB(dirboot_defer_orconn, TT_FORK),
+  T_PUBSUB(signal, TT_FORK),
+  T_PUBSUB(orconn_state, TT_FORK),
+  T_PUBSUB(orconn_state_pt, TT_FORK),
+  T_PUBSUB(orconn_state_proxy, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 5af0cce130..ed39f41560 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -32,7 +32,7 @@
 DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/dh.h>
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Run unit tests for Diffie-Hellman functionality. */
 static void
@@ -190,7 +190,7 @@ test_crypto_dh(void *arg)
     DH_get0_key(dh4, &pk, &sk);
 #else
     pk = dh4->pub_key;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
     tt_assert(pk);
     tt_int_op(BN_num_bytes(pk), OP_LE, DH1024_KEY_LEN);
     tt_int_op(BN_num_bytes(pk), OP_GT, 0);
@@ -207,7 +207,7 @@ test_crypto_dh(void *arg)
     tt_int_op(s1len, OP_GT, 0);
     tt_mem_op(s1, OP_EQ, s2, s1len);
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
  done:
   crypto_dh_free(dh1);
@@ -219,7 +219,7 @@ test_crypto_dh(void *arg)
     DH_free(dh4);
   if (pubkey_tmp)
     BN_free(pubkey_tmp);
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 }
 
 static void
@@ -248,174 +248,12 @@ test_crypto_openssl_version(void *arg)
   tt_int_op(a, OP_GE, 0);
   tt_int_op(b, OP_GE, 0);
   tt_int_op(c, OP_GE, 0);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
  done:
   ;
 }
 
-/** Run unit tests for our random number generation function and its wrappers.
- */
-static void
-test_crypto_rng(void *arg)
-{
-  int i, j, allok;
-  char data1[100], data2[100];
-  double d;
-  char *h=NULL;
-
-  /* Try out RNG. */
-  (void)arg;
-  tt_assert(! crypto_seed_rng());
-  crypto_rand(data1, 100);
-  crypto_rand(data2, 100);
-  tt_mem_op(data1,OP_NE, data2,100);
-  allok = 1;
-  for (i = 0; i < 100; ++i) {
-    uint64_t big;
-    char *host;
-    j = crypto_rand_int(100);
-    if (j < 0 || j >= 100)
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(1)<<40);
-    if (big >= (UINT64_C(1)<<40))
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(5));
-    if (big >= 5)
-      allok = 0;
-    d = crypto_rand_double();
-    tt_assert(d >= 0);
-    tt_assert(d < 1.0);
-    host = crypto_random_hostname(3,8,"www.",".onion");
-    if (strcmpstart(host,"www.") ||
-        strcmpend(host,".onion") ||
-        strlen(host) < 13 ||
-        strlen(host) > 18)
-      allok = 0;
-    tor_free(host);
-  }
-
-  /* Make sure crypto_random_hostname clips its inputs properly. */
-  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
-  tt_assert(! strcmpstart(h,"www."));
-  tt_assert(! strcmpend(h,".onion"));
-  tt_int_op(63+4+6, OP_EQ, strlen(h));
-
-  tt_assert(allok);
- done:
-  tor_free(h);
-}
-
-static void
-test_crypto_rng_range(void *arg)
-{
-  int got_smallest = 0, got_largest = 0;
-  int i;
-
-  (void)arg;
-  for (i = 0; i < 1000; ++i) {
-    int x = crypto_rand_int_range(5,9);
-    tt_int_op(x, OP_GE, 5);
-    tt_int_op(x, OP_LT, 9);
-    if (x == 5)
-      got_smallest = 1;
-    if (x == 8)
-      got_largest = 1;
-  }
-  /* These fail with probability 1/10^603. */
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  got_smallest = got_largest = 0;
-  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
-  for (i = 0; i < 1000; ++i) {
-    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
-    tt_u64_op(x, OP_GE, ten_billion);
-    tt_u64_op(x, OP_LT, ten_billion+10);
-    if (x == ten_billion)
-      got_smallest = 1;
-    if (x == ten_billion+9)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  const time_t now = time(NULL);
-  for (i = 0; i < 2000; ++i) {
-    time_t x = crypto_rand_time_range(now, now+60);
-    tt_i64_op(x, OP_GE, now);
-    tt_i64_op(x, OP_LT, now+60);
-    if (x == now)
-      got_smallest = 1;
-    if (x == now+59)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
- done:
-  ;
-}
-
-static void
-test_crypto_rng_strongest(void *arg)
-{
-  const char *how = arg;
-  int broken = 0;
-
-  if (how == NULL) {
-    ;
-  } else if (!strcmp(how, "nosyscall")) {
-    break_strongest_rng_syscall = 1;
-  } else if (!strcmp(how, "nofallback")) {
-    break_strongest_rng_fallback = 1;
-  } else if (!strcmp(how, "broken")) {
-    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
-  }
-
-#define N 128
-  uint8_t combine_and[N];
-  uint8_t combine_or[N];
-  int i, j;
-
-  memset(combine_and, 0xff, N);
-  memset(combine_or, 0, N);
-
-  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
-    uint8_t output[N];
-    memset(output, 0, N);
-    if (how == NULL) {
-      /* this one can't fail. */
-      crypto_strongest_rand(output, sizeof(output));
-    } else {
-      int r = crypto_strongest_rand_raw(output, sizeof(output));
-      if (r == -1) {
-        if (broken) {
-          goto done; /* we're fine. */
-        }
-        /* This function is allowed to break, but only if it always breaks. */
-        tt_int_op(i, OP_EQ, 0);
-        tt_skip();
-      } else {
-        tt_assert(! broken);
-      }
-    }
-    for (j = 0; j < N; ++j) {
-      combine_and[j] &= output[j];
-      combine_or[j] |= output[j];
-    }
-  }
-
-  for (j = 0; j < N; ++j) {
-    tt_int_op(combine_and[j], OP_EQ, 0);
-    tt_int_op(combine_or[j], OP_EQ, 0xff);
-  }
- done:
-  ;
-#undef N
-}
-
 /** Run unit tests for our AES128 functionality */
 static void
 test_crypto_aes128(void *arg)
@@ -551,7 +389,7 @@ test_crypto_aes128(void *arg)
                                    "\xff\xff\xff\xff\xff\xff\xff\xff"
                                    "\xff\xff\xff\xff\xff\xff\xff\xff");
   crypto_cipher_crypt_inplace(env1, data2, 64);
-  tt_assert(tor_mem_is_zero(data2, 64));
+  tt_assert(fast_mem_is_zero(data2, 64));
 
  done:
   tor_free(mem_op_hex_tmp);
@@ -1173,13 +1011,19 @@ test_crypto_sha3_xof(void *arg)
   crypto_xof_free(xof);
   memset(out, 0, sizeof(out));
 
+  /* Test one-function absorb/squeeze. */
+  crypto_xof(out, sizeof(out), msg, sizeof(msg));
+  test_memeq_hex(out, squeezed_hex);
+  memset(out, 0, sizeof(out));
+
   /* Test incremental absorb/squeeze. */
   xof = crypto_xof_new();
   tt_assert(xof);
   for (size_t i = 0; i < sizeof(msg); i++)
     crypto_xof_add_bytes(xof, msg + i, 1);
-  for (size_t i = 0; i < sizeof(out); i++)
+  for (size_t i = 0; i < sizeof(out); i++) {
     crypto_xof_squeeze_bytes(xof, out + i, 1);
+  }
   test_memeq_hex(out, squeezed_hex);
 
  done:
@@ -1903,13 +1747,13 @@ test_crypto_base32_decode(void *arg)
   /* Encode and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Encode, uppercase, and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   tor_strupper(encoded);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Change encoded string and decode. */
   if (encoded[0] == 'A' || encoded[0] == 'a')
@@ -1917,12 +1761,12 @@ test_crypto_base32_decode(void *arg)
   else
     encoded[0] = 'A';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_NE, decoded, 60);
   /* Bad encodings. */
   encoded[0] = '!';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(0, OP_GT, res);
+  tt_int_op(res, OP_LT, 0);
 
  done:
   ;
@@ -2117,7 +1961,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "e0544770bc7de853b38f9100489e3e79";
   const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88"
                                 "bd13861475516bc2cd2b6e005e805064";
-#else /* !(defined(SLOW_CURVE25519_TEST)) */
+#else /* !defined(SLOW_CURVE25519_TEST) */
   const int loop_max=200;
   const char e1_expected[]    = "bc7112cde03f97ef7008cad1bdc56be3"
                                 "c6a1037d74cceb3712e9206871dcf654";
@@ -2269,7 +2113,7 @@ test_crypto_curve25519_encode(void *arg)
 
   curve25519_secret_key_generate(&seckey, 0);
   curve25519_public_key_generate(&key1, &seckey);
-  tt_int_op(0, OP_EQ, curve25519_public_to_base64(buf, &key1));
+  curve25519_public_to_base64(buf, &key1);
   tt_int_op(CURVE25519_BASE64_PADDED_LEN, OP_EQ, strlen(buf));
 
   tt_int_op(0, OP_EQ, curve25519_public_from_base64(&key2, buf));
@@ -2328,7 +2172,7 @@ test_crypto_curve25519_persist(void *arg)
   tt_u64_op((uint64_t)st.st_size, OP_EQ,
             32+CURVE25519_PUBKEY_LEN+CURVE25519_SECKEY_LEN);
   tt_assert(fast_memeq(content, "== c25519v1: testing ==", taglen));
-  tt_assert(tor_mem_is_zero(content+taglen, 32-taglen));
+  tt_assert(fast_mem_is_zero(content+taglen, 32-taglen));
   cp = content + 32;
   tt_mem_op(keypair.seckey.secret_key,OP_EQ,
              cp,
@@ -2649,13 +2493,13 @@ test_crypto_ed25519_encode(void *arg)
 
   /* Test roundtrip. */
   tt_int_op(0, OP_EQ, ed25519_keypair_generate(&kp, 0));
-  tt_int_op(0, OP_EQ, ed25519_public_to_base64(buf, &kp.pubkey));
+  ed25519_public_to_base64(buf, &kp.pubkey);
   tt_int_op(ED25519_BASE64_LEN, OP_EQ, strlen(buf));
   tt_int_op(0, OP_EQ, ed25519_public_from_base64(&pk, buf));
   tt_mem_op(kp.pubkey.pubkey, OP_EQ, pk.pubkey, ED25519_PUBKEY_LEN);
 
   tt_int_op(0, OP_EQ, ed25519_sign(&sig1, (const uint8_t*)"ABC", 3, &kp));
-  tt_int_op(0, OP_EQ, ed25519_signature_to_base64(buf, &sig1));
+  ed25519_signature_to_base64(buf, &sig1);
   tt_int_op(0, OP_EQ, ed25519_signature_from_base64(&sig2, buf));
   tt_mem_op(sig1.sig, OP_EQ, sig2.sig, ED25519_SIG_LEN);
 
@@ -3178,15 +3022,6 @@ test_crypto_failure_modes(void *arg)
 
 struct testcase_t crypto_tests[] = {
   CRYPTO_LEGACY(formats),
-  CRYPTO_LEGACY(rng),
-  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
-  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
-  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nosyscall" },
-  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nofallback" },
-  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"broken" },
   { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL },
   { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" },
   { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" },
diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c
new file mode 100644
index 0000000000..6b7749a889
--- /dev/null
+++ b/src/test/test_crypto_rng.c
@@ -0,0 +1,332 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#define CRYPTO_RAND_PRIVATE
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+/** Run unit tests for our random number generation function and its wrappers.
+ */
+static void
+test_crypto_rng(void *arg)
+{
+  int i, j, allok;
+  char data1[100], data2[100];
+  double d;
+  char *h=NULL;
+
+  /* Try out RNG. */
+  (void)arg;
+  tt_assert(! crypto_seed_rng());
+  crypto_rand(data1, 100);
+  crypto_rand(data2, 100);
+  tt_mem_op(data1,OP_NE, data2,100);
+  allok = 1;
+  for (i = 0; i < 100; ++i) {
+    uint64_t big;
+    char *host;
+    j = crypto_rand_int(100);
+    if (j < 0 || j >= 100)
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(1)<<40);
+    if (big >= (UINT64_C(1)<<40))
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(5));
+    if (big >= 5)
+      allok = 0;
+    d = crypto_rand_double();
+    tt_assert(d >= 0);
+    tt_assert(d < 1.0);
+    host = crypto_random_hostname(3,8,"www.",".onion");
+    if (strcmpstart(host,"www.") ||
+        strcmpend(host,".onion") ||
+        strlen(host) < 13 ||
+        strlen(host) > 18)
+      allok = 0;
+    tor_free(host);
+  }
+
+  /* Make sure crypto_random_hostname clips its inputs properly. */
+  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
+  tt_assert(! strcmpstart(h,"www."));
+  tt_assert(! strcmpend(h,".onion"));
+  tt_int_op(63+4+6, OP_EQ, strlen(h));
+
+  tt_assert(allok);
+ done:
+  tor_free(h);
+}
+
+static void
+test_crypto_rng_range(void *arg)
+{
+  int got_smallest = 0, got_largest = 0;
+  int i;
+
+  (void)arg;
+  for (i = 0; i < 1000; ++i) {
+    int x = crypto_rand_int_range(5,9);
+    tt_int_op(x, OP_GE, 5);
+    tt_int_op(x, OP_LT, 9);
+    if (x == 5)
+      got_smallest = 1;
+    if (x == 8)
+      got_largest = 1;
+  }
+  /* These fail with probability 1/10^603. */
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  got_smallest = got_largest = 0;
+  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
+  for (i = 0; i < 1000; ++i) {
+    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
+    tt_u64_op(x, OP_GE, ten_billion);
+    tt_u64_op(x, OP_LT, ten_billion+10);
+    if (x == ten_billion)
+      got_smallest = 1;
+    if (x == ten_billion+9)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  const time_t now = time(NULL);
+  for (i = 0; i < 2000; ++i) {
+    time_t x = crypto_rand_time_range(now, now+60);
+    tt_i64_op(x, OP_GE, now);
+    tt_i64_op(x, OP_LT, now+60);
+    if (x == now)
+      got_smallest = 1;
+    if (x == now+59)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+ done:
+  ;
+}
+
+static void
+test_crypto_rng_strongest(void *arg)
+{
+  const char *how = arg;
+  int broken = 0;
+
+  if (how == NULL) {
+    ;
+  } else if (!strcmp(how, "nosyscall")) {
+    break_strongest_rng_syscall = 1;
+  } else if (!strcmp(how, "nofallback")) {
+    break_strongest_rng_fallback = 1;
+  } else if (!strcmp(how, "broken")) {
+    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
+  }
+
+#define N 128
+  uint8_t combine_and[N];
+  uint8_t combine_or[N];
+  int i, j;
+
+  memset(combine_and, 0xff, N);
+  memset(combine_or, 0, N);
+
+  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
+    uint8_t output[N];
+    memset(output, 0, N);
+    if (how == NULL) {
+      /* this one can't fail. */
+      crypto_strongest_rand(output, sizeof(output));
+    } else {
+      int r = crypto_strongest_rand_raw(output, sizeof(output));
+      if (r == -1) {
+        if (broken) {
+          goto done; /* we're fine. */
+        }
+        /* This function is allowed to break, but only if it always breaks. */
+        tt_int_op(i, OP_EQ, 0);
+        tt_skip();
+      } else {
+        tt_assert(! broken);
+      }
+    }
+    for (j = 0; j < N; ++j) {
+      combine_and[j] &= output[j];
+      combine_or[j] |= output[j];
+    }
+  }
+
+  for (j = 0; j < N; ++j) {
+    tt_int_op(combine_and[j], OP_EQ, 0);
+    tt_int_op(combine_or[j], OP_EQ, 0xff);
+  }
+ done:
+  ;
+#undef N
+}
+
+static void
+test_crypto_rng_fast(void *arg)
+{
+  (void)arg;
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  tt_assert(rng);
+
+  /* Rudimentary black-block test to make sure that our prng outputs
+   * have all bits sometimes on and all bits sometimes off. */
+  uint64_t m1 = 0, m2 = ~(uint64_t)0;
+  const int N = 128;
+
+  for (int i=0; i < N; ++i) {
+    uint64_t v;
+    crypto_fast_rng_getbytes(rng, (void*)&v, sizeof(v));
+    m1 |= v;
+    m2 &= v;
+  }
+
+  tt_u64_op(m1, OP_EQ, ~(uint64_t)0);
+  tt_u64_op(m2, OP_EQ, 0);
+
+  /* Check range functions. */
+  int counts[5];
+  memset(counts, 0, sizeof(counts));
+  for (int i=0; i < N; ++i) {
+    unsigned u = crypto_fast_rng_get_uint(rng, 5);
+    tt_int_op(u, OP_GE, 0);
+    tt_int_op(u, OP_LT, 5);
+    counts[u]++;
+
+    uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+
+    double d = crypto_fast_rng_get_double(rng);
+    tt_assert(d >= 0.0);
+    tt_assert(d < 1.0);
+  }
+
+  /* All values should have come up once. */
+  for (int i=0; i<5; ++i) {
+    tt_int_op(counts[i], OP_GT, 0);
+  }
+
+  /* per-thread rand_fast shouldn't crash or leak. */
+  crypto_fast_rng_t *t_rng = get_thread_fast_rng();
+  for (int i = 0; i < N; ++i) {
+    uint64_t u64 = crypto_fast_rng_get_uint64(t_rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+  }
+
+ done:
+  crypto_fast_rng_free(rng);
+}
+
+static void
+test_crypto_rng_fast_whitebox(void *arg)
+{
+  (void)arg;
+  const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream();
+  char *buf = tor_malloc_zero(buflen);
+  char *buf2 = tor_malloc_zero(buflen);
+  char *buf3 = NULL, *buf4 = NULL;
+
+  crypto_cipher_t *cipher = NULL, *cipher2 = NULL;
+  uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed, 0, sizeof(seed));
+
+  /* Start with a prng with zero key and zero IV. */
+  crypto_fast_rng_t *rng = crypto_fast_rng_new_from_seed(seed);
+  tt_assert(rng);
+
+  /* We'll use a stream cipher to keep in sync */
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+
+  /* The first 48 bytes are used for the next seed -- let's make sure we have
+   * them.
+   */
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+
+  /* if we get 128 bytes, they should match the bytes from the aes256-counter
+   * stream, starting at position 48.
+   */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 128);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* Try that again, with an odd number of bytes. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 199);
+  memset(buf2, 0, 199);
+  crypto_cipher_crypt_inplace(cipher, buf2, 199);
+  tt_mem_op(buf, OP_EQ, buf2, 199);
+
+  /* Make sure that refilling works as expected: skip all but the last 5 bytes
+   * of this steam. */
+  size_t skip = buflen - (199+128) - 5;
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, skip);
+  crypto_cipher_crypt_inplace(cipher, buf2, skip);
+
+  /* Now get the next 128 bytes. The first 5 will come from this stream, and
+   * the next 5 will come from the stream keyed by the new value of 'seed'. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 5);
+  crypto_cipher_free(cipher);
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, buf2+5, 128-5);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* And check the next 7 bytes to make sure we didn't discard anything. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 7);
+  memset(buf2, 0, 7);
+  crypto_cipher_crypt_inplace(cipher, buf2, 7);
+  tt_mem_op(buf, OP_EQ, buf2, 7);
+
+  /* Now try the optimization for long outputs. */
+  buf3 = tor_malloc(65536);
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf3, 65536);
+
+  buf4 = tor_malloc_zero(65536);
+  uint8_t seed2[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed2, 0, sizeof(seed2));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed2, sizeof(seed2));
+  cipher2 = crypto_cipher_new_with_iv_and_bits(seed2, seed2+32, 256);
+  crypto_cipher_crypt_inplace(cipher2, buf4, 65536);
+  tt_mem_op(buf3, OP_EQ, buf4, 65536);
+
+ done:
+  crypto_fast_rng_free(rng);
+  crypto_cipher_free(cipher);
+  crypto_cipher_free(cipher2);
+  tor_free(buf);
+  tor_free(buf2);
+  tor_free(buf3);
+  tor_free(buf4);
+}
+
+struct testcase_t crypto_rng_tests[] = {
+  { "rng", test_crypto_rng, 0, NULL, NULL },
+  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
+  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
+  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nosyscall" },
+  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nofallback" },
+  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"broken" },
+  { "fast", test_crypto_rng_fast, 0, NULL, NULL },
+  { "fast_whitebox", test_crypto_rng_fast_whitebox, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c
index e24aee8930..3b20dfa587 100644
--- a/src/test/test_crypto_slow.c
+++ b/src/test/test_crypto_slow.c
@@ -109,7 +109,7 @@ run_s2k_tests(const unsigned flags, const unsigned type,
             secret_to_key_derivekey(buf3, sizeof(buf3), buf, speclen,
                                     pw1, strlen(pw1)));
   tt_mem_op(buf2, OP_EQ, buf3, sizeof(buf3));
-  tt_assert(!tor_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
+  tt_assert(!fast_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
 
  done:
   ;
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 0e44c47f3f..6329ff7750 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -8,7 +8,7 @@
 
 #define BWAUTH_PRIVATE
 #define CONFIG_PRIVATE
-#define CONTROL_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
 #define DIRSERV_PRIVATE
@@ -26,13 +26,13 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/connection.h"
 #include "core/or/relay.h"
 #include "core/or/versions.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_getinfo.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/dsigs_parse.h"
@@ -91,9 +91,29 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 #define NS_MODULE dir
 
+static networkstatus_t *
+networkstatus_parse_vote_from_string_(const char *s,
+                                      const char **eos_out,
+                                      enum networkstatus_type_t ns_type)
+{
+  size_t len = strlen(s);
+  // memdup so that it won't be nul-terminated.
+  char *tmp = tor_memdup(s, len);
+  networkstatus_t *result =
+    networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type);
+  if (eos_out && *eos_out) {
+    *eos_out = s + (*eos_out - tmp);
+  }
+  tor_free(tmp);
+  return result;
+}
+
 static void
 test_dir_nicknames(void *arg)
 {
@@ -142,6 +162,269 @@ test_dir_nicknames(void *arg)
   ;
 }
 
+/* Allocate and return a new routerinfo, with the fields set from the
+ * arguments to this function.
+ *
+ * Also sets:
+ *  - random RSA identity and onion keys,
+ *  - the platform field using get_platform_str(), and
+ *  - supports_tunnelled_dir_requests to 1.
+ *
+ * If rsa_onion_keypair_out is not NULL, it is set to the onion keypair.
+ * The caller must free this keypair.
+ */
+static routerinfo_t *
+basic_routerinfo_new(const char *nickname, uint32_t ipv4_addr,
+                     uint16_t or_port, uint16_t dir_port,
+                     uint32_t bandwidthrate, uint32_t bandwidthburst,
+                     uint32_t bandwidthcapacity,
+                     time_t published_on,
+                     crypto_pk_t **rsa_onion_keypair_out)
+{
+  char platform[256];
+
+  tor_assert(nickname);
+
+  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
+  /* These keys are random: idx is ignored. */
+  pk1 = pk_generate(0);
+  pk2 = pk_generate(1);
+
+  tor_assert(pk1);
+  tor_assert(pk2);
+
+  get_platform_str(platform, sizeof(platform));
+
+  routerinfo_t *r1 = tor_malloc_zero(sizeof(routerinfo_t));
+
+  r1->nickname = tor_strdup(nickname);
+  r1->platform = tor_strdup(platform);
+
+  r1->addr = ipv4_addr;
+  r1->or_port = or_port;
+  r1->dir_port = dir_port;
+  r1->supports_tunnelled_dir_requests = 1;
+
+  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
+  r1->identity_pkey = pk2;
+
+  r1->bandwidthrate = bandwidthrate;
+  r1->bandwidthburst = bandwidthburst;
+  r1->bandwidthcapacity = bandwidthcapacity;
+
+  r1->cache_info.published_on = published_on;
+
+  if (rsa_onion_keypair_out) {
+    *rsa_onion_keypair_out = pk1;
+  } else {
+    crypto_pk_free(pk1);
+  }
+
+  return r1;
+}
+
+/* Allocate and return a new string containing a "router" line for r1. */
+static char *
+get_new_router_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "router %s %s %d 0 %d\n",
+               r1->nickname, fmt_addr32(r1->addr),
+               r1->or_port, r1->dir_port);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "platform" line for the
+ * current Tor version and OS. */
+static char *
+get_new_platform_line(void)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "platform Tor %s on %s\n",
+               VERSION, get_uname());
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "published" line for r1.
+ * r1->cache_info.published_on must be between 0 and 59 seconds. */
+static char *
+get_new_published_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_assert(r1->cache_info.published_on >= 0);
+  tor_assert(r1->cache_info.published_on <= 59);
+
+  tor_asprintf(&line,
+               "published 1970-01-01 00:00:%02u\n",
+               (unsigned)r1->cache_info.published_on);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "fingerprint" line for r1. */
+static char *
+get_new_fingerprint_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+  char fingerprint[FINGERPRINT_LEN+1];
+
+  tor_assert(r1);
+
+  tor_assert(!crypto_pk_get_fingerprint(r1->identity_pkey, fingerprint, 1));
+  tor_assert(strlen(fingerprint) > 0);
+
+  tor_asprintf(&line,
+               "fingerprint %s\n",
+               fingerprint);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "uptime" line with uptime t.
+ *
+ * You should pass a hard-coded value to this function, because even if we made
+ * it reflect uptime, that still wouldn't make it right, because the two
+ * descriptors might be made on different seconds.
+ */
+static char *
+get_new_uptime_line(time_t t)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "uptime %u\n",
+               (unsigned)t);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "bandwidth" line for r1.
+ */
+static char *
+get_new_bandwidth_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "bandwidth %u %u %u\n",
+               r1->bandwidthrate,
+               r1->bandwidthburst,
+               r1->bandwidthcapacity);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a key_name block for the
+ * RSA key pk1.
+ */
+static char *
+get_new_rsa_key_block(const char *key_name, crypto_pk_t *pk1)
+{
+  char *block = NULL;
+  char *pk1_str = NULL;
+  size_t pk1_str_len = 0;
+
+  tor_assert(key_name);
+  tor_assert(pk1);
+
+  tor_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
+                                                   &pk1_str_len));
+  tor_assert(pk1_str);
+  tor_assert(pk1_str_len);
+
+  tor_asprintf(&block,
+               "%s\n%s",
+               key_name,
+               pk1_str);
+  tor_free(pk1_str);
+
+  tor_assert(block);
+  return block;
+}
+
+/* Allocate and return a new string containing an "onion-key" block for the
+ * router r1.
+ */
+static char *
+get_new_onion_key_block(const routerinfo_t *r1)
+{
+  char *block = NULL;
+  tor_assert(r1);
+  crypto_pk_t *pk_tmp = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                  r1->onion_pkey_len);
+  block = get_new_rsa_key_block("onion-key", pk_tmp);
+  crypto_pk_free(pk_tmp);
+  return block;
+}
+
+/* Allocate and return a new string containing an "signing-key" block for the
+ * router r1.
+ */
+static char *
+get_new_signing_key_block(const routerinfo_t *r1)
+{
+  tor_assert(r1);
+  return get_new_rsa_key_block("signing-key", r1->identity_pkey);
+}
+
+/* Allocate and return a new string containing an "ntor-onion-key" line for
+ * the curve25519 public key ntor_onion_pubkey.
+ */
+static char *
+get_new_ntor_onion_key_line(const curve25519_public_key_t *ntor_onion_pubkey)
+{
+  char *line = NULL;
+  char cert_buf[256];
+  int rv = 0;
+
+  tor_assert(ntor_onion_pubkey);
+
+  rv = base64_encode(cert_buf, sizeof(cert_buf),
+                     (const char*)ntor_onion_pubkey->public_key, 32,
+                     BASE64_ENCODE_MULTILINE);
+  tor_assert(rv > 0);
+  tor_assert(strlen(cert_buf) > 0);
+
+  tor_asprintf(&line,
+               "ntor-onion-key %s",
+               cert_buf);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "bridge-distribution-request"
+ * line for options.
+ */
+static char *
+get_new_bridge_distribution_request_line(const or_options_t *options)
+{
+  if (options->BridgeRelay) {
+    return tor_strdup("bridge-distribution-request any\n");
+  } else {
+    return tor_strdup("");
+  }
+}
+
 static smartlist_t *mocked_configured_ports = NULL;
 
 /** Returns mocked_configured_ports */
@@ -151,71 +434,510 @@ mock_get_configured_ports(void)
   return mocked_configured_ports;
 }
 
-/** Run unit tests for router descriptor generation logic. */
+static tor_cert_t *
+mock_tor_cert_dup_null(const tor_cert_t *cert)
+{
+  (void)cert;
+  return NULL;
+}
+
+static crypto_pk_t *mocked_server_identitykey = NULL;
+
+/* Returns mocked_server_identitykey with no checks. */
+static crypto_pk_t *
+mock_get_server_identity_key(void)
+{
+  return mocked_server_identitykey;
+}
+
+static crypto_pk_t *mocked_onionkey = NULL;
+
+/* Returns mocked_onionkey with no checks. */
+static crypto_pk_t *
+mock_get_onion_key(void)
+{
+  return mocked_onionkey;
+}
+
+static routerinfo_t *mocked_routerinfo = NULL;
+
+/* Returns 0 and sets ri_out to mocked_routerinfo.
+ * ri_out must not be NULL. There are no other checks. */
+static int
+mock_router_build_fresh_unsigned_routerinfo(routerinfo_t **ri_out)
+{
+  tor_assert(ri_out);
+  *ri_out = mocked_routerinfo;
+  return 0;
+}
+
+static ed25519_keypair_t *mocked_master_signing_key = NULL;
+
+/* Returns mocked_master_signing_key with no checks. */
+static const ed25519_keypair_t *
+mock_get_master_signing_keypair(void)
+{
+  return mocked_master_signing_key;
+}
+
+static struct tor_cert_st *mocked_signing_key_cert = NULL;
+
+/* Returns mocked_signing_key_cert with no checks. */
+static const struct tor_cert_st *
+mock_get_master_signing_key_cert(void)
+{
+  return mocked_signing_key_cert;
+}
+
+static curve25519_keypair_t *mocked_curve25519_onion_key = NULL;
+
+/* Returns mocked_curve25519_onion_key with no checks. */
+static const curve25519_keypair_t *
+mock_get_current_curve25519_keypair(void)
+{
+  return mocked_curve25519_onion_key;
+}
+
+/* Unmock get_configured_ports() and free mocked_configured_ports. */
+static void
+cleanup_mock_configured_ports(void)
+{
+  UNMOCK(get_configured_ports);
+
+  if (mocked_configured_ports) {
+    SMARTLIST_FOREACH(mocked_configured_ports, port_cfg_t *, p, tor_free(p));
+    smartlist_free(mocked_configured_ports);
+  }
+}
+
+/* Mock get_configured_ports() with a list containing or_port and dir_port.
+ * If a port is 0, don't set it.
+ * Only sets the minimal data required for the tests to pass. */
+static void
+setup_mock_configured_ports(uint16_t or_port, uint16_t dir_port)
+{
+  cleanup_mock_configured_ports();
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  MOCK(get_configured_ports, mock_get_configured_ports);
+  mocked_configured_ports = smartlist_new();
+
+  if (or_port) {
+    port_cfg_t *or_port_cfg = tor_malloc_zero(sizeof(*or_port_cfg));
+    or_port_cfg->type = CONN_TYPE_OR_LISTENER;
+    or_port_cfg->addr.family = AF_INET;
+    or_port_cfg->port = or_port;
+    smartlist_add(mocked_configured_ports, or_port_cfg);
+  }
+
+  if (dir_port) {
+    port_cfg_t *dir_port_cfg = tor_malloc_zero(sizeof(*dir_port_cfg));
+    dir_port_cfg->type = CONN_TYPE_DIR_LISTENER;
+    dir_port_cfg->addr.family = AF_INET;
+    dir_port_cfg->port = dir_port;
+    smartlist_add(mocked_configured_ports, dir_port_cfg);
+  }
+}
+
+/* Clean up the data structures and unmock the functions needed for generating
+ * a fresh descriptor. */
+static void
+cleanup_mocks_for_fresh_descriptor(void)
+{
+  tor_free(get_options_mutable()->Nickname);
+
+  mocked_server_identitykey = NULL;
+  UNMOCK(get_server_identity_key);
+
+  crypto_pk_free(mocked_onionkey);
+  UNMOCK(get_onion_key);
+}
+
+/* Mock the data structures and functions needed for generating a fresh
+ * descriptor.
+ *
+ * Sets options->Nickname from r1->nickname.
+ * Mocks get_server_identity_key() with r1->identity_pkey.
+ *
+ * If rsa_onion_keypair is not NULL, it is used to mock get_onion_key().
+ * Otherwise, the public key in r1->onion_pkey is used to mock get_onion_key().
+ */
+static void
+setup_mocks_for_fresh_descriptor(const routerinfo_t *r1,
+                                 crypto_pk_t *rsa_onion_keypair)
+{
+  cleanup_mocks_for_fresh_descriptor();
+
+  tor_assert(r1);
+
+  /* router_build_fresh_signed_extrainfo() requires options->Nickname */
+  get_options_mutable()->Nickname = tor_strdup(r1->nickname);
+
+  /* router_build_fresh_signed_extrainfo() requires get_server_identity_key().
+   * Use the same one as the call to router_dump_router_to_string() above.
+   */
+  mocked_server_identitykey = r1->identity_pkey;
+  MOCK(get_server_identity_key, mock_get_server_identity_key);
+
+  /* router_dump_and_sign_routerinfo_descriptor_body() requires
+   * get_onion_key(). Use the same one as r1.
+   */
+  if (rsa_onion_keypair) {
+    mocked_onionkey = crypto_pk_dup_key(rsa_onion_keypair);
+  } else {
+    mocked_onionkey = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                r1->onion_pkey_len);
+  }
+  MOCK(get_onion_key, mock_get_onion_key);
+}
+
+/* Set options based on arg.
+ *
+ * b: BridgeRelay 1
+ * e: ExtraInfoStatistics 1
+ * s: sets all the individual statistics options to 1
+ *
+ * Always sets AssumeReachable to 1.
+ *
+ * Does not set ServerTransportPlugin, because it's parsed before use.
+ *
+ * Does not set BridgeRecordUsageByCountry, because the tests don't have access
+ * to a GeoIPFile or GeoIPv6File. */
 static void
-test_dir_formats(void *arg)
+setup_dir_formats_options(const char *arg, or_options_t *options)
+{
+  /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */
+  options->AssumeReachable = 1;
+
+  if (strchr(arg, 'b')) {
+    options->BridgeRelay = 1;
+  }
+
+  if (strchr(arg, 'e')) {
+    options->ExtraInfoStatistics = 1;
+  }
+
+  if (strchr(arg, 's')) {
+    options->DirReqStatistics = 1;
+    options->HiddenServiceStatistics = 1;
+    options->EntryStatistics = 1;
+    options->CellStatistics = 1;
+    options->ExitPortStatistics = 1;
+    options->ConnDirectionStatistics = 1;
+    options->PaddingStatistics = 1;
+  }
+}
+
+/* Check that routerinfos r1 and rp1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_ROUTERINFO_CONSISTENCY(r1, rp1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(rp1); \
+\
+  tt_int_op(rp1->addr,OP_EQ, r1->addr); \
+  tt_int_op(rp1->or_port,OP_EQ, r1->or_port); \
+  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); \
+  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); \
+  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); \
+  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); \
+  crypto_pk_t *rp1_onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, \
+                                                      rp1->onion_pkey_len); \
+  crypto_pk_t *r1_onion_pkey = router_get_rsa_onion_pkey(r1->onion_pkey, \
+                                                      r1->onion_pkey_len); \
+  tt_int_op(crypto_pk_cmp_keys(rp1_onion_pkey, r1_onion_pkey), OP_EQ, 0); \
+  crypto_pk_free(rp1_onion_pkey); \
+  crypto_pk_free(r1_onion_pkey); \
+  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, r1->identity_pkey), \
+            OP_EQ, 0); \
+  tt_int_op(rp1->supports_tunnelled_dir_requests, OP_EQ, \
+            r1->supports_tunnelled_dir_requests); \
+STMT_END
+
+/* Check that routerinfo r1 and extrainfo e1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_EXTRAINFO_CONSISTENCY(r1, e1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(e1); \
+\
+  tt_str_op(e1->nickname, OP_EQ, r1->nickname); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA-only
+ * router. Tor versions without ed25519 (0.2.6 and earlier) are no longer
+ * officially supported, but the authorities still accept their descriptors.
+ */
+static void
+test_dir_formats_rsa(void *arg)
 {
   char *buf = NULL;
-  char buf2[8192];
-  char platform[256];
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *pk1_str = NULL, *pk2_str = NULL, *cp;
-  size_t pk1_str_len, pk2_str_len;
-  routerinfo_t *r1=NULL, *r2=NULL;
-  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
-  routerinfo_t *rp1 = NULL, *rp2 = NULL;
-  addr_policy_t *ex1, *ex2;
-  routerlist_t *dir1 = NULL, *dir2 = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
   uint8_t *rsa_cc = NULL;
-  or_options_t *options = get_options_mutable();
-  const addr_policy_t *p;
-  time_t now = time(NULL);
-  port_cfg_t orport, dirport;
-  char cert_buf[256];
 
-  (void)arg;
-  pk1 = pk_generate(0);
-  pk2 = pk_generate(1);
+  routerinfo_t *r1 = NULL;
+  extrainfo_t *e1 = NULL;
+  routerinfo_t *rp1 = NULL;
+  extrainfo_t *ep1 = NULL;
+
+  smartlist_t *chunks = NULL;
+  const char *msg = NULL;
+  int rv = -1;
 
-  tt_assert(pk1 && pk2);
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
 
   hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
 
-  get_platform_str(platform, sizeof(platform));
-  r1 = tor_malloc_zero(sizeof(routerinfo_t));
-  r1->addr = 0xc0a80001u; /* 192.168.0.1 */
-  r1->cache_info.published_on = 0;
-  r1->or_port = 9000;
-  r1->dir_port = 9003;
-  r1->supports_tunnelled_dir_requests = 1;
-  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
-  r1->ipv6_orport = 9999;
-  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
-  /* Fake just enough of an ntor key to get by */
+  /* r1 is a minimal, RSA-only descriptor, with DirPort and IPv6 */
+  r1 = basic_routerinfo_new("Magri", 0xc0a80001u /* 192.168.0.1 */,
+                            9000, 9003,
+                            1000, 5000, 10000,
+                            0,
+                            NULL);
+
+ /* Fake just enough of an ntor key to get by */
   curve25519_keypair_t r1_onion_keypair;
   curve25519_keypair_generate(&r1_onion_keypair, 0);
   r1->onion_curve25519_pkey = tor_memdup(&r1_onion_keypair.pubkey,
                                          sizeof(curve25519_public_key_t));
-  r1->identity_pkey = crypto_pk_dup_key(pk2);
-  r1->bandwidthrate = 1000;
-  r1->bandwidthburst = 5000;
-  r1->bandwidthcapacity = 10000;
+
+  /* Now add IPv6 */
+  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
+  r1->ipv6_orport = 9999;
+
   r1->exit_policy = NULL;
-  r1->nickname = tor_strdup("Magri");
-  r1->platform = tor_strdup(platform);
 
-  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex1->policy_type = ADDR_POLICY_ACCEPT;
-  tor_addr_from_ipv4h(&ex1->addr, 0);
-  ex1->maskbits = 0;
-  ex1->prt_min = ex1->prt_max = 80;
-  ex2->policy_type = ADDR_POLICY_REJECT;
-  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
-  ex2->maskbits = 8;
-  ex2->prt_min = ex2->prt_max = 24;
-  r2 = tor_malloc_zero(sizeof(routerinfo_t));
-  r2->addr = 0x0a030201u; /* 10.3.2.1 */
+  /* XXXX+++ router_dump_to_string should really take this from ri. */
+  options->ContactInfo = tor_strdup("Magri White "
+                                    "<magri@elsewhere.example.com>");
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  tor_free(options->ContactInfo);
+  cleanup_mock_configured_ports();
+
+  /* Synthesise a router descriptor, without the signature */
+  chunks = smartlist_new();
+
+  smartlist_add(chunks, get_new_router_line(r1));
+  smartlist_add_strdup(chunks, "or-address [1:2:3:4::]:9999\n");
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r1));
+  smartlist_add(chunks, get_new_fingerprint_line(r1));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r1));
+
+  smartlist_add(chunks, get_new_onion_key_block(r1));
+  smartlist_add(chunks, get_new_signing_key_block(r1));
+
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
+
+  smartlist_add_strdup(chunks, "contact Magri White "
+                               "<magri@elsewhere.example.com>\n");
+
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r1_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "reject *:*\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
+
+  smartlist_add_strdup(chunks, "router-signature\n");
+
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+                             * twice */
+
+  tt_str_op(buf,OP_EQ, buf2);
+  tor_free(buf);
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  cleanup_mock_configured_ports();
+
+  /* Now, try to parse buf */
+  cp = buf;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  tor_free(buf);
+  routerinfo_free(rp1);
+
+  /* Test extrainfo creation.
+   * We avoid calling router_build_fresh_unsigned_routerinfo(), because it's
+   * too complex. Instead, we re-use the manually-created routerinfos.
+   */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r1, NULL);
+
+  /* router_build_fresh_signed_extrainfo() passes the result of
+   * get_master_signing_key_cert() directly to tor_cert_dup(), which fails on
+   * NULL. But we want a NULL ei->cache_info.signing_key_cert to test the
+   * non-ed key path.
+   */
+  MOCK(tor_cert_dup, mock_tor_cert_dup_null);
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  /* Test some of the low-level static functions. */
+  e1 = router_build_fresh_signed_extrainfo(r1);
+  tt_assert(e1);
+  router_update_routerinfo_from_extrainfo(r1, e1);
+  rv = router_dump_and_sign_routerinfo_descriptor_body(r1);
+  tt_assert(rv == 0);
+  msg = "";
+  rv = routerinfo_incompatible_with_extrainfo(r1->identity_pkey, e1,
+                                              &r1->cache_info, &msg);
+  /* If they are incompatible, fail and show the msg string */
+  tt_str_op(msg, OP_EQ, "");
+  tt_assert(rv == 0);
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  UNMOCK(tor_cert_dup);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, e1);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r1->cache_info.signed_descriptor_body);
+  cp = r1->cache_info.signed_descriptor_body;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  routerinfo_free(rp1);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e1->cache_info.signed_descriptor_body);
+  cp = e1->cache_info.signed_descriptor_body;
+  ep1 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, ep1);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep1);
+
+ done:
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->ContactInfo);
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r1);
+  routerinfo_free(rp1);
+
+  extrainfo_free(e1);
+  extrainfo_free(ep1);
+
+  tor_free(rsa_cc);
+
+  tor_free(buf);
+  tor_free(buf2);
+}
+
+/* Check that the exit policy in rp2 is as expected. */
+#define CHECK_PARSED_EXIT_POLICY(rp2) \
+STMT_BEGIN \
+  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); \
+ \
+  p = smartlist_get(rp2->exit_policy, 0); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); \
+  tt_assert(tor_addr_is_null(&p->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 0); \
+  tt_int_op(p->prt_min,OP_EQ, 80); \
+  tt_int_op(p->prt_max,OP_EQ, 80); \
+ \
+  p = smartlist_get(rp2->exit_policy, 1); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); \
+  tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 8); \
+  tt_int_op(p->prt_min,OP_EQ, 24); \
+  tt_int_op(p->prt_max,OP_EQ, 24); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA + ed25519
+ * router.
+ */
+static void
+test_dir_formats_rsa_ed25519(void *arg)
+{
+  char *buf = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
+  crypto_pk_t *r2_onion_pkey = NULL;
+  char cert_buf[256];
+  uint8_t *rsa_cc = NULL;
+  time_t now = time(NULL);
+
+  routerinfo_t *r2 = NULL;
+  extrainfo_t *e2 = NULL;
+  routerinfo_t *r2_out = NULL;
+  routerinfo_t *rp2 = NULL;
+  extrainfo_t *ep2 = NULL;
+  addr_policy_t *ex1, *ex2;
+  const addr_policy_t *p;
+
+  smartlist_t *chunks = NULL;
+  int rv = -1;
+
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
+
+  hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
+
+  /* r2 is a RSA + ed25519 descriptor, with an exit policy, but no DirPort or
+   * IPv6 */
+  r2 = basic_routerinfo_new("Fred", 0x0a030201u /* 10.3.2.1 */,
+                            9005, 0,
+                            3000, 3000, 3000,
+                            5,
+                            &r2_onion_pkey);
+
+  /* Fake just enough of an ntor key to get by */
+  curve25519_keypair_t r2_onion_keypair;
+  curve25519_keypair_generate(&r2_onion_keypair, 0);
+  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
+                                         sizeof(curve25519_public_key_t));
+
+  /* Now add relay ed25519 keys
+   * We can't use init_mock_ed_keys() here, because the keys are seeded */
   ed25519_keypair_t kp1, kp2;
   ed25519_secret_key_from_seed(&kp1.seckey,
                           (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
@@ -228,157 +950,78 @@ test_dir_formats(void *arg)
                                          &kp2.pubkey,
                                          now, 86400,
                                          CERT_FLAG_INCLUDE_SIGNING_KEY);
-  r2->platform = tor_strdup(platform);
-  r2->cache_info.published_on = 5;
-  r2->or_port = 9005;
-  r2->dir_port = 0;
-  r2->supports_tunnelled_dir_requests = 1;
-  router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len);
-  curve25519_keypair_t r2_onion_keypair;
-  curve25519_keypair_generate(&r2_onion_keypair, 0);
-  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
-                                         sizeof(curve25519_public_key_t));
-  r2->identity_pkey = crypto_pk_dup_key(pk1);
-  r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000;
+
+  /* Now add an exit policy */
+  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex1->policy_type = ADDR_POLICY_ACCEPT;
+  tor_addr_from_ipv4h(&ex1->addr, 0);
+  ex1->maskbits = 0;
+  ex1->prt_min = ex1->prt_max = 80;
+  ex2->policy_type = ADDR_POLICY_REJECT;
+  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
+  ex2->maskbits = 8;
+  ex2->prt_min = ex2->prt_max = 24;
+
   r2->exit_policy = smartlist_new();
   smartlist_add(r2->exit_policy, ex1);
   smartlist_add(r2->exit_policy, ex2);
-  r2->nickname = tor_strdup("Fred");
-
-  tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
-                                                    &pk1_str_len));
-  tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str,
-                                                    &pk2_str_len));
-
-  /* XXXX+++ router_dump_to_string should really take this from ri.*/
-  options->ContactInfo = tor_strdup("Magri White "
-                                    "<magri@elsewhere.example.com>");
-  /* Skip reachability checks for DirPort and tunnelled-dir-server */
-  options->AssumeReachable = 1;
 
-  /* Fake just enough of an ORPort and DirPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
-
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9000;
-  smartlist_add(mocked_configured_ports, &orport);
-
-  memset(&dirport, 0, sizeof(dirport));
-  dirport.type = CONN_TYPE_DIR_LISTENER;
-  dirport.addr.family = AF_INET;
-  dirport.port = 9003;
-  smartlist_add(mocked_configured_ports, &dirport);
-
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  tor_free(options->ContactInfo);
+  buf = router_dump_router_to_string(r2,
+                                     r2->identity_pkey, r2_onion_pkey,
+                                     &r2_onion_keypair, &kp2);
   tt_assert(buf);
 
-  strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n"
-          "or-address [1:2:3:4::]:9999\n"
-          "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:00\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-  /* XXX the "0" above is hard-coded, but even if we made it reflect
-   * uptime, that still wouldn't make it right, because the two
-   * descriptors might be made on different seconds... hm. */
-         "bandwidth 1000 5000 10000\n"
-          "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "contact Magri White <magri@elsewhere.example.com>\n",
-          sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r1_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "reject *:*\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\nrouter-signature\n", sizeof(buf2));
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
-                             * twice */
+  cleanup_mock_configured_ports();
 
-  tt_str_op(buf,OP_EQ, buf2);
-  tor_free(buf);
+  chunks = smartlist_new();
 
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-  tt_assert(buf);
-  cp = buf;
-  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp1);
-  tt_int_op(rp1->addr,OP_EQ, r1->addr);
-  tt_int_op(rp1->or_port,OP_EQ, r1->or_port);
-  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port);
-  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate);
-  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst);
-  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity);
-  crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey,
-                                                      rp1->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0);
-  tt_assert(rp1->supports_tunnelled_dir_requests);
-  //tt_assert(rp1->exit_policy == NULL);
-  tor_free(buf);
+  /* Synthesise a router descriptor, without the signatures */
+  smartlist_add(chunks, get_new_router_line(r2));
 
-  strlcpy(buf2,
-          "router Fred 10.3.2.1 9005 0 0\n"
-          "identity-ed25519\n"
-          "-----BEGIN ED25519 CERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks,
+                       "identity-ed25519\n"
+                       "-----BEGIN ED25519 CERT-----\n");
   base64_encode(cert_buf, sizeof(cert_buf),
                 (const char*)r2->cache_info.signing_key_cert->encoded,
                 r2->cache_info.signing_key_cert->encoded_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END ED25519 CERT-----\n", sizeof(buf2));
-  strlcat(buf2, "master-key-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END ED25519 CERT-----\n");
+
+  smartlist_add_strdup(chunks, "master-key-ed25519 ");
   {
     char k[ED25519_BASE64_LEN+1];
-    tt_int_op(ed25519_public_to_base64(k,
-                          &r2->cache_info.signing_key_cert->signing_key),
-              OP_GE, 0);
-    strlcat(buf2, k, sizeof(buf2));
-    strlcat(buf2, "\n", sizeof(buf2));
+    ed25519_public_to_base64(k, &r2->cache_info.signing_key_cert->signing_key);
+    smartlist_add_strdup(chunks, k);
+    smartlist_add_strdup(chunks, "\n");
   }
-  strlcat(buf2, "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:05\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk1, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-          "bandwidth 3000 3000 3000\n", sizeof(buf2));
-  strlcat(buf2, "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r2));
+  smartlist_add(chunks, get_new_fingerprint_line(r2));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r2));
+
+  smartlist_add(chunks, get_new_onion_key_block(r2));
+  smartlist_add(chunks, get_new_signing_key_block(r2));
+
   int rsa_cc_len;
-  rsa_cc = make_tap_onion_key_crosscert(pk2,
+  rsa_cc = make_tap_onion_key_crosscert(r2_onion_pkey,
                                         &kp1.pubkey,
-                                        pk1,
+                                        r2->identity_pkey,
                                         &rsa_cc_len);
   tt_assert(rsa_cc);
   base64_encode(cert_buf, sizeof(cert_buf), (char*)rsa_cc, rsa_cc_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, "onion-key-crosscert\n"
-          "-----BEGIN CROSSCERT-----\n", sizeof(buf2));
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END CROSSCERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks, "onion-key-crosscert\n"
+                       "-----BEGIN CROSSCERT-----\n");
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END CROSSCERT-----\n");
   int ntor_cc_sign;
   {
     tor_cert_t *ntor_cc = NULL;
@@ -393,112 +1036,165 @@ test_dir_formats(void *arg)
                 BASE64_ENCODE_MULTILINE);
     tor_cert_free(ntor_cc);
   }
-  tor_snprintf(buf2+strlen(buf2), sizeof(buf2)-strlen(buf2),
+  smartlist_add_asprintf(chunks,
                "ntor-onion-key-crosscert %d\n"
                "-----BEGIN ED25519 CERT-----\n"
                "%s"
                "-----END ED25519 CERT-----\n", ntor_cc_sign, cert_buf);
 
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r2_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "accept *:80\nreject 18.0.0.0/8:24\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\n", sizeof(buf2));
-  strlcat(buf2, "router-sig-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
 
-  /* Fake just enough of an ORPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r2_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "accept *:80\nreject 18.0.0.0/8:24\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
 
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9005;
-  smartlist_add(mocked_configured_ports, &orport);
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
 
-  buf = router_dump_router_to_string(r2, pk1, pk2, &r2_onion_keypair, &kp2);
-  tt_assert(buf);
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare either sig; they're never the same
                              * twice */
 
   tt_str_op(buf, OP_EQ, buf2);
   tor_free(buf);
 
-  buf = router_dump_router_to_string(r2, pk1, NULL, NULL, NULL);
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  buf = router_dump_router_to_string(r2, r2->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  cleanup_mock_configured_ports();
 
-  /* Reset for later */
+  /* Now, try to parse buf */
   cp = buf;
   rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp2);
-  tt_int_op(rp2->addr,OP_EQ, r2->addr);
-  tt_int_op(rp2->or_port,OP_EQ, r2->or_port);
-  tt_int_op(rp2->dir_port,OP_EQ, r2->dir_port);
-  tt_int_op(rp2->bandwidthrate,OP_EQ, r2->bandwidthrate);
-  tt_int_op(rp2->bandwidthburst,OP_EQ, r2->bandwidthburst);
-  tt_int_op(rp2->bandwidthcapacity,OP_EQ, r2->bandwidthcapacity);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
   tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
              r2->onion_curve25519_pkey->public_key,
              CURVE25519_PUBKEY_LEN);
-  onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey,
-                                         rp2->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0);
-  tt_assert(rp2->supports_tunnelled_dir_requests);
-
-  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2);
-
-  p = smartlist_get(rp2->exit_policy, 0);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT);
-  tt_assert(tor_addr_is_null(&p->addr));
-  tt_int_op(p->maskbits,OP_EQ, 0);
-  tt_int_op(p->prt_min,OP_EQ, 80);
-  tt_int_op(p->prt_max,OP_EQ, 80);
-
-  p = smartlist_get(rp2->exit_policy, 1);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT);
-  tt_assert(tor_addr_eq(&p->addr, &ex2->addr));
-  tt_int_op(p->maskbits,OP_EQ, 8);
-  tt_int_op(p->prt_min,OP_EQ, 24);
-  tt_int_op(p->prt_max,OP_EQ, 24);
-
-#if 0
-  /* Okay, now for the directories. */
-  {
-    fingerprint_list = smartlist_new();
-    crypto_pk_get_fingerprint(pk2, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
-    crypto_pk_get_fingerprint(pk1, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  tor_free(buf);
+  routerinfo_free(rp2);
+
+  /* Test extrainfo creation. */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r2, r2_onion_pkey);
+
+  /* router_build_fresh_descriptor() requires
+   * router_build_fresh_unsigned_routerinfo(), but the implementation is
+   * too complex. Instead, we re-use r2.
+   */
+  mocked_routerinfo = r2;
+  MOCK(router_build_fresh_unsigned_routerinfo,
+       mock_router_build_fresh_unsigned_routerinfo);
+
+  /* r2 uses ed25519, so we need to mock the ed key functions */
+  mocked_master_signing_key = &kp2;
+  MOCK(get_master_signing_keypair, mock_get_master_signing_keypair);
+
+  mocked_signing_key_cert = r2->cache_info.signing_key_cert;
+  MOCK(get_master_signing_key_cert, mock_get_master_signing_key_cert);
+
+  mocked_curve25519_onion_key = &r2_onion_keypair;
+  MOCK(get_current_curve25519_keypair, mock_get_current_curve25519_keypair);
+
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
+
+  /* Test the high-level interface. */
+  rv = router_build_fresh_descriptor(&r2_out, &e2);
+  if (rv < 0) {
+    /* router_build_fresh_descriptor() frees r2 on failure. */
+    r2 = NULL;
+    /* Get rid of an alias to rp2 */
+    r2_out = NULL;
   }
+  tt_assert(rv == 0);
+  tt_assert(r2_out);
+  tt_assert(e2);
+  /* Guaranteed by mock_router_build_fresh_unsigned_routerinfo() */
+  tt_ptr_op(r2_out, OP_EQ, r2);
+  /* Get rid of an alias to r2 */
+  r2_out = NULL;
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  mocked_routerinfo = NULL;
+  UNMOCK(router_build_fresh_unsigned_routerinfo);
+  mocked_master_signing_key = NULL;
+  UNMOCK(get_master_signing_keypair);
+  mocked_signing_key_cert = NULL;
+  UNMOCK(get_master_signing_key_cert);
+  mocked_curve25519_onion_key = NULL;
+  UNMOCK(get_current_curve25519_keypair);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, e2);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r2->cache_info.signed_descriptor_body);
+  cp = r2->cache_info.signed_descriptor_body;
+  rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
 
-#endif /* 0 */
-  dirserv_free_fingerprint_list();
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
+  tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
+            r2->onion_curve25519_pkey->public_key,
+            CURVE25519_PUBKEY_LEN);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  routerinfo_free(rp2);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e2->cache_info.signed_descriptor_body);
+  cp = e2->cache_info.signed_descriptor_body;
+  ep2 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, ep2);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep2);
 
  done:
-  if (r1)
-    routerinfo_free(r1);
-  if (r2)
-    routerinfo_free(r2);
-  if (rp2)
-    routerinfo_free(rp2);
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r2);
+  routerinfo_free(r2_out);
+  routerinfo_free(rp2);
+
+  extrainfo_free(e2);
+  extrainfo_free(ep2);
 
   tor_free(rsa_cc);
+  crypto_pk_free(r2_onion_pkey);
+
   tor_free(buf);
-  tor_free(pk1_str);
-  tor_free(pk2_str);
-  if (pk1) crypto_pk_free(pk1);
-  if (pk2) crypto_pk_free(pk2);
-  if (rp1) routerinfo_free(rp1);
-  tor_free(dir1); /* XXXX And more !*/
-  tor_free(dir2); /* And more !*/
+  tor_free(buf2);
 }
 
 #include "failing_routerdescs.inc"
@@ -1767,7 +2463,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, "", 0);
   setup_capture_of_logs(LOG_WARN);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
   expect_log_msg("Empty bandwidth file\n");
   teardown_capture_of_logs();
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1783,7 +2480,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op("", OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1794,7 +2493,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, header_lines_v100, 0);
   bw_file_headers = smartlist_new();
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1807,7 +2508,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1818,7 +2521,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100);
   write_str_to_file(fname, content, 0);
   tor_free(content);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
 
   /* Test bandwidth file including v1.1.0 bandwidth headers and
    * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */
@@ -1828,7 +2532,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1844,7 +2550,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1861,7 +2569,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1872,7 +2582,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110_no_terminator, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1883,7 +2595,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1898,7 +2612,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1913,7 +2629,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1929,7 +2647,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1946,7 +2666,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1964,7 +2686,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1985,7 +2709,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   /* force bw_file_headers to be bigger than
@@ -2014,7 +2740,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
@@ -2032,11 +2759,13 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_not_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
  done:
+  unlink(fname);
   tor_free(fname);
   tor_free(header_lines_v100);
   tor_free(header_lines_v110_no_terminator);
@@ -2863,11 +3592,17 @@ test_a_networkstatus(
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   /* Parse certificates and keys. */
-  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
   tt_assert(cert1);
-  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                           strlen(AUTHORITY_CERT_2),
+                                           NULL);
   tt_assert(cert2);
-  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                           strlen(AUTHORITY_CERT_3),
+                                           NULL);
   tt_assert(cert3);
   sign_skey_1 = crypto_pk_new();
   sign_skey_2 = crypto_pk_new();
@@ -2969,7 +3704,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_NS);
   tt_assert(consensus_text);
-  con = networkstatus_parse_vote_from_string(consensus_text, NULL,
+  con = networkstatus_parse_vote_from_string_(consensus_text, NULL,
                                              NS_TYPE_CONSENSUS);
   tt_assert(con);
   //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n",
@@ -2981,7 +3716,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_MICRODESC);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -3080,13 +3815,13 @@ test_a_networkstatus(
     tt_assert(consensus_text3);
     tt_assert(consensus_text_md2);
     tt_assert(consensus_text_md3);
-    con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL,
+    con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL,
+    con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL,
+    con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL,
+    con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL,
                                                 NS_TYPE_CONSENSUS);
     tt_assert(con2);
     tt_assert(con3);
@@ -3864,6 +4599,62 @@ mock_get_options(void)
   return mock_options;
 }
 
+/**
+ * Test dirauth_get_b64_digest_bw_file.
+ * This function should be near the other bwauth functions, but it needs
+ * mock_get_options, that is only defined here.
+ */
+
+static void
+test_dir_bwauth_bw_file_digest256(void *arg)
+{
+  (void)arg;
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  char *fname = tor_strdup(get_fname("V3BandwidthsFile"));
+  /* Initialize to a wrong digest. */
+  uint8_t digest[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+
+  /* Digest of an empty string. Initialize to a wrong digest. */
+  char digest_empty_str[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_empty_str, "", 0, DIGEST_SHA256);
+
+  /* Digest of the content. Initialize to a wrong digest. */
+  char digest_expected[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_expected, content, strlen(content), DIGEST_SHA256);
+
+  /* When the bandwidth file can not be found. */
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_empty_str, DIGEST256_LEN);
+
+  /* When there is a timestamp but it is too old. */
+  write_str_to_file(fname, content, 0);
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  /* The digest will be correct. */
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+  update_approx_time(1541171221);
+
+  /* When there is a bandwidth file and it can be read. */
+  tt_int_op(0, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+ done:
+  unlink(fname);
+  tor_free(fname);
+  update_approx_time(time(NULL));
+}
+
 static void
 reset_routerstatus(routerstatus_t *rs,
                    const char *hex_identity_digest,
@@ -6087,6 +6878,80 @@ test_dir_find_dl_min_delay(void* data)
 }
 
 static void
+test_dir_matching_flags(void *arg)
+{
+  (void) arg;
+  routerstatus_t *rs_noflags = NULL;
+  routerstatus_t *rs = NULL;
+  char *s = NULL;
+
+  smartlist_t *tokens = smartlist_new();
+  memarea_t *area = memarea_new();
+
+  int expected_val_when_unused = 0;
+
+  const char *ex_noflags =
+    "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 "
+       "192.168.0.1 9001 0\n"
+    "m thisoneislongerbecauseitisa256bitmddigest33\n"
+    "s\n";
+  const char *cp = ex_noflags;
+  rs_noflags = routerstatus_parse_entry_from_string(
+         area, &cp,
+         cp + strlen(cp),
+         tokens, NULL, NULL,
+         MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);
+  tt_assert(rs_noflags);
+
+#define FLAG(string, field) STMT_BEGIN {        \
+    tor_asprintf(&s,\
+                 "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \
+                 "192.168.0.1 9001 0\n"                                 \
+                 "m thisoneislongerbecauseitisa256bitmddigest33\n"      \
+                 "s %s\n", string);                                     \
+    cp = s;                                                             \
+    rs =  routerstatus_parse_entry_from_string(                         \
+      area, &cp,                                                        \
+      cp + strlen(cp),                                                  \
+      tokens, NULL, NULL,                                               \
+      MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);                  \
+    /* the field should usually be 0 when no flags are listed */        \
+    tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused);      \
+    /* the field should be 1 when this flags islisted */                \
+    tt_int_op(rs->field, OP_EQ, 1);                                     \
+    tor_free(s);                                                        \
+    routerstatus_free(rs);                                              \
+} STMT_END
+
+  FLAG("Authority", is_authority);
+  FLAG("BadExit", is_bad_exit);
+  FLAG("Exit", is_exit);
+  FLAG("Fast", is_fast);
+  FLAG("Guard", is_possible_guard);
+  FLAG("HSDir", is_hs_dir);
+  FLAG("Stable", is_stable);
+  FLAG("StaleDesc", is_staledesc);
+  FLAG("V2Dir", is_v2_dir);
+
+  // These flags are assumed to be set whether they're declared or not.
+  expected_val_when_unused = 1;
+  FLAG("Running", is_flagged_running);
+  FLAG("Valid", is_valid);
+  expected_val_when_unused = 0;
+
+  // These flags are no longer used, but still parsed.
+  FLAG("Named", is_named);
+  FLAG("Unnamed", is_unnamed);
+
+ done:
+  tor_free(s);
+  routerstatus_free(rs);
+  routerstatus_free(rs_noflags);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+}
+
+static void
 test_dir_assumed_flags(void *arg)
 {
   (void)arg;
@@ -6101,9 +6966,10 @@ test_dir_assumed_flags(void *arg)
        "192.168.0.1 9001 0\n"
     "m thisoneislongerbecauseitisa256bitmddigest33\n"
     "s Fast Guard Stable\n";
+  const char *eos = str1 + strlen(str1);
 
   const char *cp = str1;
-  rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
+  rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL,
                                             24, FLAV_MICRODESC);
   tt_assert(rs);
   tt_assert(rs->is_flagged_running);
@@ -6356,7 +7222,22 @@ test_dir_format_versions_list(void *arg)
 
 struct testcase_t dir_tests[] = {
   DIR_LEGACY(nicknames),
-  DIR_LEGACY(formats),
+  /* extrainfo without any stats */
+  DIR_ARG(formats_rsa, TT_FORK, ""),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, ""),
+  /* on a bridge */
+  DIR_ARG(formats_rsa, TT_FORK, "b"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "b"),
+  /* extrainfo with basic stats */
+  DIR_ARG(formats_rsa, TT_FORK, "e"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "e"),
+  DIR_ARG(formats_rsa, TT_FORK, "be"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "be"),
+  /* extrainfo with all stats */
+  DIR_ARG(formats_rsa, TT_FORK, "es"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "es"),
+  DIR_ARG(formats_rsa, TT_FORK, "bes"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "bes"),
   DIR(routerinfo_parsing, 0),
   DIR(extrainfo_parsing, 0),
   DIR(parse_router_list, TT_FORK),
@@ -6370,6 +7251,7 @@ struct testcase_t dir_tests[] = {
   DIR_LEGACY(measured_bw_kb_line_is_after_headers),
   DIR_LEGACY(measured_bw_kb_cache),
   DIR_LEGACY(dirserv_read_measured_bandwidths),
+  DIR(bwauth_bw_file_digest256, 0),
   DIR_LEGACY(param_voting),
   DIR(param_voting_lookup, 0),
   DIR_LEGACY(v3_networkstatus),
@@ -6410,6 +7292,7 @@ struct testcase_t dir_tests[] = {
   DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"),
   DIR_ARG(find_dl_min_delay, TT_FORK, "car"),
   DIR(assumed_flags, 0),
+  DIR(matching_flags, 0),
   DIR(networkstatus_compute_bw_weights_v10, 0),
   DIR(platform_str, 0),
   DIR(networkstatus_consensus_has_ipv6, TT_FORK),
diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c
index 3723d6c31b..0b87e29873 100644
--- a/src/test/test_dir_common.c
+++ b/src/test/test_dir_common.c
@@ -42,14 +42,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1,
 {
   /* Parse certificates and keys. */
   authority_cert_t *cert;
-  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                          strlen(AUTHORITY_CERT_1),
+                                          NULL);
   tt_assert(cert);
   tt_assert(cert->identity_key);
   *cert1 = cert;
   tt_assert(*cert1);
-  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                            strlen(AUTHORITY_CERT_2),
+                                            NULL);
   tt_assert(*cert2);
-  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                            strlen(AUTHORITY_CERT_3),
+                                            NULL);
   tt_assert(*cert3);
   *sign_skey_1 = crypto_pk_new();
   *sign_skey_2 = crypto_pk_new();
@@ -266,7 +272,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out,
   /* dump the vote and try to parse it. */
   v_text = format_networkstatus_vote(sign_skey, vote);
   tt_assert(v_text);
-  *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE);
+  *vote_out = networkstatus_parse_vote_from_string(v_text,
+                                                   strlen(v_text),
+                                                   NULL, NS_TYPE_VOTE);
 
  done:
   if (v_text)
@@ -424,4 +432,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert,
 
   return 0;
 }
-
diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h
index d6c5241b14..619dc83eb9 100644
--- a/src/test/test_dir_common.h
+++ b/src/test/test_dir_common.h
@@ -3,6 +3,9 @@
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_DIR_COMMON_H
+#define TOR_TEST_DIR_COMMON_H
+
 #include "core/or/or.h"
 #include "feature/nodelist/networkstatus.h"
 
@@ -49,3 +52,4 @@ int dir_common_construct_vote_3(networkstatus_t **vote,
                         networkstatus_t **vote_out, int *n_vrs, time_t now,
                         int clear_rl);
 
+#endif /* !defined(TOR_TEST_DIR_COMMON_H) */
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index 90691fff94..edfd0c74e1 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -72,6 +72,8 @@ ENABLE_GCC_WARNING(overlength-strings)
 #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \
   "Consensus not signed by sufficient number of requested authorities\r\n\r\n"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
 static dir_connection_t *
 new_dir_conn(void)
 {
@@ -477,8 +479,7 @@ static or_options_t *mock_options = NULL;
 static void
 init_mock_options(void)
 {
-  mock_options = tor_malloc(sizeof(or_options_t));
-  memset(mock_options, 0, sizeof(or_options_t));
+  mock_options = options_new();
   mock_options->TestingTorNetwork = 1;
   mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp"));
   mock_options->CacheDirectory = tor_strdup(mock_options->DataDirectory);
@@ -1275,7 +1276,9 @@ test_dir_handle_get_server_keys_authority(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
@@ -1425,7 +1428,9 @@ test_dir_handle_get_server_keys_sk(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
@@ -2217,6 +2222,31 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data)
     tor_free(header);
 }
 
+static void
+test_dir_handle_get_status_vote_next_bandwidth_not_found(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL;
+  (void) data;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwdith"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(NOT_FOUND, OP_EQ, header);
+
+  done:
+    UNMOCK(connection_write_to_buf_impl_);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+}
+
 NS_DECL(const char*,
 dirvote_get_pending_consensus, (consensus_flavor_t flav));
 
@@ -2393,7 +2423,9 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2455,6 +2487,85 @@ test_dir_handle_get_status_vote_next_authority(void* data)
 }
 
 static void
+test_dir_handle_get_status_vote_next_bandwidth(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL, *body = NULL;
+  size_t body_used = 0;
+  (void) data;
+
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  init_mock_options();
+  MOCK(get_options, mock_get_options);
+  mock_options->V3BandwidthsFile = tor_strdup(
+    get_fname_rnd("V3BandwidthsFile")
+  );
+
+  write_str_to_file(mock_options->V3BandwidthsFile, content, 0);
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Type: text/plain\r\n"));
+  tt_assert(strstr(header, "Content-Encoding: identity\r\n"));
+  tt_assert(strstr(header, "Content-Length: 167\r\n"));
+
+  /* Check cache lifetime */
+  char expbuf[RFC1123_TIME_LEN+1];
+  time_t now = approx_time();
+  /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */
+  format_rfc1123_time(expbuf, (time_t)(now + 30*60));
+  char *expires = NULL;
+  /* Change to 'Cache-control: max-age=%d' if using http/1.1. */
+  tor_asprintf(&expires, "Expires: %s\r\n", expbuf);
+  tt_assert(strstr(header, expires));
+
+  tt_int_op(body_used, OP_EQ, strlen(body));
+  tt_str_op(content, OP_EQ, body);
+
+  tor_free(header);
+  tor_free(body);
+
+  /* Request the file using compression, the result should be the same. */
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth.z"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Encoding: deflate\r\n"));
+
+  /* Since using connection_write_to_buf_mock instead of mocking
+   * connection_buf_add_compress, the content is not actually compressed.
+   * If it would, the size and content would be different than the original.
+  */
+
+ done:
+  UNMOCK(get_options);
+  UNMOCK(connection_write_to_buf_impl_);
+  connection_free_minimal(TO_CONN(conn));
+  tor_free(header);
+  tor_free(body);
+  tor_free(expires);
+  or_options_free(mock_options);
+}
+
+static void
 test_dir_handle_get_status_vote_current_authority(void* data)
 {
   dir_connection_t *conn = NULL;
@@ -2471,7 +2582,9 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2627,6 +2740,8 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_current_authority, 0),
   DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_authority, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth_not_found, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth, 0),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK),
diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c
new file mode 100644
index 0000000000..a62c18e0c9
--- /dev/null
+++ b/src/test/test_dispatch.c
@@ -0,0 +1,278 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static dispatch_t *dispatcher_in_use=NULL;
+
+static void
+test_dispatch_max_in_u16_sl(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  uint16_t nums[] = { 10, 20, 30 };
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[1]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[0]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[2]);
+  tt_int_op(30, OP_EQ, max_in_u16_sl(sl, -1));
+
+ done:
+  smartlist_free(sl);
+}
+
+/* Construct an empty dispatch_t. */
+static void
+test_dispatch_empty(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+
+  cfg = dcfg_new();
+  d = dispatch_new(cfg);
+  tt_assert(d);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+static int total_recv1_simple = 0;
+static int total_recv2_simple = 0;
+
+static void
+simple_recv1(const msg_t *m)
+{
+  total_recv1_simple += m->aux_data__.u64;
+}
+
+static char *recv2_received = NULL;
+
+static void
+simple_recv2(const msg_t *m)
+{
+  tor_free(recv2_received);
+  recv2_received = dispatch_fmt_msg_data(dispatcher_in_use, m);
+
+  total_recv2_simple += m->aux_data__.u64*10;
+}
+
+/* Construct a dispatch_t with two messages, make sure that they both get
+ * delivered. */
+static void
+test_dispatch_simple(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  r += dcfg_add_recv(cfg,0,1,simple_recv1);
+  r += dcfg_msg_set_type(cfg,1,0);
+  r += dcfg_msg_set_chan(cfg,1,0);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2); /* second copy */
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = {.u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 7);
+  tt_int_op(total_recv2_simple, OP_EQ, 0);
+
+  total_recv1_simple = 0;
+  r = dispatch_send(d, 99, 0, 1, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+  tt_int_op(total_recv2_simple, OP_EQ, 140);
+
+  tt_str_op(recv2_received, OP_EQ, "<>"); // no format function was set.
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(recv2_received);
+}
+
+/* Construct a dispatch_t with a message and no reciever; make sure that it
+ * gets dropped properly. */
+static void
+test_dispatch_no_recipient(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = { .u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+struct coord { int x; int y; };
+static void
+free_coord(msg_aux_data_t d)
+{
+  tor_free(d.ptr);
+}
+static char *
+fmt_coord(msg_aux_data_t d)
+{
+  char *v;
+  struct coord *c = d.ptr;
+  tor_asprintf(&v, "[%d, %d]", c->x, c->y);
+  return v;
+}
+static dispatch_typefns_t coord_fns = {
+  .fmt_fn = fmt_coord,
+  .free_fn = free_coord,
+};
+static void
+alert_run_immediate(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)arg;
+  dispatch_flush(d, ch, INT_MAX);
+}
+
+static char *received_data=NULL;
+
+static void
+recv_typed_data(const msg_t *m)
+{
+  tor_free(received_data);
+  received_data = dispatch_fmt_msg_data(dispatcher_in_use, m);
+}
+
+static void
+test_dispatch_with_types(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,5,3);
+  r += dcfg_msg_set_chan(cfg,5,2);
+  r += dcfg_add_recv(cfg,5,0,recv_typed_data);
+  r += dcfg_type_set_fns(cfg,3,&coord_fns);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  /* Make this message get run immediately. */
+  r = dispatch_set_alert_fn(d, 2, alert_run_immediate, NULL);
+  tt_int_op(r, OP_EQ, 0);
+
+  struct coord *xy = tor_malloc(sizeof(*xy));
+  xy->x = 13;
+  xy->y = 37;
+  msg_aux_data_t data = {.ptr = xy};
+  r = dispatch_send(d, 99/*sender*/, 2/*channel*/, 5/*msg*/, 3/*type*/, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_str_op(received_data, OP_EQ, "[13, 37]");
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(received_data);
+  dispatcher_in_use = NULL;
+}
+
+static void
+test_dispatch_bad_type_setup(void *arg)
+{
+  (void)arg;
+  static dispatch_typefns_t fns;
+  dispatch_cfg_t *cfg = dcfg_new();
+
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &coord_fns));
+
+  fns = coord_fns;
+  fns.fmt_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  fns.free_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+ done:
+  dcfg_free(cfg);
+}
+
+#define T(name)                                                 \
+  { #name, test_dispatch_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t dispatch_tests[] = {
+  T(max_in_u16_sl),
+  T(empty),
+  T(simple),
+  T(no_recipient),
+  T(with_types),
+  T(bad_type_setup),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_dns.c b/src/test/test_dns.c
index 41a56f65d8..51ff8729d0 100644
--- a/src/test/test_dns.c
+++ b/src/test/test_dns.c
@@ -1,6 +1,7 @@
 /* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#include "orconfig.h"
 #include "core/or/or.h"
 #include "test/test.h"
 
@@ -13,9 +14,71 @@
 
 #include "core/or/edge_connection_st.h"
 #include "core/or/or_circuit_st.h"
+#include "app/config/or_options_st.h"
+#include "app/config/config.h"
+
+#include <event2/event.h>
+#include <event2/dns.h>
 
 #define NS_MODULE dns
 
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+#define NS_SUBMODULE configure_nameservers_fallback
+
+static or_options_t options = {
+  .ORPort_set = 1,
+};
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &options;
+}
+
+static void
+NS(test_main)(void *arg)
+{
+  (void)arg;
+  tor_addr_t *nameserver_addr = NULL;
+
+  MOCK(get_options, mock_get_options);
+
+  options.ServerDNSResolvConfFile = (char *)"no_such_file!!!";
+
+  dns_init(); // calls configure_nameservers()
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+
+#ifndef _WIN32
+  tor_free(nameserver_addr);
+
+  options.ServerDNSResolvConfFile = (char *)"/dev/null";
+
+  dns_init();
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+#endif /* !defined(_WIN32) */
+
+  UNMOCK(get_options);
+
+ done:
+  tor_free(nameserver_addr);
+  return;
+}
+
+#undef NS_SUBMODULE
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
 #define NS_SUBMODULE clip_ttl
 
 static void
@@ -736,6 +799,9 @@ NS(test_main)(void *arg)
 #undef NS_SUBMODULE
 
 struct testcase_t dns_tests[] = {
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+   TEST_CASE(configure_nameservers_fallback),
+#endif
    TEST_CASE(clip_ttl),
    TEST_CASE(resolve),
    TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve),
diff --git a/src/test/test_dos.c b/src/test/test_dos.c
index 01d7cd006e..c17cfedbf6 100644
--- a/src/test/test_dos.c
+++ b/src/test/test_dos.c
@@ -411,7 +411,7 @@ test_dos_bucket_refill(void *arg)
   }
   tt_uint_op(current_circ_count, OP_EQ, 0);
   tt_uint_op(dos_stats->cc_stats.circuit_bucket, OP_EQ, current_circ_count);
-#endif
+#endif /* SIZEOF_TIME_T == 8 */
 
  done:
   tor_free(chan);
diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c
index fc7c5d5800..8f2d507743 100644
--- a/src/test/test_entryconn.c
+++ b/src/test/test_entryconn.c
@@ -11,7 +11,7 @@
 
 #include "feature/client/addressmap.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "feature/nodelist/nodelist.h"
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index a486b13ae1..d59b1c7153 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -5,6 +5,7 @@
 
 #define CIRCUITLIST_PRIVATE
 #define CIRCUITBUILD_PRIVATE
+#define CONFIG_PRIVATE
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
@@ -17,7 +18,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dirclient/dirclient.h"
@@ -67,7 +68,7 @@ static networkstatus_t *dummy_consensus = NULL;
 
 static smartlist_t *big_fake_net_nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 bfn_mock_nodelist_get_list(void)
 {
   return big_fake_net_nodes;
@@ -127,6 +128,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
   return 1; /* NOP */
 }
 
+#define REASONABLY_FUTURE " reasonably-future"
+#define REASONABLY_PAST " reasonably-past"
+
 /* Unittest setup function: Setup a fake network. */
 static void *
 big_fake_network_setup(const struct testcase_t *testcase)
@@ -138,9 +142,10 @@ big_fake_network_setup(const struct testcase_t *testcase)
   const int N_NODES = 271;
 
   const char *argument = testcase->setup_data;
-  int reasonably_live_consensus = 0;
+  int reasonably_future_consensus = 0, reasonably_past_consensus = 0;
   if (argument) {
-    reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL;
+    reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL;
+    reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL;
   }
 
   big_fake_net_nodes = smartlist_new();
@@ -193,16 +198,21 @@ big_fake_network_setup(const struct testcase_t *testcase)
       n->md->exit_policy = parse_short_policy("accept 443");
     }
 
+    n->nodelist_idx = smartlist_len(big_fake_net_nodes);
     smartlist_add(big_fake_net_nodes, n);
   }
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
   dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
-  if (reasonably_live_consensus) {
-    /* Make the dummy consensus valid from 4 hours ago, but expired an hour
+  if (reasonably_future_consensus) {
+    /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */
+    dummy_consensus->valid_after = approx_time() + 6*3600;
+    dummy_consensus->valid_until = approx_time() + 7*3600;
+  } else if (reasonably_past_consensus) {
+    /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours
      * ago. */
-    dummy_consensus->valid_after = approx_time() - 4*3600;
-    dummy_consensus->valid_until = approx_time() - 3600;
+    dummy_consensus->valid_after = approx_time() - 16*3600;
+    dummy_consensus->valid_until = approx_time() - 12*3600;
   } else {
     /* Make the dummy consensus valid for an hour either side of now. */
     dummy_consensus->valid_after = approx_time() - 3600;
@@ -226,12 +236,12 @@ mock_randomize_time_no_randomization(time_t a, time_t b)
   return a;
 }
 
-static or_options_t mocked_options;
+static or_options_t *mocked_options;
 
 static const or_options_t *
 mock_get_options(void)
 {
-  return &mocked_options;
+  return mocked_options;
 }
 
 #define TEST_IPV4_ADDR "123.45.67.89"
@@ -250,7 +260,7 @@ test_node_preferred_orport(void *arg)
   tor_addr_port_t ap;
 
   /* Setup options */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  mocked_options = options_new();
   /* We don't test ClientPreferIPv6ORPort here, because it's used in
    * nodelist_set_consensus to setup node.ipv6_preferred, which we set
    * directly. */
@@ -273,8 +283,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're only using IPv4, regardless
    * of whether we prefer it or not */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 0;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 0;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -287,8 +297,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
    * don't prefer the IPv6 address */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -296,28 +306,29 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv6 if we prefer it and
    * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 1;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
-  mocked_options.ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv4 = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
   /* Check the preferred address is IPv6 if we don't prefer it, but
    * ClientUseIPv4 is 0 */
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
+  mocked_options->ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv6 = 1;
+  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(mocked_options);
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
  done:
+  or_options_free(mocked_options);
   UNMOCK(get_options);
 }
 
@@ -3039,13 +3050,17 @@ static const struct testcase_setup_t upgrade_circuits = {
 
 #define BFN_TEST(name) \
   EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &big_fake_network, (void*)("reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_PAST) }
 
 #define UPGRADE_TEST(name, arg) \
   EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &upgrade_circuits, (void*)(arg " reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_PAST) }
 
 struct testcase_t entrynodes_tests[] = {
   NO_PREFIX_TEST(node_preferred_orport),
diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c
index 0c34d37a71..cb53a4e662 100644
--- a/src/test/test_extorport.c
+++ b/src/test/test_extorport.c
@@ -5,11 +5,11 @@
 #define EXT_ORPORT_PRIVATE
 #define MAINLOOP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/relay/ext_orport.h"
 #include "core/mainloop/mainloop.h"
@@ -18,6 +18,7 @@
 
 #include "test/test.h"
 #include "test/test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -176,7 +177,7 @@ test_ext_or_init_auth(void *arg)
   /* Shouldn't be initialized already, or our tests will be a bit
    * meaningless */
   ext_or_auth_cookie = tor_malloc_zero(32);
-  tt_assert(tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Now make sure we use a temporary file */
   fn = get_fname("ext_cookie_file");
@@ -201,7 +202,7 @@ test_ext_or_init_auth(void *arg)
   tt_mem_op(cp,OP_EQ, "! Extended ORPort Auth Cookie !\x0a", 32);
   tt_mem_op(cp+32,OP_EQ, ext_or_auth_cookie, 32);
   memcpy(cookie0, ext_or_auth_cookie, 32);
-  tt_assert(!tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(!fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Operation should be idempotent. */
   tt_int_op(0, OP_EQ, init_ext_or_cookie_authentication(1));
@@ -303,16 +304,6 @@ test_ext_or_cookie_auth(void *arg)
 }
 
 static void
-crypto_rand_return_tse_str(char *to, size_t n)
-{
-  if (n != 32) {
-    TT_FAIL(("Asked for %d bytes, not 32", (int)n));
-    return;
-  }
-  memcpy(to, "te road There is always another ", 32);
-}
-
-static void
 test_ext_or_cookie_auth_testvec(void *arg)
 {
   char *reply=NULL, *client_hash=NULL;
@@ -326,7 +317,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
   memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32);
   ext_or_auth_cookie_is_set = 1;
 
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
 
   tt_int_op(0, OP_EQ,
             handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply,
@@ -351,7 +342,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
                  "33b3cd77ff79bd80c2074bbf438119a2");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(reply);
   tor_free(client_hash);
   tor_free(mem_op_hex_tmp);
@@ -414,9 +405,9 @@ do_ext_or_handshake(or_connection_t *conn)
   CONTAINS("\x01\x00", 2);
   WRITE("\x01", 1);
   WRITE("But when I look ahead up the whi", 32);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tt_int_op(TO_CONN(conn)->state, OP_EQ,
             EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH);
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
@@ -481,9 +472,9 @@ test_ext_or_handshake(void *arg)
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
   /* send the rest of the nonce. */
   WRITE("ahead up the whi", 16);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   /* We should get the right reply from the server. */
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
            "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21"
@@ -582,7 +573,7 @@ test_ext_or_handshake(void *arg)
 
  done:
   UNMOCK(connection_write_to_buf_impl_);
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   if (conn)
     connection_free_minimal(TO_CONN(conn));
 #undef CONTAINS
@@ -596,6 +587,6 @@ struct testcase_t extorport_tests[] = {
   { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL },
   { "cookie_auth_testvec", test_ext_or_cookie_auth_testvec, TT_FORK,
     NULL, NULL },
-  { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL },
+  { "handshake", test_ext_or_handshake, TT_FORK, &helper_pubsub_setup, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c
index 802d0a9ebe..8eb3c2c928 100644
--- a/src/test/test_helpers.c
+++ b/src/test/test_helpers.c
@@ -14,15 +14,20 @@
 #include "orconfig.h"
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/nodelist.h"
 #include "core/or/relay.h"
 #include "feature/nodelist/routerlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
 
@@ -78,7 +83,7 @@ helper_setup_fake_routerlist(void)
 {
   int retval;
   routerlist_t *our_routerlist = NULL;
-  smartlist_t *our_nodelist = NULL;
+  const smartlist_t *our_nodelist = NULL;
 
   /* Read the file that contains our test descriptors. */
 
@@ -290,7 +295,7 @@ helper_parse_options(const char *conf)
   if (ret != 0) {
     goto done;
   }
-  ret = config_assign(&options_format, opt, line, 0, &msg);
+  ret = config_assign(get_options_mgr(), opt, line, 0, &msg);
   if (ret != 0) {
     goto done;
   }
@@ -303,3 +308,54 @@ helper_parse_options(const char *conf)
   }
   return opt;
 }
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Setup helper for tests that need pubsub active
+ *
+ * Does not hook up mainloop events.  Does set immediate delivery for
+ * all channels.
+ */
+void *
+helper_setup_pubsub(const struct testcase_t *testcase)
+{
+  dispatch_t *dispatcher = NULL;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  channel_id_t chan = get_channel_id("orconn");
+
+  (void)testcase;
+  (void)subsystems_add_pubsub(builder);
+  dispatcher = pubsub_builder_finalize(builder, NULL);
+  tor_assert(dispatcher);
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  chan = get_channel_id("ocirc");
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  return dispatcher;
+}
+
+/**
+ * Cleanup helper for tests that need pubsub active
+ */
+int
+helper_cleanup_pubsub(const struct testcase_t *testcase, void *dispatcher_)
+{
+  dispatch_t *dispatcher = dispatcher_;
+
+  (void)testcase;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+const struct testcase_setup_t helper_pubsub_setup = {
+  helper_setup_pubsub, helper_cleanup_pubsub
+};
diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h
index 9e376a563d..d82072bb34 100644
--- a/src/test/test_helpers.h
+++ b/src/test/test_helpers.h
@@ -7,6 +7,7 @@
 #define BUFFERS_PRIVATE
 
 #include "core/or/or.h"
+#include "tinytest.h"
 
 const char *get_yesterday_date_str(void);
 
@@ -31,5 +32,10 @@ or_options_t *helper_parse_options(const char *conf);
 
 extern const char TEST_DESCRIPTORS[];
 
+void *helper_setup_pubsub(const struct testcase_t *);
+int helper_cleanup_pubsub(const struct testcase_t *, void *);
+
+extern const struct testcase_setup_t helper_pubsub_setup;
+
 #endif /* !defined(TOR_TEST_HELPERS_H) */
 
diff --git a/src/test/test_hs.c b/src/test/test_hs.c
index a611b46ca6..2b69aae547 100644
--- a/src/test/test_hs.c
+++ b/src/test/test_hs.c
@@ -6,7 +6,7 @@
  * \brief Unit tests for hidden service.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define RENDCOMMON_PRIVATE
 #define RENDSERVICE_PRIVATE
@@ -15,6 +15,8 @@
 #include "core/or/or.h"
 #include "test/test.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
 #include "feature/rend/rendcommon.h"
@@ -321,6 +323,16 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg,OP_EQ, expected_msg);
   tor_free(received_msg);
 
+  /* test HSDir rate limited */
+  rend_query.auth_type = REND_NO_AUTH;
+  control_event_hsv2_descriptor_failed(&rend_query.base_, NULL,
+                                     "QUERY_RATE_LIMITED");
+  expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \
+                 "UNKNOWN REASON=QUERY_RATE_LIMITED\r\n";
+  tt_assert(received_msg);
+  tt_str_op(received_msg,OP_EQ, expected_msg);
+  tor_free(received_msg);
+
   /* Test invalid content with no HSDir fingerprint. */
   char *exp_msg;
   control_event_hs_descriptor_content(rend_query.onion_address,
@@ -436,7 +448,7 @@ test_hs_rend_data(void *arg)
   tt_int_op(client_v2->auth_type, OP_EQ, REND_BASIC_AUTH);
   tt_int_op(strlen(client_v2->onion_address), OP_EQ, 0);
   tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie,
+  tt_int_op(fast_mem_is_zero(client_v2->descriptor_cookie,
                             sizeof(client_v2->descriptor_cookie)), OP_EQ, 1);
   tt_assert(client->hsdirs_fp);
   tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
index 9182829116..86ac7e7fb1 100644
--- a/src/test/test_hs_cache.c
+++ b/src/test/test_hs_cache.c
@@ -10,6 +10,7 @@
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
 #define HS_CACHE_PRIVATE
+#define TOR_CHANNEL_INTERNAL_
 
 #include "trunnel/ed25519_cert.h"
 #include "feature/hs/hs_cache.h"
@@ -20,7 +21,12 @@
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_http.h"
 #include "lib/crypt_ops/crypto_format.h"
+#include "core/or/circuitlist.h"
+#include "core/or/channel.h"
 
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 
@@ -232,22 +238,31 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
   /* The dir conn we are going to simulate */
   dir_connection_t *conn = NULL;
+  edge_connection_t *edge_conn = NULL;
+  or_circuit_t *or_circ = NULL;
 
   /* First extract the blinded public key that we are going to use in our
      query, and then build the actual query string. */
   {
     char hsdir_cache_key[ED25519_BASE64_LEN+1];
 
-    retval = ed25519_public_to_base64(hsdir_cache_key,
-                                      blinded_key);
-    tt_int_op(retval, OP_EQ, 0);
+    ed25519_public_to_base64(hsdir_cache_key, blinded_key);
     tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key);
   }
 
   /* Simulate an HTTP GET request to the HSDir */
   conn = dir_connection_new(AF_INET);
+  tt_assert(conn);
+  TO_CONN(conn)->linked = 1; /* Signal that it is encrypted. */
   tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
-  TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */
+
+  /* Pretend this conn is anonymous. */
+  edge_conn = edge_connection_new(CONN_TYPE_EXIT, AF_INET);
+  TO_CONN(conn)->linked_conn = TO_CONN(edge_conn);
+  or_circ = or_circuit_new(0, NULL);
+  or_circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+  edge_conn->on_circuit = TO_CIRCUIT(or_circ);
+
   retval = directory_handle_command_get(conn, hsdir_query_str,
                                         NULL, 0);
   tt_int_op(retval, OP_EQ, 0);
@@ -264,8 +279,11 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
  done:
   tor_free(hsdir_query_str);
-  if (conn)
+  if (conn) {
+    tor_free(or_circ->p_chan);
+    connection_free_minimal(TO_CONN(conn)->linked_conn);
     connection_free_minimal(TO_CONN(conn));
+  }
 
   return received_desc;
 }
@@ -487,7 +505,7 @@ test_client_cache(void *arg)
                                        NULL, &published_desc_str);
     tt_int_op(retval, OP_EQ, 0);
     memcpy(wanted_subcredential, published_desc->subcredential, DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
+    tt_assert(!fast_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
   }
 
   /* Test handle_response_fetch_hsdesc_v3() */
diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c
index f8af631c8b..403509fbc8 100644
--- a/src/test/test_hs_cell.c
+++ b/src/test/test_hs_cell.c
@@ -20,6 +20,7 @@
 #include "feature/hs/hs_service.h"
 
 /* Trunnel. */
+#include "trunnel/hs/cell_common.h"
 #include "trunnel/hs/cell_establish_intro.h"
 
 /** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we
@@ -38,11 +39,13 @@ test_gen_establish_intro_cell(void *arg)
   /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
      attempt to parse it. */
   {
+    hs_service_config_t config;
+    memset(&config, 0, sizeof(config));
     /* We only need the auth key pair here. */
-    hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+    hs_service_intro_point_t *ip = service_intro_point_new(NULL);
     /* Auth key pair is generated in the constructor so we are all set for
      * using this IP object. */
-    ret = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+    ret = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
     service_intro_point_free(ip);
     tt_u64_op(ret, OP_GT, 0);
   }
@@ -97,6 +100,9 @@ test_gen_establish_intro_cell_bad(void *arg)
   trn_cell_establish_intro_t *cell = NULL;
   char circ_nonce[DIGEST_LEN] = {0};
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed);
 
@@ -107,8 +113,8 @@ test_gen_establish_intro_cell_bad(void *arg)
      ed25519_sign_prefixed() function and make it fail. */
   cell = trn_cell_establish_intro_new();
   tt_assert(cell);
-  ip = service_intro_point_new(NULL, 0, 0);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL);
+  ip = service_intro_point_new(NULL);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, NULL);
   service_intro_point_free(ip);
   expect_log_msg_containing("Unable to make signature for "
                             "ESTABLISH_INTRO cell.");
@@ -120,11 +126,97 @@ test_gen_establish_intro_cell_bad(void *arg)
   UNMOCK(ed25519_sign_prefixed);
 }
 
+static void
+test_gen_establish_intro_dos_ext(void *arg)
+{
+  ssize_t ret;
+  hs_service_config_t config;
+  hs_service_intro_point_t *ip = NULL;
+  trn_cell_extension_t *extensions = NULL;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  (void) arg;
+
+  memset(&config, 0, sizeof(config));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+
+  /* Case 1: No DoS parameters so no extension to be built. */
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 0);
+  trn_cell_extension_free(extensions);
+  extensions = NULL;
+
+  /* Case 2: Enable the DoS extension. Parameter set to 0 should indicate to
+   * disable the defense on the intro point but there should be an extension
+   * nonetheless in the cell. */
+  config.has_dos_defense_enabled = 1;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  const trn_cell_extension_field_t *field =
+    trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  const trn_cell_extension_dos_param_t *param =
+    trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+  /* Case 3: Enable the DoS extension. Parameter set to some normal values. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 42;
+  config.intro_dos_burst_per_sec = 250;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  field = trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 42);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 250);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+ done:
+  service_intro_point_free(ip);
+  trn_cell_extension_dos_free(dos);
+  trn_cell_extension_free(extensions);
+}
+
 struct testcase_t hs_cell_tests[] = {
   { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK,
     NULL, NULL },
   { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK,
     NULL, NULL },
+  { "gen_establish_intro_dos_ext", test_gen_establish_intro_dos_ext, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c
index 2f2bb45581..b777dafdfb 100644
--- a/src/test/test_hs_client.c
+++ b/src/test/test_hs_client.c
@@ -14,6 +14,7 @@
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -36,6 +37,7 @@
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_cache.h"
+#include "feature/rend/rendcache.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "core/mainloop/connection.h"
@@ -44,6 +46,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "core/or/entry_connection_st.h"
 #include "core/or/extend_info_st.h"
@@ -157,8 +160,7 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out,
     or_circ->rend_data = rend_data_dup(conn_rend_data);
   } else {
     /* prop224: Setup hs ident on the circuit */
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
   }
 
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -241,12 +243,14 @@ test_e2e_rend_circuit_setup_legacy(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -311,12 +315,14 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check that the crypt path has prop224 algorithm parameters */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -395,7 +401,7 @@ test_client_pick_intro(void *arg)
     tt_assert(fetched_desc);
     tt_mem_op(fetched_desc->subcredential, OP_EQ, desc->subcredential,
               DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)fetched_desc->subcredential,
+    tt_assert(!fast_mem_is_zero((char*)fetched_desc->subcredential,
                                DIGEST256_LEN));
     tor_free(encoded);
   }
@@ -403,6 +409,9 @@ test_client_pick_intro(void *arg)
   /* 2) Mark all intro points except _the chosen one_ as failed. Then query the
    *   desc and get a random intro: check that we got _the chosen one_. */
   {
+    /* Tell hs_get_extend_info_from_lspecs() to skip the private address check.
+     */
+    get_options_mutable()->ExtendAllowPrivateAddresses = 1;
     /* Pick the chosen intro point and get its ei */
     hs_desc_intro_point_t *chosen_intro_point =
       smartlist_get(desc->encrypted_data.intro_points, 0);
@@ -430,7 +439,7 @@ test_client_pick_intro(void *arg)
     for (int i = 0; i < 64; ++i) {
       extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
       tor_assert(ip);
-      tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
+      tt_assert(!fast_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
       tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest,
                 DIGEST_LEN);
       extend_info_free(ip);
@@ -476,6 +485,18 @@ test_client_pick_intro(void *arg)
     SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                             hs_desc_intro_point_t *, ip) {
       extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip);
+      /* desc_intro_point_to_extend_info() doesn't return IPv6 intro points
+       * yet, because we can't extend to them. See #24404, #24451, and #24181.
+       */
+      if (intro_ei == NULL) {
+        /* Pretend we're making a direct connection, and that we can use IPv6
+         */
+        get_options_mutable()->ClientUseIPv6 = 1;
+        intro_ei = hs_get_extend_info_from_lspecs(ip->link_specifiers,
+                                                  &ip->onion_key, 1);
+        tt_assert(tor_addr_family(&intro_ei->addr) == AF_INET6);
+      }
+      tt_assert(intro_ei);
       if (intro_ei) {
         const char *ptr;
         char ip_addr[TOR_ADDR_BUF_LEN];
@@ -942,8 +963,7 @@ test_close_intro_circuits_new_desc(void *arg)
     const hs_desc_intro_point_t *ip =
       smartlist_get(desc1->encrypted_data.intro_points, 0);
     tt_assert(ip);
-    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey,
-                                           HS_IDENT_CIRCUIT_INTRO);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
     ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
                         &ip->auth_key_cert->signed_key);
   }
@@ -986,6 +1006,91 @@ test_close_intro_circuits_new_desc(void *arg)
   UNMOCK(networkstatus_get_live_consensus);
 }
 
+static void
+test_close_intro_circuits_cache_clean(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  circuit_t *circ = NULL;
+  origin_circuit_t *ocirc = NULL;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  hs_init();
+  rend_cache_init();
+
+  /* This is needed because of the client cache expiration timestamp is based
+   * on having a consensus. See cached_client_descriptor_has_expired(). */
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Create and add to the global list a dummy client introduction circuits.
+   * We'll then make sure the hs_ident is attached to a dummy descriptor. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+  /* Build the first descriptor and cache it. */
+  {
+    char *encoded;
+    desc1 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc1);
+    ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* Store it */
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, 0);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* We'll pick one introduction point and associate it with the circuit. */
+  {
+    const hs_desc_intro_point_t *ip =
+      smartlist_get(desc1->encrypted_data.intro_points, 0);
+    tt_assert(ip);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+    ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+  }
+
+  /* Before we are about to clean up the intro circuits, make sure it is
+   * actually there. */
+  tt_assert(circuit_get_next_intro_circ(NULL, true));
+
+  /* Cleanup the client cache. The ns valid after time is what decides if the
+   * descriptor has expired so put it in the future enough (72h) so we are
+   * sure to always expire. */
+  mock_ns.valid_after = approx_time() + (72 * 24 * 60 * 60);
+  hs_cache_clean_as_client(0);
+
+  /* Once stored, our intro circuit should be closed because it is related to
+   * an old introduction point that doesn't exists anymore. */
+  tt_assert(!circuit_get_next_intro_circ(NULL, true));
+
+ done:
+  circuit_free(circ);
+  hs_descriptor_free(desc1);
+  hs_free_all();
+  rend_cache_free_all();
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
 struct testcase_t hs_client_tests[] = {
   { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy,
     TT_FORK, NULL, NULL },
@@ -1005,6 +1110,8 @@ struct testcase_t hs_client_tests[] = {
     TT_FORK, NULL, NULL },
   { "close_intro_circuits_new_desc", test_close_intro_circuits_new_desc,
     TT_FORK, NULL, NULL },
+  { "close_intro_circuits_cache_clean", test_close_intro_circuits_cache_clean,
+    TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c
index 2aff179687..de3f7e04f7 100644
--- a/src/test/test_hs_common.c
+++ b/src/test/test_hs_common.c
@@ -275,7 +275,7 @@ test_start_time_of_next_time_period(void *arg)
 static void
 cleanup_nodelist(void)
 {
-  smartlist_t *nodelist = nodelist_get_list();
+  const smartlist_t *nodelist = nodelist_get_list();
   SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
     tor_free(node->md);
     node->md = NULL;
@@ -502,6 +502,7 @@ test_desc_reupload_logic(void *arg)
                 pubkey_hex, strlen(pubkey_hex));
   hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
   service = tor_malloc_zero(sizeof(hs_service_t));
+  tt_assert(service);
   memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
   ed25519_secret_key_generate(&service->keys.identity_sk, 0);
   ed25519_public_key_generate(&service->keys.identity_pk,
@@ -603,6 +604,10 @@ test_desc_reupload_logic(void *arg)
   SMARTLIST_FOREACH(ns->routerstatus_list,
                     routerstatus_t *, rs, routerstatus_free(rs));
   smartlist_clear(ns->routerstatus_list);
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   networkstatus_vote_free(ns);
   cleanup_nodelist();
   hs_free_all();
@@ -630,7 +635,7 @@ test_disaster_srv(void *arg)
   get_disaster_srv(1, srv_one);
   get_disaster_srv(2, srv_two);
 
-  /* Check that the cached ones where updated */
+  /* Check that the cached ones were updated */
   tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
   tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
 
diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c
index c2c556307d..71e1529216 100644
--- a/src/test/test_hs_config.c
+++ b/src/test/test_hs_config.c
@@ -12,6 +12,7 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
@@ -272,6 +273,7 @@ test_valid_service_v2(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. Basic client authorization. */
   {
@@ -314,7 +316,7 @@ test_valid_service_v2(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -392,6 +394,7 @@ test_valid_service_v3(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. */
   {
@@ -448,7 +451,7 @@ test_valid_service_v3(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -489,6 +492,111 @@ test_staging_service_v3(void *arg)
   hs_free_all();
 }
 
+static void
+test_dos_parameters(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  hs_init();
+
+  /* Valid configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_INFO);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    expect_log_msg_containing("Service INTRO2 DoS defenses rate set to: 42");
+    expect_log_msg_containing("Service INTRO2 DoS defenses burst set to: 87");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid rate. Value of 2^37. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 137438953472\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceEnableIntroDoSRatePerSec must "
+                              "be between 0 and 2147483647, "
+                              "not 137438953472");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid burst. Value of 2^38. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 274877906944\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceEnableIntroDoSBurstPerSec must "
+                              "be between 0 and 2147483647, "
+                              "not 274877906944");
+    teardown_capture_of_logs();
+  }
+
+  /* Burst is smaller than rate. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 27\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Hidden service DoS defenses burst (27) can "
+                              "not be smaller than the rate value (42).");
+    teardown_capture_of_logs();
+  }
+
+  /* Negative value. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec -1\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 42\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceEnableIntroDoSRatePerSec must be "
+                              "between 0 and 2147483647, not -1");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  hs_free_all();
+}
+
 struct testcase_t hs_config_tests[] = {
   /* Invalid service not specific to any version. */
   { "invalid_service", test_invalid_service, TT_FORK,
@@ -512,6 +620,9 @@ struct testcase_t hs_config_tests[] = {
   { "staging_service_v3", test_staging_service_v3, TT_FORK,
     NULL, NULL },
 
+  /* Test HS DoS parameters. */
+  { "dos_parameters", test_dos_parameters, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c
index ba67712f1b..7cedc987bb 100644
--- a/src/test/test_hs_control.c
+++ b/src/test/test_hs_control.c
@@ -6,11 +6,13 @@
  * \brief Unit tests for hidden service control port event and command.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
 
 #include "core/or/or.h"
 #include "test/test.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_control.h"
@@ -105,8 +107,7 @@ test_hs_desc_event(void *arg)
   memset(&blinded_pk, 'B', sizeof(blinded_pk));
   memset(&hsdir_rs, 0, sizeof(hsdir_rs));
   memcpy(hsdir_rs.identity_digest, HSDIR_EXIST_ID, DIGEST_LEN);
-  ret = ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   memcpy(&ident.identity_pk, &identity_kp.pubkey,
          sizeof(ed25519_public_key_t));
   memcpy(&ident.blinded_pk, &blinded_pk, sizeof(blinded_pk));
diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c
index de584ed47a..6fe5573c0f 100644
--- a/src/test/test_hs_descriptor.c
+++ b/src/test/test_hs_descriptor.c
@@ -21,6 +21,7 @@
 #include "test/hs_test_helpers.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 DISABLE_GCC_WARNING(overlength-strings)
@@ -30,13 +31,6 @@ DISABLE_GCC_WARNING(overlength-strings)
 #include "test_hs_descriptor.inc"
 ENABLE_GCC_WARNING(overlength-strings)
 
-/* Mock function to fill all bytes with 1 */
-static void
-mock_crypto_strongest_rand(uint8_t *out, size_t out_len)
-{
-  memset(out, 1, out_len);
-}
-
 /* Test certificate encoding put in a descriptor. */
 static void
 test_cert_encoding(void *arg)
@@ -132,7 +126,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -149,7 +143,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -166,7 +160,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -179,115 +173,6 @@ test_descriptor_padding(void *arg)
 }
 
 static void
-test_link_specifier(void *arg)
-{
-  ssize_t ret;
-  hs_desc_link_specifier_t spec;
-  smartlist_t *link_specifiers = smartlist_new();
-  char buf[256];
-  char *b64 = NULL;
-  link_specifier_t *ls = NULL;
-
-  (void) arg;
-
-  /* Always this port. */
-  spec.u.ap.port = 42;
-  smartlist_add(link_specifiers, &spec);
-
-  /* Test IPv4 for starter. */
-  {
-    uint32_t ipv4;
-
-    spec.type = LS_IPV4;
-    ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4");
-    tt_int_op(ret, OP_EQ, AF_INET);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 8);
-    /* Should be 2 bytes for port and 4 bytes for IPv4. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 6);
-    ipv4 = link_specifier_get_un_ipv4_addr(ls);
-    tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), OP_EQ, ipv4);
-    tt_int_op(link_specifier_get_un_ipv4_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test IPv6. */
-  {
-    uint8_t ipv6[16];
-
-    spec.type = LS_IPV6;
-    ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]");
-    tt_int_op(ret, OP_EQ, AF_INET6);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 20);
-    /* Should be 2 bytes for port and 16 bytes for IPv6. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 18);
-    for (unsigned int i = 0; i < sizeof(ipv6); i++) {
-      ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i);
-    }
-    tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), OP_EQ, ipv6,
-              sizeof(ipv6));
-    tt_int_op(link_specifier_get_un_ipv6_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test legacy. */
-  {
-    uint8_t *id;
-
-    spec.type = LS_LEGACY_ID;
-    memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id));
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    /* 20 bytes digest + 1 byte type + 1 byte len. */
-    tt_int_op(ret, OP_EQ, 22);
-    tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN);
-    /* Digest length is 20 bytes. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN);
-    id = link_specifier_getarray_un_legacy_id(ls);
-    tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
- done:
-  link_specifier_free(ls);
-  tor_free(b64);
-  smartlist_free(link_specifiers);
-}
-
-static void
 test_encode_descriptor(void *arg)
 {
   int ret;
@@ -848,8 +733,7 @@ test_desc_signature(void *arg)
   ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data),
                               "Tor onion service descriptor sig v3", &kp);
   tt_int_op(ret, OP_EQ, 0);
-  ret = ed25519_signature_to_base64(sig_b64, &sig);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_signature_to_base64(sig_b64, &sig);
   /* Build the descriptor that should be valid. */
   tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64);
   ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc));
@@ -909,7 +793,7 @@ test_build_authorized_client(void *arg)
                 client_pubkey_b16,
                 strlen(client_pubkey_b16));
 
-  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+  testing_enable_prefilled_rng("\x01", 1);
 
   hs_desc_build_authorized_client(subcredential,
                                   &client_auth_pk, &auth_ephemeral_sk,
@@ -925,15 +809,13 @@ test_build_authorized_client(void *arg)
  done:
   tor_free(desc_client);
   tor_free(mem_op_hex_tmp);
-  UNMOCK(crypto_strongest_rand_);
+  testing_disable_prefilled_rng();
 }
 
 struct testcase_t hs_descriptor[] = {
   /* Encoding tests. */
   { "cert_encoding", test_cert_encoding, TT_FORK,
     NULL, NULL },
-  { "link_specifier", test_link_specifier, TT_FORK,
-    NULL, NULL },
   { "encode_descriptor", test_encode_descriptor, TT_FORK,
     NULL, NULL },
   { "descriptor_padding", test_descriptor_padding, TT_FORK,
diff --git a/src/test/test_hs_dos.c b/src/test/test_hs_dos.c
new file mode 100644
index 0000000000..f68639e24a
--- /dev/null
+++ b/src/test/test_hs_dos.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cell.c
+ * \brief Test hidden service cell functionality.
+ */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define HS_DOS_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+
+#include "feature/hs/hs_dos.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/nodelist/networkstatus.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_ns_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  current_ns_consensus->net_params = smartlist_new();
+  smartlist_add(current_ns_consensus->net_params,
+                (void *) "HiddenServiceEnableIntroDoSDefense=1");
+  hs_dos_consensus_has_changed(current_ns_consensus);
+}
+
+static void
+free_mock_consensus(void)
+{
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_can_send_intro2(void *arg)
+{
+  uint32_t now = (uint32_t) approx_time();
+  or_circuit_t *or_circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  hs_dos_init();
+
+  get_options_mutable()->ORPort_set = 1;
+  setup_mock_consensus();
+
+  or_circ =  or_circuit_new(1, NULL);
+
+  /* Make that circuit a service intro point. */
+  circuit_change_purpose(TO_CIRCUIT(or_circ), CIRCUIT_PURPOSE_INTRO_POINT);
+  hs_dos_setup_default_intro2_defenses(or_circ);
+  or_circ->introduce2_dos_defense_enabled = 1;
+
+  /* Brand new circuit, we should be able to send INTRODUCE2 cells. */
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+
+  /* Simulate that 10 cells have arrived in 1 second. There should be no
+   * refill since the bucket is already at maximum on the first cell. */
+  update_approx_time(++now);
+  for (int i = 0; i < 10; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 10);
+
+  /* Fully refill the bucket minus 1 cell. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Receive an INTRODUCE2 at each second. We should have the bucket full
+   * since at every second it gets refilled. */
+  for (int i = 0; i < 10; i++) {
+    update_approx_time(++now);
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  /* Last check if we can send the cell decrements the bucket so minus 1. */
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Manually reset bucket for next test. */
+  token_bucket_ctr_reset(&or_circ->introduce2_bucket, now);
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL));
+
+  /* Do a full burst in the current second which should empty the bucket and
+   * we shouldn't be allowed to send one more cell after that. We go minus 1
+   * cell else the very last check if we can send the INTRO2 cell returns
+   * false because the bucket goes down to 0. */
+  for (uint32_t i = 0; i < get_intro2_burst_consensus_param(NULL) - 1; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 1);
+  /* Get the last remaining cell, we shouldn't be allowed to send it. */
+  tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+
+  /* Make sure the next 100 cells aren't allowed and bucket stays at 0. */
+  for (int i = 0; i < 100; i++) {
+    tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+    tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+  }
+
+  /* One second has passed, we should have the rate minus 1 cell added. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_rate_consensus_param(NULL) - 1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(or_circ));
+
+  hs_free_all();
+  free_mock_consensus();
+}
+
+static void
+test_validate_dos_extension_params(void *arg)
+{
+  bool ret;
+
+  (void) arg;
+
+  /* Validate the default values. */
+  ret = cell_dos_extension_parameters_are_valid(
+                                      get_intro2_rate_consensus_param(NULL),
+                                      get_intro2_burst_consensus_param(NULL));
+  tt_assert(ret);
+
+  /* Valid custom rate/burst. */
+  ret = cell_dos_extension_parameters_are_valid(17, 42);
+  tt_assert(ret);
+  ret = cell_dos_extension_parameters_are_valid(INT32_MAX, INT32_MAX);
+  tt_assert(ret);
+
+  /* Invalid rate. */
+  ret = cell_dos_extension_parameters_are_valid(UINT64_MAX, 42);
+  tt_assert(!ret);
+
+  /* Invalid burst. */
+  ret = cell_dos_extension_parameters_are_valid(42, UINT64_MAX);
+  tt_assert(!ret);
+
+  /* Value of 0 is valid (but should disable defenses) */
+  ret = cell_dos_extension_parameters_are_valid(0, 0);
+  tt_assert(ret);
+
+  /* Can't have burst smaller than rate. */
+  ret = cell_dos_extension_parameters_are_valid(42, 40);
+  tt_assert(!ret);
+
+ done:
+  return;
+}
+
+struct testcase_t hs_dos_tests[] = {
+  { "can_send_intro2", test_can_send_intro2, TT_FORK,
+    NULL, NULL },
+  { "validate_dos_extension_params", test_validate_dos_extension_params,
+    TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c
index 558fc32c54..feb934d93c 100644
--- a/src/test/test_hs_intropoint.c
+++ b/src/test/test_hs_intropoint.c
@@ -16,6 +16,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 
 #include "core/or/or.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "ht.h"
@@ -25,6 +26,8 @@
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 
@@ -43,6 +46,9 @@ new_establish_intro_cell(const char *circ_nonce,
   uint8_t buf[RELAY_PAYLOAD_SIZE] = {0};
   trn_cell_establish_intro_t *cell = NULL;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Ensure that *cell_out is NULL such that we can use to check if we need to
    * free `cell` in case of an error. */
@@ -50,9 +56,9 @@ new_establish_intro_cell(const char *circ_nonce,
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
   tt_i64_op(cell_len, OP_GT, 0);
 
   cell_len = trn_cell_establish_intro_parse(&cell, buf, sizeof(buf));
@@ -73,12 +79,15 @@ new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out)
 {
   ssize_t cell_len = 0;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell_out);
   tt_i64_op(cell_len, OP_GT, 0);
 
  done:
@@ -118,6 +127,8 @@ helper_create_intro_circuit(void)
   or_circuit_t *circ = or_circuit_new(0, NULL);
   tt_assert(circ);
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+  token_bucket_ctr_init(&circ->introduce2_bucket, 100, 100,
+                        (uint32_t) approx_time());
  done:
   return circ;
 }
@@ -693,6 +704,17 @@ test_introduce1_suitable_circuit(void *arg)
     tt_int_op(ret, OP_EQ, 0);
   }
 
+  /* Single hop circuit should not be allowed. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+    circ->p_chan->is_client = 1;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    tor_free(circ->p_chan);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
  done:
   ;
 }
@@ -888,43 +910,213 @@ test_received_introduce1_handling(void *arg)
   UNMOCK(relay_send_command_from_edge_);
 }
 
+static void
+test_received_establish_intro_dos_ext(void *arg)
+{
+  int ret;
+  ssize_t cell_len = 0;
+  uint8_t cell[RELAY_PAYLOAD_SIZE] = {0};
+  char circ_nonce[DIGEST_LEN] = {0};
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  hs_circuitmap_init();
+
+  /* Setup. */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+  memset(&config, 0, sizeof(config));
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 13;
+  config.intro_dos_burst_per_sec = 42;
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+  /* The INTRO2 bucket should be 0 at this point. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 0);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 0);
+
+  /* Case 1: Build encoded cell. Usable DoS parameters. */
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be set to the burst value. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 42);
+  /* Validate the config of the intro2 bucket. */
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 13);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 42);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 1);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 2: Build encoded cell. Bad DoS parameters. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = UINT_MAX;
+  config.intro_dos_burst_per_sec = 13;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 3: Build encoded cell. Burst is smaller than rate. Not allowed. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 87;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 4: Build encoded cell. Rate is 0 but burst is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 0;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 5: Build encoded cell. Burst is 0 but rate is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 45;
+  config.intro_dos_burst_per_sec = 0;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  service_intro_point_free(ip);
+  hs_circuitmap_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+static void *
+hs_subsystem_setup_fn(const struct testcase_t *tc)
+{
+  (void) tc;
+
+  return NULL;
+}
+
+static int
+hs_subsystem_cleanup_fn(const struct testcase_t *tc, void *arg)
+{
+  (void) tc;
+  (void) arg;
+
+  return 1;
+}
+
+static struct testcase_setup_t test_setup = {
+  hs_subsystem_setup_fn, hs_subsystem_cleanup_fn
+};
+
 struct testcase_t hs_intropoint_tests[] = {
   { "intro_point_registration",
-    test_intro_point_registration, TT_FORK, NULL, NULL },
+    test_intro_point_registration, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype",
-    test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype2",
-    test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype2, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_purpose",
-    test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_purpose, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig",
-    test_establish_intro_wrong_sig, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig_len",
-    test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_auth_key_len",
-    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_mac",
-    test_establish_intro_wrong_mac, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_mac, TT_FORK, NULL, &test_setup},
 
   { "introduce1_suitable_circuit",
-    test_introduce1_suitable_circuit, TT_FORK, NULL, NULL },
+    test_introduce1_suitable_circuit, TT_FORK, NULL, &test_setup},
 
   { "introduce1_is_legacy",
-    test_introduce1_is_legacy, TT_FORK, NULL, NULL },
+    test_introduce1_is_legacy, TT_FORK, NULL, &test_setup},
 
   { "introduce1_validation",
-    test_introduce1_validation, TT_FORK, NULL, NULL },
+    test_introduce1_validation, TT_FORK, NULL, &test_setup},
 
   { "received_introduce1_handling",
-    test_received_introduce1_handling, TT_FORK, NULL, NULL },
+    test_received_introduce1_handling, TT_FORK, NULL, &test_setup},
+
+  { "received_establish_intro_dos_ext",
+    test_received_establish_intro_dos_ext, TT_FORK, NULL, &test_setup},
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c
index 32b08ecf37..66194cee3d 100644
--- a/src/test/test_hs_service.c
+++ b/src/test/test_hs_service.c
@@ -21,6 +21,7 @@
 #define STATEFILE_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define HS_CLIENT_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -60,6 +61,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -169,8 +171,7 @@ test_e2e_rend_circuit_setup(void *arg)
     tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
     tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
 
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
 
     TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
   }
@@ -193,12 +194,14 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
@@ -267,7 +270,7 @@ helper_clone_authorized_client(const hs_service_authorized_client_t *client)
 
 /* Helper: Return a newly allocated service object with the identity keypair
  * sets and the current descriptor. Then register it to the global map.
- * Caller should us hs_free_all() to free this service or remove it from the
+ * Caller should use hs_free_all() to free this service or remove it from the
  * global map before freeing. */
 static hs_service_t *
 helper_create_service(void)
@@ -289,6 +292,20 @@ helper_create_service(void)
   return service;
 }
 
+/* Helper: Deallocate a given service object, its child objects and
+ * remove it from onion service map.
+ * */
+static void
+helper_destroy_service(hs_service_t *service)
+{
+  if (!service)
+    return;
+
+  remove_service(get_hs_service_map(), service);
+
+  hs_service_free(service);
+}
+
 /* Helper: Return a newly allocated service object with clients. */
 static hs_service_t *
 helper_create_service_with_clients(int num_clients)
@@ -314,17 +331,18 @@ helper_create_service_with_clients(int num_clients)
 static hs_service_intro_point_t *
 helper_create_service_ip(void)
 {
-  hs_desc_link_specifier_t *ls;
-  hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+  link_specifier_t *ls;
+  hs_service_intro_point_t *ip = service_intro_point_new(NULL);
   tor_assert(ip);
   /* Add a first unused link specifier. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_IPV4;
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
   smartlist_add(ip->base.link_specifiers, ls);
   /* Add a second link specifier used by a test. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_LEGACY_ID;
-  memset(ls->u.legacy_id, 'A', sizeof(ls->u.legacy_id));
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memset(link_specifier_getarray_un_legacy_id(ls), 'A',
+         link_specifier_getlen_un_legacy_id(ls));
   smartlist_add(ip->base.link_specifiers, ls);
 
   return ip;
@@ -378,11 +396,11 @@ test_load_keys(void *arg)
   tt_assert(s);
 
   /* Ok we have the service object. Validate few things. */
-  tt_assert(!tor_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
+  tt_assert(!fast_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
   tt_int_op(hs_address_is_valid(s->onion_address), OP_EQ, 1);
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_sk.seckey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_sk.seckey,
                              ED25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_pk.pubkey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_pk.pubkey,
                              ED25519_PUBKEY_LEN));
   /* Check onion address from identity key. */
   hs_build_address(&s->keys.identity_pk, s->config.version, addr);
@@ -656,13 +674,15 @@ test_service_intro_point(void *arg)
 
   (void) arg;
 
+  update_approx_time(1481621834);
+
   /* Test simple creation of an object. */
   {
-    time_t now = time(NULL);
+    time_t now = approx_time();
     ip = helper_create_service_ip();
     tt_assert(ip);
     /* Make sure the authentication keypair is not zeroes. */
-    tt_int_op(tor_mem_is_zero((const char *) &ip->auth_key_kp,
+    tt_int_op(fast_mem_is_zero((const char *) &ip->auth_key_kp,
                               sizeof(ed25519_keypair_t)), OP_EQ, 0);
     /* The introduce2_max MUST be in that range. */
     tt_u64_op(ip->introduce2_max, OP_GE,
@@ -797,10 +817,11 @@ test_helper_functions(void *arg)
     const node_t *node = get_node_from_intro_point(ip);
     tt_ptr_op(node, OP_EQ, &mock_node);
     SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                            hs_desc_link_specifier_t *, ls) {
-      if (ls->type == LS_LEGACY_ID) {
+                            link_specifier_t *, ls) {
+      if (link_specifier_get_ls_type(ls) == LS_LEGACY_ID) {
         /* Change legacy id in link specifier which is not the mock node. */
-        memset(ls->u.legacy_id, 'B', sizeof(ls->u.legacy_id));
+        memset(link_specifier_getarray_un_legacy_id(ls), 'B',
+               link_specifier_getlen_un_legacy_id(ls));
       }
     } SMARTLIST_FOREACH_END(ls);
     node = get_node_from_intro_point(ip);
@@ -858,6 +879,10 @@ test_helper_functions(void *arg)
 
  done:
   /* This will free the service and all objects associated to it. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_service_free_all();
   UNMOCK(node_get_by_id);
 }
@@ -867,7 +892,7 @@ static void
 test_intro_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   origin_circuit_t *circ = NULL;
 
   (void) arg;
@@ -915,6 +940,10 @@ test_intro_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -929,7 +958,7 @@ test_intro_established(void *arg)
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -990,6 +1019,10 @@ test_intro_established(void *arg)
  done:
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1001,7 +1034,7 @@ test_rdv_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
 
   (void) arg;
 
@@ -1032,6 +1065,10 @@ test_rdv_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -1069,8 +1106,7 @@ test_closing_intro_circs(void *arg)
 
   /* Initialize intro circuit */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Register circuit in the circuitmap . */
   hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
                                                     &ip->auth_key_kp.pubkey);
@@ -1096,8 +1132,7 @@ test_closing_intro_circs(void *arg)
   /* Now pretend that a new intro point circ was launched and opened. Check
    * that the intro point will be established correctly. */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&intro_circ->hs_ident->intro_auth_pk,
                       &ip->auth_key_kp.pubkey);
   /* Register circuit in the circuitmap . */
@@ -1119,6 +1154,10 @@ test_closing_intro_circs(void *arg)
     circuit_free_(TO_CIRCUIT(intro_circ));
   }
   /* Frees the service object. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(assert_circuit_ok);
 }
@@ -1131,7 +1170,7 @@ test_introduce2(void *arg)
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -1141,7 +1180,7 @@ test_introduce2(void *arg)
   MOCK(get_or_state,
        get_or_state_replacement);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
   tt_assert(circ);
@@ -1198,6 +1237,10 @@ test_introduce2(void *arg)
   dummy_state = NULL;
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1221,6 +1264,7 @@ test_service_event(void *arg)
 
   /* Set a service for this circuit. */
   service = helper_create_service();
+  tt_assert(service);
   ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
                       &service->keys.identity_pk);
 
@@ -1256,8 +1300,14 @@ test_service_event(void *arg)
     run_housekeeping_event(now);
     tt_int_op(digest256map_size(service->desc_current->intro_points.map),
               OP_EQ, 1);
+    /* No removal if we have an established circuit after retries. */
+    ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 1);
     /* Remove the IP object at once for the next test. */
     ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    ip->circuit_established = 0;
     run_housekeeping_event(now);
     tt_int_op(digest256map_size(service->desc_current->intro_points.map),
               OP_EQ, 0);
@@ -1282,6 +1332,10 @@ test_service_event(void *arg)
  done:
   hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1292,12 +1346,12 @@ test_rotate_descriptors(void *arg)
 {
   int ret;
   time_t next_rotation_time, now;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_descriptor_t *desc_next;
 
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   hs_init();
   MOCK(get_or_state, get_or_state_replacement);
@@ -1384,6 +1438,10 @@ test_rotate_descriptors(void *arg)
   tt_assert(service->desc_next);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(get_or_state);
   UNMOCK(circuit_mark_for_close_);
@@ -1396,9 +1454,8 @@ static void
 test_build_update_descriptors(void *arg)
 {
   int ret;
-  time_t now = time(NULL);
   node_t *node;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip_cur, *ip_next;
   routerinfo_t ri;
 
@@ -1411,7 +1468,7 @@ test_build_update_descriptors(void *arg)
   MOCK(networkstatus_get_live_consensus,
        mock_networkstatus_get_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1422,7 +1479,8 @@ test_build_update_descriptors(void *arg)
   voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
-  now = mock_ns.valid_after+1;
+
+  time_t now = mock_ns.valid_after+1;
 
   /* Create a service without a current descriptor to trigger a build. */
   service = helper_create_service();
@@ -1546,9 +1604,9 @@ test_build_update_descriptors(void *arg)
   tt_int_op(smartlist_len(ip_cur->base.link_specifiers), OP_EQ, 3);
   /* Make sure we have a valid encryption keypair generated when we pick an
    * intro point in the update process. */
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
                              CURVE25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
                              CURVE25519_PUBKEY_LEN));
   tt_u64_op(ip_cur->time_to_expire, OP_GE, now +
             INTRO_POINT_LIFETIME_MIN_SECONDS);
@@ -1614,6 +1672,10 @@ test_build_update_descriptors(void *arg)
   tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   nodelist_free_all();
 }
@@ -1626,6 +1688,7 @@ test_build_descriptors(void *arg)
 {
   int ret;
   time_t now = time(NULL);
+  hs_service_t *last_service = NULL;
 
   (void) arg;
 
@@ -1636,7 +1699,7 @@ test_build_descriptors(void *arg)
   MOCK(networkstatus_get_live_consensus,
        mock_networkstatus_get_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1650,19 +1713,27 @@ test_build_descriptors(void *arg)
    * is disabled. */
   {
     hs_service_t *service = helper_create_service();
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
     build_all_descriptors(now);
+    tt_assert(service->desc_current);
+    tt_assert(service->desc_current->desc);
+
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(0);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1670,12 +1741,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is not a multiple of 16. */
   {
     hs_service_t *service = helper_create_service_with_clients(20);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1683,12 +1758,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Do not generate any fake desc client when the number of clients is
    * a multiple of 16 but not zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(32);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1696,9 +1775,13 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
  done:
+  helper_destroy_service(last_service);
   hs_free_all();
 }
 
@@ -1717,7 +1800,7 @@ test_upload_descriptors(void *arg)
   MOCK(networkstatus_get_live_consensus,
        mock_networkstatus_get_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1847,9 +1930,9 @@ test_rendezvous1_parsing(void *arg)
   }
 
   /* Send out the RENDEZVOUS1 and make sure that our mock func worked */
-  tt_assert(tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(fast_mem_is_zero(rend1_payload, 32));
   hs_circ_service_rp_has_opened(service, service_circ);
-  tt_assert(!tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(!fast_mem_is_zero(rend1_payload, 32));
   tt_int_op(rend1_payload_len, OP_EQ, HS_LEGACY_RENDEZVOUS_CELL_SIZE);
 
   /******************************/
diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c
index 4a6d90d97e..104e973b1f 100644
--- a/src/test/test_introduce.c
+++ b/src/test/test_introduce.c
@@ -383,8 +383,10 @@ make_intro_from_plaintext(
 
   /* Output the cell */
   *cell_out = cell;
+  cell = NULL;
 
  done:
+  tor_free(cell);
   return cell_len;
 }
 
@@ -535,4 +537,3 @@ struct testcase_t introduce_tests[] = {
   INTRODUCE_LEGACY(late_parse_v3),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh
index 3474210607..54abb4a2fa 100755
--- a/src/test/test_key_expiration.sh
+++ b/src/test/test_key_expiration.sh
@@ -6,14 +6,14 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -47,11 +47,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_key_expiration_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -60,10 +60,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh
index 7afff271cb..cbdfd1909c 100755
--- a/src/test/test_keygen.sh
+++ b/src/test/test_keygen.sh
@@ -6,14 +6,14 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -64,11 +64,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_keygen_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_keygen_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -77,10 +77,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
@@ -144,7 +144,9 @@ ME="${DATA_DIR}/case2a"
 SRC="${DATA_DIR}/orig"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: `cat ${ME}/stdout`" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Somehow succeeded when missing secret key, certs: $(cat "${ME}/stdout")"
+fi
 check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key"
 
 grep "We needed to load a secret key.*but couldn't find it" "${ME}/stdout" >/dev/null || die "Tor didn't declare that it was missing a secret key"
@@ -281,7 +283,9 @@ SRC="${DATA_DIR}/encrypted"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_secret_key_encrypted" "${ME}/keys/"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Tor started with encrypted secret key and no certs" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Tor started with encrypted secret key and no certs"
+fi
 check_no_file "${ME}/keys/ed25519_signing_cert"
 check_no_file "${ME}/keys/ed25519_signing_secret_key"
 
@@ -370,7 +374,9 @@ mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
 cp "${OTHER}/keys/ed25519_master_id_secret_key" "${ME}/keys/"
 
-${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout" && die "Successfully started with mismatched keys!?" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout"; then
+  die "Successfully started with mismatched keys!?"
+fi
 
 grep "public_key does not match.*secret_key" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a key mismatch"
 
@@ -386,7 +392,9 @@ ME="${DATA_DIR}/case11a"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout" && die "Successfully started with passphrase-fd but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout"; then
+  die "Successfully started with passphrase-fd but no keygen?"
+fi
 
 grep "passphrase-fd specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -402,7 +410,9 @@ ME="${DATA_DIR}/case11b"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout" && die "Successfully started with no-passphrase but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with no-passphrase but no keygen?"
+fi
 
 grep "no-passphrase specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -418,7 +428,9 @@ ME="${DATA_DIR}/case11C"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout" && die "Successfully started with newpass but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout"; then
+  die "Successfully started with newpass but no keygen?"
+fi
 
 grep "newpass specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -456,7 +468,9 @@ ME="${DATA_DIR}/case11E"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd?"
+fi
 
 grep "Invalid --passphrase-fd value" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -473,7 +487,9 @@ ME="${DATA_DIR}/case11F"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd combination?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd combination?"
+fi
 
 grep "no-passphrase specified with --passphrase-fd" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c
index 34f59f26cd..5e78e1ce4d 100644
--- a/src/test/test_link_handshake.c
+++ b/src/test/test_link_handshake.c
@@ -263,7 +263,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_assert(c1->handshake_state->authenticated_rsa);
     tt_assert(! c1->handshake_state->authenticated_ed25519);
   }
-  tt_assert(! tor_mem_is_zero(
+  tt_assert(! fast_mem_is_zero(
                 (char*)c1->handshake_state->authenticated_rsa_peer_id, 20));
 
   chan2 = tor_malloc_zero(sizeof(*chan2));
@@ -290,7 +290,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_ptr_op(c2->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
   }
   tt_assert(c2->handshake_state->certs->id_cert);
-  tt_assert(tor_mem_is_zero(
+  tt_assert(fast_mem_is_zero(
               (char*)c2->handshake_state->authenticated_rsa_peer_id, 20));
   /* no authentication has happened yet, since we haen't gotten an AUTH cell.
    */
@@ -948,7 +948,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_int_op(36, OP_EQ, cell1->payload_len);
   tt_int_op(36, OP_EQ, cell2->payload_len);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_int_op(0, OP_EQ, cell1->circ_id);
   tt_int_op(0, OP_EQ, cell2->circ_id);
   tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell1->command);
@@ -960,7 +960,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 4);
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 4);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_mem_op(cell1->payload, OP_NE, cell2->payload, 32);
 
  done:
diff --git a/src/test/test_logging.c b/src/test/test_logging.c
index 95a2fce757..203ce64e32 100644
--- a/src/test/test_logging.c
+++ b/src/test/test_logging.c
@@ -9,14 +9,13 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 
 static void
-dummy_cb_fn(int severity, uint32_t domain, const char *msg)
+dummy_cb_fn(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)severity; (void)domain; (void)msg;
 }
@@ -36,7 +35,7 @@ test_get_sigsafe_err_fds(void *arg)
 
   set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug);
   set_log_severity_config(LOG_WARN, LOG_ERR, &no_bug);
-  no_bug.masks[0] &= ~(LD_BUG|LD_GENERAL);
+  no_bug.masks[SEVERITY_MASK_IDX(LOG_ERR)] &= ~(LD_BUG|LD_GENERAL);
   set_log_severity_config(LOG_INFO, LOG_NOTICE, &no_bug2);
 
   /* Add some logs; make sure the output is as expected. */
@@ -117,22 +116,27 @@ test_sigsafe_err(void *arg)
   content = read_file_to_str(fn, 0, NULL);
 
   tt_ptr_op(content, OP_NE, NULL);
-  tor_split_lines(lines, content, (int)strlen(content));
+  smartlist_split_string(lines, content, "\n", 0, 0);
   tt_int_op(smartlist_len(lines), OP_GE, 5);
 
-  if (strstr(smartlist_get(lines, 0), "opening new log file"))
+  if (strstr(smartlist_get(lines, 0), "opening new log file")) {
+    void *item = smartlist_get(lines, 0);
     smartlist_del_keeporder(lines, 0);
+    tor_free(item);
+  }
+
   tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool"));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 1), "=============="));
-  tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal."));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 3), "=============="));
-  tt_str_op(smartlist_get(lines, 4), OP_EQ,
+  tt_str_op(smartlist_get(lines, 1), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 2), "=============="));
+  tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal."));
+  tt_str_op(smartlist_get(lines, 4), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 5), "=============="));
+  tt_str_op(smartlist_get(lines, 6), OP_EQ,
             "Testing any attempt to manually log from a signal.");
 
  done:
   tor_free(content);
+  SMARTLIST_FOREACH(lines, char *, x, tor_free(x));
   smartlist_free(lines);
 }
 
diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c
index 089ea812cf..ed6b8a9b66 100644
--- a/src/test/test_mainloop.c
+++ b/src/test/test_mainloop.c
@@ -6,11 +6,23 @@
  * \brief Tests for functions closely related to the Tor main loop
  */
 
+#define CONFIG_PRIVATE
+#define MAINLOOP_PRIVATE
+#define STATEFILE_PRIVATE
+
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
 #include "core/or/or.h"
+#include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+
+#include "feature/hs/hs_service.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/config/or_state_st.h"
 
 static const uint64_t BILLION = 1000000000;
 
@@ -131,12 +143,227 @@ test_mainloop_update_time_jumps(void *arg)
   monotime_disable_test_mocking();
 }
 
+static int schedule_rescan_called = 0;
+static void
+mock_schedule_rescan_periodic_events(void)
+{
+  ++schedule_rescan_called;
+}
+
+static void
+test_mainloop_user_activity(void *arg)
+{
+  (void)arg;
+  const time_t start = 1542658829;
+  update_approx_time(start);
+
+  MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events);
+
+  reset_user_activity(start);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  set_network_participation(false);
+
+  // reset can move backwards and forwards, but does not change network
+  // participation.
+  reset_user_activity(start-10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10);
+  reset_user_activity(start+10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  tt_int_op(schedule_rescan_called, OP_EQ, 0);
+  tt_int_op(false, OP_EQ, is_participating_on_network());
+
+  // "note" can only move forward.  Calling it from a non-participating
+  // state makes us rescan the periodic callbacks and set participation.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // Calling it again will move us forward, but not call rescan again.
+  note_user_activity(start+25);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We won't move backwards.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We _will_ adjust if the clock jumps though.
+  netstatus_note_clock_jumped(500);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525);
+
+  netstatus_note_clock_jumped(-400);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125);
+
+ done:
+  UNMOCK(schedule_rescan_periodic_events);
+}
+
+static unsigned int
+mock_get_num_services(void)
+{
+  return 1;
+}
+
+static connection_t *
+mock_connection_gbtu(int type)
+{
+  (void) type;
+  return (void *)"hello fellow connections";
+}
+
+static void
+test_mainloop_check_participation(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  const time_t start = 1542658829;
+  const time_t ONE_DAY = 24*60*60;
+
+  // Suppose we've been idle for a day or two
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  check_network_participation_callback(start, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+  // suppose we've been idle for 2 days... but we are a server.
+  reset_user_activity(start - 2*ONE_DAY);
+  options->ORPort_set = 1;
+  set_network_participation(true);
+  check_network_participation_callback(start+2, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2);
+  options->ORPort_set = 0;
+
+  // idle for 2 days, but we have a hidden service.
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  MOCK(hs_service_get_num_services, mock_get_num_services);
+  check_network_participation_callback(start+3, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3);
+  UNMOCK(hs_service_get_num_services);
+
+  // idle for 2 days but we have at least one user connection
+  MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu);
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+  check_network_participation_callback(start+10, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  // as above, but DormantTimeoutDisabledByIdleStreams is not set
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 0;
+  check_network_participation_callback(start+13, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+  UNMOCK(connection_get_by_type_nonlinked);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+
+  // idle for 2 days but DormantClientTimeout is 3 days
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantClientTimeout = ONE_DAY * 3;
+  check_network_participation_callback(start+30, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+ done:
+  or_options_free(options);
+  UNMOCK(hs_service_get_num_services);
+  UNMOCK(connection_get_by_type_nonlinked);
+}
+
+static void
+test_mainloop_dormant_load_state(void *arg)
+{
+  (void)arg;
+  or_state_t *state = or_state_new();
+  const time_t start = 1543956575;
+
+  reset_user_activity(0);
+  set_network_participation(false);
+
+  // When we construct a new state, it starts out in "auto" mode.
+  tt_int_op(state->Dormant, OP_EQ, -1);
+
+  // Initializing from "auto" makes us start out (by default) non-Dormant,
+  // with activity right now.
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  // Initializing from dormant clears the last user activity time, and
+  // makes us dormant.
+  state->Dormant = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(! is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, 0);
+
+  // Initializing from non-dormant sets the last user activity time, and
+  // makes us non-dormant.
+  state->Dormant = 0;
+  state->MinutesSinceUserActivity = 123;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60);
+
+  // If we would start dormant, but DormantCanceledByStartup is set, then
+  // we start up non-dormant.
+  state->Dormant = 1;
+  get_options_mutable()->DormantCanceledByStartup = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+ done:
+  or_state_free(state);
+}
+
+static void
+test_mainloop_dormant_save_state(void *arg)
+{
+  (void)arg;
+  or_state_t *state = or_state_new();
+  const time_t start = 1543956575;
+
+  // Can we save a non-dormant state correctly?
+  reset_user_activity(start - 1000);
+  set_network_participation(true);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 0);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60);
+
+  // Can we save a dormant state correctly?
+  set_network_participation(false);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 1);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0);
+
+ done:
+  or_state_free(state);
+}
+
 #define MAINLOOP_TEST(name) \
   { #name, test_mainloop_## name , TT_FORK, NULL, NULL }
 
 struct testcase_t mainloop_tests[] = {
   MAINLOOP_TEST(update_time_normal),
   MAINLOOP_TEST(update_time_jumps),
+  MAINLOOP_TEST(user_activity),
+  MAINLOOP_TEST(check_participation),
+  MAINLOOP_TEST(dormant_load_state),
+  MAINLOOP_TEST(dormant_save_state),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c
index 4c4317d81a..804e6c546a 100644
--- a/src/test/test_microdesc.c
+++ b/src/test/test_microdesc.c
@@ -11,6 +11,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
 
@@ -20,6 +21,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -70,6 +72,7 @@ test_md_cache(void *data)
   const char *test_md3_noannotation = strchr(test_md3, '\n')+1;
   time_t time1, time2, time3;
   char *fn = NULL, *s = NULL;
+  char *encoded_family = NULL;
   (void)data;
 
   options = get_options_mutable();
@@ -172,8 +175,9 @@ test_md_cache(void *data)
 
   tt_ptr_op(md1->family, OP_EQ, NULL);
   tt_ptr_op(md3->family, OP_NE, NULL);
-  tt_int_op(smartlist_len(md3->family), OP_EQ, 3);
-  tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX");
+
+  encoded_family = nodefamily_format(md3->family);
+  tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez");
 
   /* Now rebuild the cache! */
   tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0);
@@ -254,6 +258,7 @@ test_md_cache(void *data)
   smartlist_free(wanted);
   tor_free(s);
   tor_free(fn);
+  tor_free(encoded_family);
 }
 
 static const char truncated_md[] =
@@ -417,6 +422,28 @@ static const char test_md2_21[] =
   "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
   "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
 
+static const char test_md2_withfamily_28[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family OtherNode !Strange\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
+static const char test_md2_withfamily_29[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
 static void
 test_md_generate(void *arg)
 {
@@ -447,6 +474,17 @@ test_md_generate(void *arg)
   tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey,
                               &ri->cache_info.signing_key_cert->signing_key));
 
+  // Try family encoding.
+  microdesc_free(md);
+  ri->declared_family = smartlist_new();
+  smartlist_add_strdup(ri->declared_family, "OtherNode !Strange");
+  md = dirvote_create_microdescriptor(ri, 28);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_28);
+
+  microdesc_free(md);
+  md = dirvote_create_microdescriptor(ri, 29);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_29);
+
  done:
   microdesc_free(md);
   routerinfo_free(ri);
@@ -611,6 +649,41 @@ static const char MD_PARSE_TEST_DATA[] =
   "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
   "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   "p6 allow 80\n"
+  /* Good 11: Normal, non-exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 12: Normal, exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 13: Normal, exit relay with only ipv6 exit policy */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p6 accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   ;
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 ENABLE_GCC_WARNING(overlength-strings)
@@ -628,7 +701,7 @@ test_md_parse(void *arg)
   smartlist_t *mds = microdescs_parse_from_string(MD_PARSE_TEST_DATA,
                                                   NULL, 1, SAVED_NOWHERE,
                                                   invalid);
-  tt_int_op(smartlist_len(mds), OP_EQ, 11);
+  tt_int_op(smartlist_len(mds), OP_EQ, 14);
   tt_int_op(smartlist_len(invalid), OP_EQ, 4);
 
   test_memeq_hex(smartlist_get(invalid,0),
@@ -675,6 +748,21 @@ test_md_parse(void *arg)
   tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
   tt_int_op(md->ipv6_orport, OP_EQ, 9090);
 
+  md = smartlist_get(mds, 11);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 1);
+
+  md = smartlist_get(mds, 12);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
+  md = smartlist_get(mds, 13);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
  done:
   SMARTLIST_FOREACH(mds, microdesc_t *, mdsc, microdesc_free(mdsc));
   smartlist_free(mds);
@@ -683,6 +771,80 @@ test_md_parse(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+static void
+test_md_parse_id_ed25519(void *arg)
+{
+  (void)arg;
+
+  /* A correct MD with an ed25519 ID ... and an unspecified ID type,
+   * which is permitted. */
+  const char GOOD_MD[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id wumpus dodecahedron\n";
+
+  smartlist_t *mds = NULL;
+  const microdesc_t *md;
+
+  mds = microdescs_parse_from_string(GOOD_MD,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 1);
+  md = smartlist_get(mds, 0);
+  tt_mem_op(md->ed25519_identity_pkey, OP_EQ,
+            "This isn't actually a public key", ED25519_PUBKEY_LEN);
+  SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key appears twice. */
+  const char DUPLICATE_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n";
+
+  setup_capture_of_logs(LOG_WARN);
+  mds = microdescs_parse_from_string(DUPLICATE_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Extra ed25519 key");
+  mock_clean_saved_logs();
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key is invalid. */
+  const char BOGUS_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyZZZZZZZZZZZ\n";
+
+  mds = microdescs_parse_from_string(BOGUS_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Bogus ed25519 key");
+
+ done:
+  if (mds) {
+    SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+    smartlist_free(mds);
+  }
+  teardown_capture_of_logs();
+}
+
 static int mock_rgsbd_called = 0;
 static routerstatus_t *mock_rgsbd_val_a = NULL;
 static routerstatus_t *mock_rgsbd_val_b = NULL;
@@ -816,6 +978,7 @@ struct testcase_t microdesc_tests[] = {
   { "broken_cache", test_md_cache_broken, TT_FORK, NULL, NULL },
   { "generate", test_md_generate, 0, NULL, NULL },
   { "parse", test_md_parse, 0, NULL, NULL },
+  { "parse_id_ed25519", test_md_parse_id_ed25519, 0, NULL, NULL },
   { "reject_cache", test_md_reject_cache, TT_FORK, NULL, NULL },
   { "corrupt_desc", test_md_corrupt_desc, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
diff --git a/src/test/test_namemap.c b/src/test/test_namemap.c
new file mode 100644
index 0000000000..df77d4e2de
--- /dev/null
+++ b/src/test/test_namemap.c
@@ -0,0 +1,174 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/malloc/malloc.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static void
+test_namemap_empty(void *arg)
+{
+  (void)arg;
+
+  namemap_t m;
+  namemap_init(&m);
+  namemap_t m2 = NAMEMAP_INIT();
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m2));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+ done:
+  namemap_clear(&m);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_toolong(void *arg)
+{
+  (void)arg;
+  namemap_t m;
+  char *ok = NULL;
+  char *toolong = NULL;
+  namemap_init(&m);
+
+  ok = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+1);
+  memset(ok, 'x', MAX_NAMEMAP_NAME_LEN);
+
+  toolong = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+2);
+  memset(toolong, 'x', MAX_NAMEMAP_NAME_LEN+1);
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+  unsigned u1 = namemap_get_or_create_id(&m, toolong);
+  unsigned u2 = namemap_get_or_create_id(&m, ok);
+  tt_uint_op(u1, OP_EQ, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+
+  tt_str_op(ok, OP_EQ, namemap_get_name(&m, u2));
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m, u1));
+
+ done:
+  tor_free(ok);
+  tor_free(toolong);
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_blackbox(void *arg)
+{
+  (void)arg;
+
+  namemap_t m1, m2;
+  namemap_init(&m1);
+  namemap_init(&m2);
+
+  unsigned u1 = namemap_get_or_create_id(&m1, "hello");
+  unsigned u2 = namemap_get_or_create_id(&m1, "world");
+  tt_uint_op(u1, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u1, OP_NE, u2);
+
+  tt_uint_op(u1, OP_EQ, namemap_get_id(&m1, "hello"));
+  tt_uint_op(u1, OP_EQ, namemap_get_or_create_id(&m1, "hello"));
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m1, "world"));
+  tt_uint_op(u2, OP_EQ, namemap_get_or_create_id(&m1, "world"));
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "HELLO"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+
+  unsigned u3 = namemap_get_or_create_id(&m2, "hola");
+  tt_uint_op(u3, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_or_create_id(&m2, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  unsigned int u4 = namemap_get_or_create_id(&m1, "hola");
+  tt_uint_op(u4, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u4, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  tt_str_op("hello", OP_EQ, namemap_get_name(&m1, u1));
+  tt_str_op("world", OP_EQ, namemap_get_name(&m1, u2));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m2, u3));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m1, u4));
+
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m2, u3 + 10));
+
+ done:
+  namemap_clear(&m1);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_internals(void *arg)
+{
+  (void)arg;
+  // This test actually assumes know something about the identity layout.
+  namemap_t m;
+  namemap_init(&m);
+
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+
+ done:
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_fmt(void *arg)
+{
+  (void)arg;
+  namemap_t m = NAMEMAP_INIT();
+
+  unsigned a = namemap_get_or_create_id(&m, "greetings");
+  unsigned b = namemap_get_or_create_id(&m, "earthlings");
+
+  tt_str_op(namemap_fmt_name(&m, a), OP_EQ, "greetings");
+  tt_str_op(namemap_fmt_name(&m, b), OP_EQ, "earthlings");
+  tt_int_op(a, OP_NE, 100);
+  tt_int_op(b, OP_NE, 100);
+  tt_str_op(namemap_fmt_name(&m, 100), OP_EQ, "{100}");
+
+ done:
+  namemap_clear(&m);
+}
+
+#define T(name) \
+  { #name, test_namemap_ ## name , 0, NULL, NULL }
+
+struct testcase_t namemap_tests[] = {
+  T(empty),
+  T(toolong),
+  T(blackbox),
+  T(internals),
+  T(fmt),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c
new file mode 100644
index 0000000000..27d276d42f
--- /dev/null
+++ b/src/test/test_netinfo.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "trunnel/netinfo.h"
+#include "test/test.h"
+
+static void
+test_netinfo_unsupported_addr(void *arg)
+{
+  const uint8_t wire_data[] =
+    { // TIME
+      0x00, 0x00, 0x00, 0x01,
+      // OTHERADDR
+      0x04, // ATYPE
+      0x04, // ALEN
+      0x08, 0x08, 0x08, 0x08, // AVAL
+      0x01, // NMYADDR
+      0x03, // ATYPE (unsupported)
+      0x05, // ALEN
+      'a', 'd', 'r', 'r', '!' // AVAL (unsupported)
+    };
+
+  (void)arg;
+
+  netinfo_cell_t *parsed_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data,
+                  sizeof(wire_data));
+
+  tt_assert(parsed == sizeof(wire_data));
+
+  netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0);
+  tt_assert(addr);
+
+  tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr));
+  tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr));
+
+ done:
+  netinfo_cell_free(parsed_cell);
+}
+
+struct testcase_t netinfo_tests[] = {
+  { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c
index 53eb0413e5..b8ca56bb81 100644
--- a/src/test/test_nodelist.c
+++ b/src/test/test_nodelist.c
@@ -6,15 +6,21 @@
  * \brief Unit tests for nodelist related functions.
  **/
 
+#define NODELIST_PRIVATE
+
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/torcert.h"
 
+#include "core/or/extend_info_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodefamily_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
@@ -72,7 +78,7 @@ test_nodelist_node_get_verbose_nickname_not_named(void *arg)
 }
 
 /** A node should be considered a directory server if it has an open dirport
- * of it accepts tunnelled directory requests.
+ * or it accepts tunnelled directory requests.
  */
 static void
 test_nodelist_node_is_dir(void *arg)
@@ -231,6 +237,1015 @@ test_nodelist_ed_id(void *arg)
 #undef N_NODES
 }
 
+static void
+test_nodelist_nodefamily(void *arg)
+{
+  (void)arg;
+  /* hex ID digests */
+  const char h1[] = "5B435D6869206861206C65207363617270652070";
+  const char h2[] = "75C3B220616E6461726520696E206769726F2061";
+  const char h3[] = "2074726F766172206461206D616E67696172652C";
+  const char h4[] = "206D656E747265206E6F6E2076616C65206C2769";
+  const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669";
+
+  /* binary ID digests */
+  uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN],
+    d5[DIGEST_LEN];
+  base16_decode((char*)d1, sizeof(d1), h1, strlen(h1));
+  base16_decode((char*)d2, sizeof(d2), h2, strlen(h2));
+  base16_decode((char*)d3, sizeof(d3), h3, strlen(h3));
+  base16_decode((char*)d4, sizeof(d4), h4, strlen(h4));
+  base16_decode((char*)d5, sizeof(d5), h5, strlen(h5));
+
+  char *enc=NULL, *enc2=NULL;
+
+  nodefamily_t *nf1 = NULL;
+  nodefamily_t *nf2 = NULL;
+  nodefamily_t *nf3 = NULL;
+
+  enc = nodefamily_format(NULL);
+  tt_str_op(enc, OP_EQ, "");
+  tor_free(enc);
+
+  /* Make sure that sorting and de-duplication work. */
+  tor_asprintf(&enc, "$%s hello", h1);
+  nf1 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf1);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "hello hello $%s hello", h1);
+  nf2 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf2);
+  tt_ptr_op(nf1, OP_EQ, nf2);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "%s $%s hello", h1, h1);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(! nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_nickname(nf1, "hello"));
+  tt_assert(nodefamily_contains_nickname(nf1, "HELLO"));
+  tt_assert(! nodefamily_contains_nickname(nf1, "goodbye"));
+
+  tt_int_op(nf1->refcnt, OP_EQ, 3);
+  nodefamily_free(nf3);
+  tt_int_op(nf1->refcnt, OP_EQ, 2);
+
+  /* Try parsing with a provided self RSA digest. */
+  nf3 = nodefamily_parse("hello ", d1, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+
+  /* Do we get the expected result when we re-encode? */
+  tor_asprintf(&enc, "$%s hello", h1);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc2, OP_EQ, enc);
+  tor_free(enc2);
+  tor_free(enc);
+
+  /* Make sure that we get a different result if we give a different digest. */
+  nodefamily_free(nf3);
+  tor_asprintf(&enc, "hello $%s hello", h3);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_NE, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf3, d3));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d2));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d1));
+  tt_assert(nodefamily_contains_nickname(nf3, "hello"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "goodbye"));
+
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+
+  /* Try one with several digests, all with nicknames appended, in different
+     formats. */
+  tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4);
+  nf1 = nodefamily_parse(enc, d5, 0);
+  tt_assert(nf1);
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d3));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d5));
+  /* Nicknames aren't preserved when ids are present, since node naming is
+   * deprecated */
+  tt_assert(! nodefamily_contains_nickname(nf3, "res"));
+  tor_free(enc);
+  tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc, OP_EQ, enc2);
+  tor_free(enc);
+  tor_free(enc2);
+
+  /* Try ones where we parse the empty string. */
+  nf2 = nodefamily_parse("", NULL, 0);
+  nf3 = nodefamily_parse("", d4, 0);
+  tt_assert(nf2);
+  tt_assert(nf3);
+  tt_ptr_op(nf2, OP_NE, nf3);
+
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf3, d4));
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d5));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d5));
+  tt_assert(! nodefamily_contains_nickname(nf2, "fred"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "bosco"));
+
+  /* The NULL family should contain nothing. */
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d4));
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d5));
+
+ done:
+  tor_free(enc);
+  tor_free(enc2);
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+  nodefamily_free_all();
+}
+
+static void
+test_nodelist_nodefamily_parse_err(void *arg)
+{
+  (void)arg;
+  nodefamily_t *nf1 = NULL;
+  char *enc = NULL;
+  const char *semibogus =
+    "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname
+    "$jkASDFLkjsadfjhkl " // not hex
+    "$7468696e67732d696e2d7468656d73656c766573 " // ok
+    "reticulatogranulate "// ok
+    "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex
+    "$616273656e746d696e6465646e6573736573" // too short for hex
+    ;
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // We only get two items when we parse this.
+  for (int reject = 0; reject <= 1; ++reject) {
+    for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) {
+      unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0;
+      flags |= reject ? NF_REJECT_MALFORMED : 0;
+      nf1 = nodefamily_parse(semibogus, NULL, flags);
+      if (reject) {
+        tt_assert(nf1 == NULL);
+      } else {
+        tt_assert(nf1);
+        enc = nodefamily_format(nf1);
+        tt_str_op(enc, OP_EQ,
+                  "$7468696E67732D696E2D7468656D73656C766573 "
+                  "reticulatogranulate");
+        tor_free(enc);
+      }
+
+      if (log_at_warn) {
+        expect_log_msg_containing("$616273656e746d696e6465646e6573736573");
+        expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf");
+      } else {
+        tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0);
+      }
+      mock_clean_saved_logs();
+    }
+  }
+
+ done:
+  tor_free(enc);
+  nodefamily_free(nf1);
+  teardown_capture_of_logs();
+}
+
+static const node_t *
+mock_node_get_by_id(const char *id)
+{
+  if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  memcpy(fake_node->identity, id, DIGEST_LEN);
+  return fake_node;
+}
+
+static const node_t *
+mock_node_get_by_nickname(const char *nn, unsigned flags)
+{
+  (void)flags;
+  if (!strcmp(nn, "nonesuch"))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  strlcpy(fake_node->identity, nn, DIGEST_LEN);
+  return fake_node;
+}
+
+static void
+test_nodelist_nodefamily_lookup(void *arg)
+{
+  (void)arg;
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  smartlist_t *sl = smartlist_new();
+  nodefamily_t *nf1 = NULL;
+  char *mem_op_hex_tmp = NULL;
+
+  // 'null' is allowed.
+  nodefamily_add_nodes_to_smartlist(NULL, sl);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
+
+  // Try a real family
+  nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE "
+                         "$2121212121212121212121212121212121212121 "
+                         "$3333333333333333333333333333333333333333 "
+                         "erewhon nonesuch", NULL, 0);
+  tt_assert(nf1);
+  nodefamily_add_nodes_to_smartlist(nf1, sl);
+  // There were 5 elements; 2 were dropped because the mocked lookup failed.
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+
+  const node_t *n = smartlist_get(sl, 0);
+  test_memeq_hex(n->identity, "3333333333333333333333333333333333333333");
+  n = smartlist_get(sl, 1);
+  test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE");
+  n = smartlist_get(sl, 2);
+  tt_str_op(n->identity, OP_EQ, "erewhon");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node));
+  smartlist_free(sl);
+  nodefamily_free(nf1);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_nodelist_nickname_matches(void *arg)
+{
+  (void)arg;
+  node_t mock_node;
+  routerstatus_t mock_rs;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+
+  strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname));
+  mock_node.rs = &mock_rs;
+  memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN);
+
+#define match(x) tt_assert(node_nickname_matches(&mock_node, (x)))
+#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x)))
+
+  match("evilgeniuses");
+  match("EvilGeniuses");
+  match("EvilGeniuses");
+  match("2e666f7261626574746572746f6d6f72726f772e");
+  match("2E666F7261626574746572746F6D6F72726F772E");
+  match("$2e666f7261626574746572746f6d6f72726f772e");
+  match("$2E666F7261626574746572746F6D6F72726F772E");
+  match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses");
+  match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES");
+
+  no_match("evilgenius");
+  no_match("evilgeniuseses");
+  no_match("evil.genius");
+  no_match("$2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E~fred");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES");
+ done:
+  ;
+}
+
+static void
+test_nodelist_node_nodefamily(void *arg)
+{
+  (void)arg;
+  node_t mock_node1;
+  routerstatus_t mock_rs;
+  microdesc_t mock_md;
+
+  node_t mock_node2;
+  routerinfo_t mock_ri;
+
+  smartlist_t *nodes=smartlist_new();
+
+  memset(&mock_node1, 0, sizeof(mock_node1));
+  memset(&mock_node2, 0, sizeof(mock_node2));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+  memset(&mock_md, 0, sizeof(mock_md));
+  memset(&mock_ri, 0, sizeof(mock_ri));
+
+  mock_node1.rs = &mock_rs;
+  mock_node1.md = &mock_md;
+
+  mock_node2.ri = &mock_ri;
+
+  strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname));
+  mock_ri.nickname = tor_strdup("nodetwo");
+
+  memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN);
+  memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN);
+
+  // empty families.
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain nodes, but not these nodes
+  mock_ri.declared_family = smartlist_new();
+  smartlist_add(mock_ri.declared_family, (char*)"NodeThree");
+  mock_md.family = nodefamily_parse("NodeFour", NULL, 0);
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain one another.
+  smartlist_add(mock_ri.declared_family, (char*)
+                "4e6f64654f6e654e6f6465314e6f64654f6e6531");
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  nodefamily_free(mock_md.family);
+  mock_md.family = nodefamily_parse(
+            "NodeFour "
+            "5365636f6e644e6f64655765277265546573746e", NULL, 0);
+  tt_assert(node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  // Try looking up families now.
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  node_lookup_declared_family(nodes, &mock_node1);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  const node_t *n = smartlist_get(nodes, 0);
+  tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN);
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "nodefour");
+
+  // free, try the other one.
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_clear(nodes);
+
+  node_lookup_declared_family(nodes, &mock_node2);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  n = smartlist_get(nodes, 0);
+  // This gets a truncated hex hex ID since it was looked up by name
+  tt_str_op(n->identity, OP_EQ, "NodeThree");
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  smartlist_free(mock_ri.declared_family);
+  nodefamily_free(mock_md.family);
+  tor_free(mock_ri.nickname);
+  // use tor_free, these aren't real nodes
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_free(nodes);
+}
+
+static void
+test_nodelist_nodefamily_canonicalize(void *arg)
+{
+  (void)arg;
+  char *c = NULL;
+
+  c = nodefamily_canonicalize("", NULL, 0);
+  tt_str_op(c, OP_EQ, "");
+  tor_free(c);
+
+  uint8_t own_id[20];
+  memset(own_id, 0, sizeof(own_id));
+  c = nodefamily_canonicalize(
+           "alice BOB caroL %potrzebie !!!@#@# "
+           "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred "
+           "ffffffffffffffffffffffffffffffffffffffff "
+           "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0);
+  tt_str_op(c, OP_EQ,
+           "!!!@#@# "
+           "$0000000000000000000000000000000000000000 "
+           "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB "
+           "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC "
+           "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF "
+           "%potrzebie "
+           "alice bob carol");
+
+ done:
+  tor_free(c);
+}
+
+/** format_node_description() should return
+ * "Fingerprint~Nickname at IPv4 and [IPv6]".
+ * The nickname and addresses are optional.
+ */
+static void
+test_nodelist_format_node_description(void *arg)
+{
+  char mock_digest[DIGEST_LEN];
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_null_ip;
+  tor_addr_t mock_ipv4;
+  tor_addr_t mock_ipv6;
+
+  char ndesc[NODE_DESC_BUF_LEN];
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(ndesc, 0, sizeof(ndesc));
+  memset(mock_digest, 0, sizeof(mock_digest));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_null_ip, 0, sizeof(mock_null_ip));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ipv6, 0, sizeof(mock_ipv6));
+
+  /* Set variables */
+  memcpy(mock_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  tor_addr_parse(&mock_ipv6, "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Test function with variables */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  /* format node description should use ~ because named is deprecated */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~""TestOR7890123456789");
+
+  /* Try a null IP address, rather than NULL */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_null_ip,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789");
+
+  /* Try some real IP addresses */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               &mock_ipv4,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA at 111.222.233.244");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               tor_addr_to_ipv4h(&mock_ipv4));
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = format_node_description(NULL, NULL, NULL, NULL, 0);
+  tt_str_op(rv, OP_EQ, "<NULL BUFFER>");
+
+  rv = format_node_description(ndesc, NULL, NULL, NULL, 0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(rv, OP_EQ, "<NULL ID DIGEST>");
+
+ done:
+  return;
+}
+
+/** router_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_router_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+  routerinfo_t mock_ri_ipv4;
+  routerinfo_t mock_ri_ipv6;
+  routerinfo_t mock_ri_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_ipv4, 0, sizeof(mock_ri_ipv4));
+  memset(&mock_ri_ipv6, 0, sizeof(mock_ri_ipv6));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerinfos.
+   * mock_nickname is referenced from all 3 routerinfos.
+   * That's ok, all their memory is static. */
+  memcpy(&mock_ri_ipv4, &mock_ri_dual, sizeof(mock_ri_ipv4));
+  memcpy(&mock_ri_ipv6, &mock_ri_dual, sizeof(mock_ri_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_ri_ipv4.ipv6_addr, 0, sizeof(mock_ri_ipv4.ipv6_addr));
+  mock_ri_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = router_describe(&mock_ri_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = router_describe(&mock_ri_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = router_describe(&mock_ri_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = router_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerinfos */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.ri = &mock_ri_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.ri = &mock_ri_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = &mock_ri_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** node_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only and routerstatus-only node_describe() tests are in
+ * test_nodelist_router_describe() and test_nodelist_routerstatus_describe(),
+ * so we can re-use their mocked variables.
+ */
+static void
+test_nodelist_node_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Routerinfos */
+  routerinfo_t mock_ri_dual;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Routerstatuses */
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_dual;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB"
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "Bbb",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "2.2.2.2");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[bbbb::bbbb]");
+
+  /* Create and modify the other routerstatus. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  /* Clear the unnecessary IPv6 address */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+
+  /* Microdescs */
+  microdesc_t mock_md_null;
+  microdesc_t mock_md_ipv6;
+
+  /* Clear variables */
+  memset(&mock_md_null, 0, sizeof(mock_md_null));
+  memset(&mock_md_ipv6, 0, sizeof(mock_md_ipv6));
+
+  /* Set up the microdesc */
+  tor_addr_parse(&mock_md_ipv6.ipv6_addr,
+                 "[eeee::6000:6000]");
+
+  /* Set up the node */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC"
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC",
+         sizeof(mock_node.identity));
+
+  /* Test that the routerinfo and routerstatus work separately, but the
+   * identity comes from the node */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerstatus overrides the routerinfo */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the microdesc IPv6 is used if the routerinfo doesn't have IPv6
+   */
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerinfo doesn't change the results above
+   */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* test NULL handling */
+  rv = node_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  mock_node.ri = NULL;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "<null rs and ri>");
+
+ done:
+  return;
+}
+
+/** routerstatus_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ *
+ * The routerstatus-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_routerstatus_describe(void *arg)
+{
+  tor_addr_t mock_ipv4;
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_ipv6;
+  routerstatus_t mock_rs_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_ipv6, 0, sizeof(mock_rs_ipv6));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "TestOR7890123456789",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerstatuses. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  memcpy(&mock_rs_ipv6, &mock_rs_dual, sizeof(mock_rs_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+  mock_rs_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = routerstatus_describe(&mock_rs_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = routerstatus_describe(&mock_rs_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = routerstatus_describe(&mock_rs_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = routerstatus_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerstatuses */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.rs = &mock_rs_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.rs = &mock_rs_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.rs = &mock_rs_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** extend_info_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ */
+static void
+test_nodelist_extend_info_describe(void *arg)
+{
+  extend_info_t mock_ei_ipv4;
+  extend_info_t mock_ei_ipv6;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ei_ipv4, 0, sizeof(mock_ei_ipv4));
+  memset(&mock_ei_ipv6, 0, sizeof(mock_ei_ipv6));
+
+  /* Set up the IPv4 extend info */
+  memcpy(mock_ei_ipv4.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ei_ipv4.identity_digest));
+  strlcpy(mock_ei_ipv4.nickname, "TestOR7890123456789",
+          sizeof(mock_ei_ipv4.nickname));
+  tor_addr_parse(&mock_ei_ipv4.addr, "111.222.233.244");
+
+  /* Create and modify the other extend info. */
+  memcpy(&mock_ei_ipv6, &mock_ei_ipv4, sizeof(mock_ei_ipv6));
+  tor_addr_parse(&mock_ei_ipv6.addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = extend_info_describe(&mock_ei_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = extend_info_describe(&mock_ei_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Extend infos only have one IP address, so there is no dual case */
+
+  /* test NULL handling */
+  rv = extend_info_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+ done:
+  return;
+}
+
+/** router_get_verbose_nickname() should return "Fingerprint~Nickname"
+ */
+static void
+test_nodelist_router_get_verbose_nickname(void *arg)
+{
+  routerinfo_t mock_ri;
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+
+  char vname[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  (void) arg;
+
+  memset(&mock_ri, 0, sizeof(routerinfo_t));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  mock_ri.nickname = mock_nickname;
+
+  /* verbose nickname should use ~ because named is deprecated */
+  strlcpy(mock_nickname, "TestOR", sizeof(mock_nickname));
+  memcpy(mock_ri.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         DIGEST_LEN);
+  router_get_verbose_nickname(vname, &mock_ri);
+  tt_str_op(vname, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR");
+
+  /* test NULL router handling */
+  router_get_verbose_nickname(vname, NULL);
+  tt_str_op(vname, OP_EQ, "<null>");
+
+  router_get_verbose_nickname(NULL, &mock_ri);
+  router_get_verbose_nickname(NULL, NULL);
+
+ done:
+  return;
+}
+
 #define NODE(name, flags) \
   { #name, test_nodelist_##name, (flags), NULL, NULL }
 
@@ -239,6 +1254,17 @@ struct testcase_t nodelist_tests[] = {
   NODE(node_get_verbose_nickname_not_named, TT_FORK),
   NODE(node_is_dir, TT_FORK),
   NODE(ed_id, TT_FORK),
+  NODE(nodefamily, TT_FORK),
+  NODE(nodefamily_parse_err, TT_FORK),
+  NODE(nodefamily_lookup, TT_FORK),
+  NODE(nickname_matches, 0),
+  NODE(node_nodefamily, TT_FORK),
+  NODE(nodefamily_canonicalize, 0),
+  NODE(format_node_description, 0),
+  NODE(router_describe, 0),
+  NODE(node_describe, 0),
+  NODE(routerstatus_describe, 0),
+  NODE(extend_info_describe, 0),
+  NODE(router_get_verbose_nickname, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_oom.c b/src/test/test_oom.c
index b813fa43a3..da6b2ee14d 100644
--- a/src/test/test_oom.c
+++ b/src/test/test_oom.c
@@ -8,7 +8,7 @@
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "lib/evloop/compat_libevent.h"
 #include "core/mainloop/connection.h"
diff --git a/src/test/test_options.c b/src/test/test_options.c
index 0e52967a23..a2641425fe 100644
--- a/src/test/test_options.c
+++ b/src/test/test_options.c
@@ -5,7 +5,7 @@
 
 #define CONFIG_PRIVATE
 #include "core/or/or.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "app/config/config.h"
 #include "test/test.h"
 #include "lib/geoip/geoip.h"
@@ -14,6 +14,7 @@
 #include "feature/nodelist/routerset.h"
 #include "core/mainloop/mainloop.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "lib/sandbox/sandbox.h"
 #include "lib/memarea/memarea.h"
@@ -31,14 +32,14 @@
 
 typedef struct {
   int severity;
-  uint32_t domain;
+  log_domain_mask_t domain;
   char *msg;
 } logmsg_t;
 
 static smartlist_t *messages = NULL;
 
 static void
-log_cback(int severity, uint32_t domain, const char *msg)
+log_cback(int severity, log_domain_mask_t domain, const char *msg)
 {
   logmsg_t *x = tor_malloc(sizeof(*x));
   x->severity = severity;
@@ -54,9 +55,9 @@ setup_log_callback(void)
 {
   log_severity_list_t lst;
   memset(&lst, 0, sizeof(lst));
-  lst.masks[LOG_ERR - LOG_ERR] = ~0;
-  lst.masks[LOG_WARN - LOG_ERR] = ~0;
-  lst.masks[LOG_NOTICE - LOG_ERR] = ~0;
+  lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = LD_ALL_DOMAINS;
   add_callback_log(&lst, log_cback);
   mark_logs_temp();
 }
@@ -96,7 +97,7 @@ clear_log_messages(void)
     opt->command = CMD_RUN_TOR;              \
     options_init(opt);                       \
                                              \
-    dflt = config_dup(&options_format, opt); \
+    dflt = config_dup(get_options_mgr(), opt); \
     clear_log_messages();                    \
   } while (0)
 
@@ -196,7 +197,7 @@ test_options_validate_impl(const char *configuration,
   if (r)
     goto done;
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   if (phase == PH_ASSIGN) {
     if (test_options_checkmsgs(configuration, expect_errmsg,
                                expect_log_severity,
@@ -241,6 +242,7 @@ test_options_validate(void *arg)
   (void)arg;
   setup_log_callback();
   sandbox_disable_getaddrinfo_cache();
+  mock_hostname_resolver();
 
   WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE);
 
@@ -258,13 +260,17 @@ test_options_validate(void *arg)
   WANT_ERR("BridgeRelay 1\nDirCache 0",
            "We're a bridge but DirCache is disabled.", PH_VALIDATE);
 
+  // XXXX We should replace this with a more full error message once #29211
+  // XXXX is done.  It is truncated for now because at the current stage
+  // XXXX of refactoring, we can't give a full error message like before.
   WANT_ERR_LOG("HeartbeatPeriod 21 snarks",
-               "Interval 'HeartbeatPeriod 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               "malformed or out of bounds", LOG_WARN,
+               "Unknown unit 'snarks'.",
                PH_ASSIGN);
+  // XXXX As above.
   WANT_ERR_LOG("LogTimeGranularity 21 snarks",
-               "Msec interval 'LogTimeGranularity 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               "malformed or out of bounds", LOG_WARN,
+               "Unknown unit 'snarks'.",
                PH_ASSIGN);
   OK("HeartbeatPeriod 1 hour", PH_VALIDATE);
   OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE);
@@ -278,6 +284,7 @@ test_options_validate(void *arg)
 
   close_temp_logs();
   clear_log_messages();
+  unmock_hostname_resolver();
   return;
 }
 
@@ -300,7 +307,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled */
@@ -323,7 +330,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled, Bridge */
@@ -346,7 +353,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 200 MB RAM available, DirCache disabled */
@@ -429,10 +436,12 @@ get_options_test_data(const char *conf)
   // with options_init(), but about a dozen tests break when I do that.
   // Being kinda lame and just fixing the immedate breakage for now..
   result->opt->ConnectionPadding = -1; // default must be "auto"
+  result->opt->DormantClientTimeout = 1800; // must be over 600.
+  result->opt->CircuitPadding = 1; // default must be "1"
 
   rv = config_get_lines(conf, &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->opt, cl, 0, &msg);
+  rv = config_assign(get_options_mgr(), result->opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
@@ -443,7 +452,7 @@ get_options_test_data(const char *conf)
   result->opt->TokenBucketRefillInterval = 1;
   rv = config_get_lines(TEST_OPTIONS_OLD_VALUES, &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->def_opt, cl, 0, &msg);
+  rv = config_assign(get_options_mgr(), result->def_opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
@@ -1171,7 +1180,7 @@ test_options_validate__transproxy(void *ignored)
   // Assert that a test has run for some TransProxyType
   tt_assert(tdata);
 
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   tdata = get_options_test_data("TransPort 127.0.0.1:555\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -1346,29 +1355,6 @@ test_options_validate__token_bucket(void *ignored)
 }
 
 static void
-test_options_validate__recommended_packages(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_WARN);
-  options_test_data_t *tdata = get_options_test_data(
-            "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n"
-            "RecommendedPackages invalid-package-line\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg("Invalid RecommendedPackage line "
-            "invalid-package-line will be ignored\n");
-
- done:
-  escaped(NULL); // This will free the leaking memory from the previous escaped
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__fetch_dir(void *ignored)
 {
   (void)ignored;
@@ -4203,7 +4189,6 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(exclude_nodes),
   LOCAL_VALIDATE_TEST(node_families),
   LOCAL_VALIDATE_TEST(token_bucket),
-  LOCAL_VALIDATE_TEST(recommended_packages),
   LOCAL_VALIDATE_TEST(fetch_dir),
   LOCAL_VALIDATE_TEST(conn_limit),
   LOCAL_VALIDATE_TEST(paths_needed),
diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c
new file mode 100644
index 0000000000..0c8f467a45
--- /dev/null
+++ b/src/test/test_parsecommon.c
@@ -0,0 +1,594 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/memarea/memarea.h"
+#include "lib/encoding/binascii.h"
+#include "feature/dirparse/parsecommon.h"
+#include "test/log_test_helpers.h"
+
+static void
+test_parsecommon_tokenize_string_null(void *arg)
+{
+
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  const char *str_with_null = "a\0bccccccccc";
+
+  int retval =
+  tokenize_string(area, str_with_null,
+                  str_with_null + 3,
+                  tokens, NULL, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_multiple_lines(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, GE(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup(
+          "hibernating 0\nuptime 1024\n"
+          "published 2018-10-15 10:00:00\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(smartlist_len(tokens), OP_EQ, 3);
+  directory_token_t *token = smartlist_get(tokens, 0);
+
+  tt_int_op(token->tp, OP_EQ, K_HIBERNATING);
+
+  token = smartlist_get(tokens, 1);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+
+  token = smartlist_get(tokens, 2);
+
+  tt_int_op(token->tp, OP_EQ, K_PUBLISHED);
+
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_min_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(2), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // Missing "uptime"
+  char *str = tor_strdup("uptime 1024\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_max_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "uptime" expected once, but occurs twice in input.
+  char *str = tor_strdup(
+                  "uptime 1024\nuptime 2048\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_start(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the first line.
+  char *str = tor_strdup(
+                  "uptime 1024\nclient-name Alice\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_end(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the last line.
+  char *str = tor_strdup(
+                  "client-name Alice\nuptime 1024\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_no_annotations(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_success(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "1024");
+  tt_assert(!token->object_type);
+  tt_int_op(token->object_size, OP_EQ, 0);
+  tt_assert(!token->object_body);
+
+  tt_ptr_op(*s, OP_EQ, end);
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_concat_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "proto A=1 B=2";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_PROTO);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "A=1 B=2");
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_parse_keys(void *arg)
+{
+  (void)arg;
+
+  memarea_t *area = memarea_new();
+  const char *str =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n"
+    "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n"
+    "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = (const char **)&str;
+  directory_token_t *token = NULL;
+  directory_token_t *token2 = NULL;
+
+  token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024);
+
+  token = get_next_token(area, s, end, &rule);
+  tt_assert(token);
+
+  tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY);
+  tt_int_op(token->n_args, OP_EQ, 0);
+  tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY");
+  tt_int_op(token->object_size, OP_EQ, 140);
+  tt_assert(token->object_body);
+  tt_assert(token->key);
+  tt_assert(!token->error);
+
+  const char *str2 =
+    "client-key\n"
+    "-----BEGIN RSA PRIVATE KEY-----\n"
+    "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n"
+    "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n"
+    "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n"
+    "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n"
+    "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n"
+    "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n"
+    "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n"
+    "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n"
+    "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n"
+    "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n"
+    "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n"
+    "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n"
+    "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"
+    "-----END RSA PRIVATE KEY-----\n";
+
+  const char *end2 = str2 + strlen(str2);
+  const char **s2 = (const char **)&str2;
+
+  token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS,
+                           NEED_SKEY_1024);
+
+  token2 = get_next_token(area, s2, end2, &rule2);
+  tt_assert(token2);
+
+  tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY);
+  tt_int_op(token2->n_args, OP_EQ, 0);
+  tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY");
+  tt_int_op(token2->object_size, OP_EQ, 608);
+  tt_assert(token2->object_body);
+  tt_assert(token2->key);
+  tt_assert(!token->error);
+
+ done:
+  if (token) token_clear(token);
+  if (token2) token_clear(token2);
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_object(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE);
+  tt_int_op(token->n_args, OP_EQ, 2);
+  tt_str_op(token->args[0], OP_EQ,
+                  "0232AF901C31A04EE9848595AF9BB7620D4C5B2E");
+  tt_str_op(token->args[1], OP_EQ,
+                  "CD1FD971855430880D3C31E0331C5C55800C2F79");
+
+  tt_assert(!token->error);
+
+  char decoded[256];
+  const char *signature =
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+  tt_assert(signature);
+  size_t signature_len = strlen(signature);
+  base64_decode(decoded, sizeof(decoded), signature, signature_len);
+
+  tt_str_op(token->object_type, OP_EQ, "SIGNATURE");
+  tt_int_op(token->object_size, OP_EQ, 256);
+  tt_mem_op(token->object_body, OP_EQ, decoded, 256);
+
+  tt_assert(!token->key);
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_err_too_many_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024 1024 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too many arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_too_few_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too few arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_obj_missing_endline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_beginline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-Z---\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_tag_mismatch(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SOMETHINGELSE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ,
+            "Malformed object: mismatched end tag SIGNATURE");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_base64(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "%%@%%%%%%%!!!'\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+#define PARSECOMMON_TEST(name) \
+  { #name, test_parsecommon_ ## name, 0, NULL, NULL }
+
+struct testcase_t parsecommon_tests[] = {
+  PARSECOMMON_TEST(tokenize_string_null),
+  PARSECOMMON_TEST(tokenize_string_multiple_lines),
+  PARSECOMMON_TEST(tokenize_string_min_cnt),
+  PARSECOMMON_TEST(tokenize_string_max_cnt),
+  PARSECOMMON_TEST(tokenize_string_at_start),
+  PARSECOMMON_TEST(tokenize_string_at_end),
+  PARSECOMMON_TEST(tokenize_string_no_annotations),
+  PARSECOMMON_TEST(get_next_token_success),
+  PARSECOMMON_TEST(get_next_token_concat_args),
+  PARSECOMMON_TEST(get_next_token_parse_keys),
+  PARSECOMMON_TEST(get_next_token_object),
+  PARSECOMMON_TEST(get_next_token_err_too_many_args),
+  PARSECOMMON_TEST(get_next_token_err_too_few_args),
+  PARSECOMMON_TEST(get_next_token_err_obj_missing_endline),
+  PARSECOMMON_TEST(get_next_token_err_bad_beginline),
+  PARSECOMMON_TEST(get_next_token_err_tag_mismatch),
+  PARSECOMMON_TEST(get_next_token_err_bad_base64),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_parseconf.sh b/src/test/test_parseconf.sh
new file mode 100755
index 0000000000..eeb80cdfa7
--- /dev/null
+++ b/src/test/test_parseconf.sh
@@ -0,0 +1,204 @@
+#!/bin/sh
+# Copyright 2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+# Integration test script for verifying that Tor configurations are parsed as
+# we expect.
+#
+# Valid configurations are tested with --dump-config, which parses and
+# validates the configuration before writing it out.  We then make sure that
+# the result is what we expect, before parsing and dumping it again to make
+# sure that there is no change.
+#
+# Invalid configurations are tested with --verify-config, which parses
+# and validates the configuration. We capture its output and make sure that
+# it contains the error message we expect.
+
+# This script looks for its test cases as individual directories in
+# src/test/conf_examples/.  Each test may have these files:
+#
+# torrc -- Usually needed. This file is passed to Tor on the command line
+#      with the "-f" flag. (If you omit it, you'll test Tor's behavior when
+#      it receives a nonexistent configuration file.)
+#
+# torrc.defaults -- Optional. If present, it is passed to Tor on the command
+#      line with the --defaults-torrc option. If this file is absent, an empty
+#      file is passed instead to prevent Tor from reading the system defaults.
+#
+# cmdline -- Optional. If present, it contains command-line arguments that
+#      will be passed to Tor.
+#
+# expected -- If this file is present, then it should be the expected result
+#      of "--dump-config short" for this test case.  Exactly one of
+#      "expected" or "error" must be present, or the test will fail.
+#
+# error -- If this file is present, then it contains a regex that must be
+#      matched by some line in the output of "--verify-config", which must
+#      fail. Exactly one of "expected" or "error" must be present, or the
+#      test will fail.
+
+umask 077
+set -e
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+# make a safe space for temporary files
+DATA_DIR=$(mktemp -d -t tor_parseconf_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# This is where we look for examples
+EXAMPLEDIR="$(dirname "$0")"/conf_examples
+
+case "$(uname -s)" in
+    CYGWIN*) WINDOWS=1;;
+    MINGW*) WINDOWS=1;;
+    MSYS*) WINDOWS=1;;
+    *) WINDOWS=0;;
+esac
+
+####
+# BUG WORKAROUND FOR 31757:
+#  On Appveyor, it seems that Tor sometimes randomly fails to produce
+#  output with --dump-config.  Whil we are figuring this out, do not treat
+#  windows errors as hard failures.
+####
+if test "$WINDOWS" = 1; then
+    EXITCODE=0
+else
+    EXITCODE=1
+fi
+
+die() { echo "$1" >&2 ; exit "$EXITCODE"; }
+
+if test "$WINDOWS" = 1; then
+    FILTER="dos2unix"
+else
+    FILTER="cat"
+fi
+
+touch "${DATA_DIR}/EMPTY" || die "Couldn't create empty file."
+
+for dir in "${EXAMPLEDIR}"/*; do
+    if ! test -d "${dir}"; then
+       # Only count directories.
+       continue
+    fi
+
+    testname="$(basename "${dir}")"
+    # We use printf since "echo -n" is not standard
+    printf "%s: " "$testname"
+
+    PREV_DIR="$(pwd)"
+    cd "${dir}"
+
+    if test -f "./torrc.defaults"; then
+        DEFAULTS="./torrc.defaults"
+    else
+        DEFAULTS="${DATA_DIR}/EMPTY"
+    fi
+
+    if test -f "./cmdline"; then
+        CMDLINE="$(cat ./cmdline)"
+    else
+        CMDLINE=""
+    fi
+
+    if test -f "./expected"; then
+        if test -f "./error"; then
+            echo "FAIL: Found both ${dir}/expected and ${dir}/error."
+            echo "(Only one of these files should exist.)"
+            exit $EXITCODE
+        fi
+
+        # This case should succeed: run dump-config and see if it does.
+
+        "${TOR_BINARY}" -f "./torrc" \
+                        --defaults-torrc "${DEFAULTS}" \
+                        --dump-config short \
+                        ${CMDLINE} \
+                        | "${FILTER}" > "${DATA_DIR}/output.${testname}" \
+                        || die "Failure: Tor exited."
+
+        if cmp "./expected" "${DATA_DIR}/output.${testname}">/dev/null ; then
+            # Check round-trip.
+            "${TOR_BINARY}" -f "${DATA_DIR}/output.${testname}" \
+                            --defaults-torrc "${DATA_DIR}/empty" \
+                            --dump-config short \
+                            | "${FILTER}" \
+                            > "${DATA_DIR}/output_2.${testname}" \
+                        || die "Failure: Tor exited on round-trip."
+
+            if ! cmp "${DATA_DIR}/output.${testname}" \
+                 "${DATA_DIR}/output_2.${testname}"; then
+                echo "Failure: did not match on round-trip."
+                exit $EXITCODE
+            fi
+
+            echo "OK"
+        else
+            echo "FAIL"
+            if test "$(wc -c < "${DATA_DIR}/output.${testname}")" = 0; then
+                # There was no output -- probably we failed.
+                "${TOR_BINARY}" -f "./torrc" \
+                                --defaults-torrc "${DEFAULTS}" \
+                                --verify-config \
+                                ${CMDLINE} || true
+            fi
+            diff -u "./expected" "${DATA_DIR}/output.${testname}" || /bin/true
+            exit $EXITCODE
+        fi
+
+   elif test -f "./error"; then
+        # This case should fail: run verify-config and see if it does.
+
+        "${TOR_BINARY}" --verify-config \
+                        -f ./torrc \
+                        --defaults-torrc "${DEFAULTS}" \
+                        ${CMDLINE} \
+                        > "${DATA_DIR}/output.${testname}" \
+                        && die "Failure: Tor did not report an error."
+
+        expect_err="$(cat ./error)"
+        if grep "${expect_err}" "${DATA_DIR}/output.${testname}" >/dev/null; then
+            echo "OK"
+        else
+            echo "FAIL"
+            echo "Expected error: ${expect_err}"
+            echo "Tor said:"
+            cat "${DATA_DIR}/output.${testname}"
+            exit $EXITCODE
+        fi
+
+    else
+        # This case is not actually configured with a success or a failure.
+        # call that an error.
+
+        echo "FAIL: Did not find ${dir}/expected or ${dir}/error."
+        exit $EXITCODE
+    fi
+
+    cd "${PREV_DIR}"
+
+done
diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c
index 4a53639dad..267156a908 100644
--- a/src/test/test_periodic_event.c
+++ b/src/test/test_periodic_event.c
@@ -19,6 +19,7 @@
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_service.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/periodic.h"
 
 /** Helper function: This is replaced in some tests for the event callbacks so
@@ -50,16 +51,21 @@ test_pe_initialize(void *arg)
    * need to run the main loop and then wait for a second delaying the unit
    * tests. Instead, we'll test the callback work indepedently elsewhere. */
   initialize_periodic_events();
+  periodic_events_connect_all();
+  set_network_participation(false);
+  rescan_periodic_events(get_options());
 
   /* Validate that all events have been set up. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     tt_assert(item->ev);
     tt_assert(item->fn);
     tt_u64_op(item->last_action_time, OP_EQ, 0);
     /* Every event must have role(s) assign to it. This is done statically. */
     tt_u64_op(item->roles, OP_NE, 0);
-    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
  done:
@@ -79,17 +85,20 @@ test_pe_launch(void *arg)
    * network gets enabled. */
   consider_hibernation(time(NULL));
 
+  set_network_participation(true);
+
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
   options = get_options_mutable();
   options->SocksPort_set = 1;
   periodic_events_on_new_options(options);
+
 #if 0
   /* Lets make sure that before intialization, we can't scan the periodic
    * events list and launch them. Lets try by being a Client. */
@@ -99,20 +108,20 @@ test_pe_launch(void *arg)
     periodic_event_item_t *item = &periodic_events[i];
     tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
   }
-#endif
+#endif /* 0 */
 
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Now that we've initialized, rescan the list to launch. */
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
-    } else {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
-    }
+  int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL|
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = !!(item->roles & mask);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
     // enabled or not, the event has not yet been run.
     tt_u64_op(item->last_action_time, OP_EQ, 0);
   }
@@ -124,10 +133,11 @@ test_pe_launch(void *arg)
 
   unsigned roles = get_my_roles(options);
   tt_uint_op(roles, OP_EQ,
-             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER);
+             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER|
+             PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     /* Only Client role should be disabled. */
     if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
@@ -144,17 +154,23 @@ test_pe_launch(void *arg)
   /* Disable everything and we'll enable them ALL. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
+  options->DisableNetwork = 1;
+  set_network_participation(false);
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
   /* Enable everything. */
   options->SocksPort_set = 1; options->ORPort_set = 1;
   options->BridgeRelay = 1; options->AuthoritativeDir = 1;
   options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1;
+  options->DisableNetwork = 0;
+  set_network_participation(true);
   register_dummy_hidden_service(&service);
   periodic_events_on_new_options(options);
   /* Note down the reference because we need to remove this service from the
@@ -163,9 +179,10 @@ test_pe_launch(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ,
+              (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV));
   }
 
  done:
@@ -187,42 +204,49 @@ test_pe_get_roles(void *arg)
 
   or_options_t *options = get_options_mutable();
   tt_assert(options);
+  set_network_participation(true);
+
+  const int ALL = PERIODIC_EVENT_ROLE_ALL |
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
 
   /* Nothing configured, should be no roles. */
+  tt_assert(net_is_disabled());
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Indicate we have a SocksPort, roles should be come Client. */
   options->SocksPort_set = 1;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT);
+  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL);
 
   /* Now, we'll add a ORPort so should now be a Relay + Client. */
   options->ORPort_set = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_DIRSERVER | ALL));
 
   /* Now add a Bridge. */
   options->BridgeRelay = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER);
   /* Unset client so we can solely test Router role. */
   options->SocksPort_set = 0;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER |
+            ALL);
 
   /* Reset options so we can test authorities. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
   options->BridgeRelay = 0;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Now upgrade to Dirauth. */
   options->DirPort_set = 1;
@@ -230,7 +254,7 @@ test_pe_get_roles(void *arg)
   options->V3AuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Now Bridge Authority. */
@@ -238,7 +262,7 @@ test_pe_get_roles(void *arg)
   options->BridgeAuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Move that bridge auth to become a relay. */
@@ -246,7 +270,7 @@ test_pe_get_roles(void *arg)
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY
-             | PERIODIC_EVENT_ROLE_DIRSERVER));
+             | PERIODIC_EVENT_ROLE_DIRSERVER|ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* And now an Hidden service. */
@@ -257,7 +281,8 @@ test_pe_get_roles(void *arg)
   remove_service(get_hs_service_map(), &service);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
  done:
@@ -277,12 +302,13 @@ test_pe_hs_service(void *arg)
   consider_hibernation(time(NULL));
   /* Initialize the events so we can enable them */
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
@@ -295,8 +321,8 @@ test_pe_hs_service(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
     }
@@ -306,8 +332,8 @@ test_pe_hs_service(void *arg)
   /* Remove the service from the global map, it should trigger a rescan and
    * disable the HS service events. */
   remove_service(get_hs_service_map(), &service);
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
     }
diff --git a/src/test/test_policy.c b/src/test/test_policy.c
index 9c001c294a..e58bb3d174 100644
--- a/src/test/test_policy.c
+++ b/src/test/test_policy.c
@@ -6,13 +6,18 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/circuitbuild.h"
 #include "core/or/policies.h"
 #include "feature/dirparse/policy_parse.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_descriptor.h"
 #include "feature/relay/router.h"
 #include "lib/encoding/confline.h"
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #include "core/or/addr_policy_st.h"
+#include "core/or/extend_info_st.h"
 #include "core/or/port_cfg_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -2024,6 +2029,115 @@ test_policies_fascist_firewall_allows_address(void *arg)
                            expect_ap); \
   STMT_END
 
+/* Check that fascist_firewall_choose_address_ls() returns the expected
+ * results. */
+#define CHECK_CHOSEN_ADDR_NULL_LS() \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(NULL, 1, &chosen_ls_ap); \
+    expect_single_log_msg("Unknown or missing link specifiers"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_CHOSEN_ADDR_LS(fake_ls, pref_only, expect_rv, expect_ap) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, pref_only, &chosen_ls_ap); \
+    if (smartlist_len(fake_ls) == 0) { \
+      expect_single_log_msg("Link specifiers are empty"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_ls_ap.addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, chosen_ls_ap.port); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_LS_LEGACY_ONLY(fake_ls) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, 0, &chosen_ls_ap); \
+    expect_single_log_msg("None of our link specifiers have IPv4 or IPv6"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS(fake_ls, direct_conn, expect_ap) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    if (fake_ls == NULL) { \
+      tt_ptr_op(ei, OP_EQ, NULL); \
+      expect_single_log_msg("Specified link specifiers is null"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &ei->addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, ei->port); \
+      extend_info_free(ei); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(fake_ls) \
+  STMT_BEGIN \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, NULL, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg("Specified onion key is null"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(fake_ls, direct_conn) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(fake_ls, msg_level, msg) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(msg_level); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg(msg); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+/** Mock the preferred address function to return zero (prefer IPv4). */
+static int
+mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void)
+{
+  return 0;
+}
+
+/** Mock the preferred address function to return one (prefer IPv6). */
+static int
+mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6(void)
+{
+  return 1;
+}
+
 /** Run unit tests for fascist_firewall_choose_address */
 static void
 test_policies_fascist_firewall_choose_address(void *arg)
@@ -2422,6 +2536,177 @@ test_policies_fascist_firewall_choose_address(void *arg)
   CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1,
                        ipv4_dir_ap);
 
+  /* Test ClientAutoIPv6ORPort and pretend we prefer IPv4. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientAutoIPv6ORPort = 1;
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+  MOCK(fascist_firewall_rand_prefer_ipv6_addr,
+       mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4);
+  /* Simulate the initialisation of fake_node.ipv6_preferred */
+  fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(
+                                                                &mock_options);
+
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1,
+                       ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1,
+                       ipv4_or_ap);
+
+  UNMOCK(fascist_firewall_rand_prefer_ipv6_addr);
+
+  /* Test ClientAutoIPv6ORPort and pretend we prefer IPv6. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientAutoIPv6ORPort = 1;
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+  MOCK(fascist_firewall_rand_prefer_ipv6_addr,
+       mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6);
+  /* Simulate the initialisation of fake_node.ipv6_preferred */
+  fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(
+                                                                &mock_options);
+
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1,
+                       ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1,
+                       ipv6_or_ap);
+
+  UNMOCK(fascist_firewall_rand_prefer_ipv6_addr);
+
+  /* Test firewall_choose_address_ls(). To do this, we make a fake link
+   * specifier. */
+  smartlist_t *lspecs = smartlist_new(),
+              *lspecs_blank = smartlist_new(),
+              *lspecs_v4 = smartlist_new(),
+              *lspecs_v6 = smartlist_new(),
+              *lspecs_no_legacy = smartlist_new(),
+              *lspecs_legacy_only = smartlist_new();
+  link_specifier_t *fake_ls;
+
+  /* IPv4 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(fake_ls,
+                                  tor_addr_to_ipv4h(&ipv4_or_ap.addr));
+  link_specifier_set_un_ipv4_port(fake_ls, ipv4_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, sizeof(ipv4_or_ap.addr.addr.in_addr) +
+                            sizeof(ipv4_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_no_legacy, fake_ls);
+
+  /* IPv6 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV6);
+  size_t addr_len = link_specifier_getlen_un_ipv6_addr(fake_ls);
+  const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ipv6_or_ap.addr);
+  uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(fake_ls);
+  memcpy(ipv6_array, in6_addr, addr_len);
+  link_specifier_set_un_ipv6_port(fake_ls, ipv6_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, addr_len + sizeof(ipv6_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Legacy ID link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_LEGACY_ID);
+  uint8_t *legacy_id = link_specifier_getarray_un_legacy_id(fake_ls);
+  memset(legacy_id, 'A', sizeof(*legacy_id));
+  link_specifier_set_ls_len(fake_ls,
+                            link_specifier_getlen_un_legacy_id(fake_ls));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_legacy_only, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Check with bogus requests. */
+  tor_addr_port_t null_ap; \
+  tor_addr_make_null(&null_ap.addr, AF_UNSPEC); \
+  null_ap.port = 0; \
+
+  /* Check for a null link state. */
+  CHECK_CHOSEN_ADDR_NULL_LS();
+  CHECK_HS_EXTEND_INFO_ADDR_LS(NULL, 1, null_ap);
+
+  /* Check for a blank link state. */
+  CHECK_CHOSEN_ADDR_LS(lspecs_blank, 0, 0, null_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_blank, 0);
+
+  /* Check for a link state with only a Legacy ID. */
+  CHECK_LS_LEGACY_ONLY(lspecs_legacy_only);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_legacy_only, 0);
+  smartlist_free(lspecs_legacy_only);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(lspecs_blank);
+  smartlist_free(lspecs_blank);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(lspecs_no_legacy, LOG_WARN,
+                                          "Missing Legacy ID in link state");
+  smartlist_free(lspecs_no_legacy);
+
+  /* Enable both IPv4 and IPv6. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Prefer IPv4, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* Prefer IPv6, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* IPv4-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v6, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 1);
+
+  /* IPv6-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 0;
+  mock_options.ClientUseIPv6 = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v4, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v4, 1);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+
+  smartlist_free(lspecs_v4);
+  smartlist_free(lspecs_v6);
+
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, lspec, \
+                    link_specifier_free(lspec)); \
+  smartlist_free(lspecs);
+
  done:
   UNMOCK(get_options);
 }
diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c
new file mode 100644
index 0000000000..0ecbf65f41
--- /dev/null
+++ b/src/test/test_prob_distr.c
@@ -0,0 +1,1402 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_prob_distr.c
+ * \brief Test probability distributions.
+ * \detail
+ *
+ * For each probability distribution we do two kinds of tests:
+ *
+ * a) We do numerical deterministic testing of their cdf/icdf/sf/isf functions
+ *    and the various relationships between them for each distribution. We also
+ *    do deterministic tests on their sampling functions. Test vectors for
+ *    these tests were computed from alternative implementations and were
+ *    eyeballed to make sure they make sense
+ *    (e.g. src/test/prob_distr_mpfr_ref.c computes logit(p) using GNU mpfr
+ *    with 200-bit precision and is then tested in test_logit_logistic()).
+ *
+ * b) We do stochastic hypothesis testing (G-test) to ensure that sampling from
+ *    the given distributions is distributed properly. The stochastic tests are
+ *    slow and their false positive rate is not well suited for CI, so they are
+ *    currently disabled-by-default and put into 'tests-slow'.
+ */
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "test/test.h"
+
+#include "core/or/or.h"
+
+#include "lib/math/prob_distr.h"
+#include "lib/math/fp.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "test/rng_test_helpers.h"
+
+#include <float.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/**
+ * Return floor(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)floor(d).
+ */
+static size_t
+floor_to_size_t(double d)
+{
+  double integral_d = floor(d);
+  return (size_t)integral_d;
+}
+
+/**
+ * Return ceil(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)ceil(d).
+ */
+static size_t
+ceil_to_size_t(double d)
+{
+  double integral_d = ceil(d);
+  return (size_t)integral_d;
+}
+
+/*
+ * Geometric(p) distribution, supported on {1, 2, 3, ...}.
+ *
+ * Compute the probability mass function Geom(n; p) of the number of
+ * trials before the first success when success has probability p.
+ */
+static double
+logpmf_geometric(unsigned n, double p)
+{
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1) {
+    if (n == 1)
+      return 0;
+    else
+      return -HUGE_VAL;
+  }
+  return (n - 1)*log1p(-p) + log(p);
+}
+
+/**
+ * Compute the logistic function, translated in output by 1/2:
+ * logistichalf(x) = logistic(x) - 1/2.  Well-conditioned on the entire
+ * real plane, with maximum condition number 1 at 0.
+ *
+ * This implementation gives relative error bounded by 5 eps.
+ */
+static double
+logistichalf(double x)
+{
+  /*
+   * Rewrite this with the identity
+   *
+   *  1/(1 + e^{-x}) - 1/2
+   *  = (1 - 1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1 - e^{-x})/[2 (1 + e^{-x})]
+   *  = -(e^{-x} - 1)/[2 (1 + e^{-x})],
+   *
+   * which we can evaluate by -expm1(-x)/[2 (1 + exp(-x))].
+   *
+   * Suppose exp has error d0, + has error d1, expm1 has error
+   * d2, and / has error d3, so we evaluate
+   *
+   *  -(1 + d2) (1 + d3) (e^{-x} - 1)
+   *    / [2 (1 + d1) (1 + (1 + d0) e^{-x})].
+   *
+   * In the denominator,
+   *
+   *  1 + (1 + d0) e^{-x}
+   *  = 1 + e^{-x} + d0 e^{-x}
+   *  = (1 + e^{-x}) (1 + d0 e^{-x}/(1 + e^{-x})),
+   *
+   * so the relative error of the numerator is
+   *
+   *  d' = d2 + d3 + d2 d3,
+   * and of the denominator,
+   *  d'' = d1 + d0 e^{-x}/(1 + e^{-x}) + d0 d1 e^{-x}/(1 + e^{-x})
+   *      = d1 + d0 L(-x) + d0 d1 L(-x),
+   *
+   * where L(-x) is logistic(-x).  By Lemma 1 the relative error
+   * of the quotient is bounded by
+   *
+   *  2|d2 + d3 + d2 d3 - d1 - d0 L(x) + d0 d1 L(x)|,
+   *
+   * Since 0 < L(x) < 1, this is bounded by
+   *
+   *  2|d2| + 2|d3| + 2|d2 d3| + 2|d1| + 2|d0| + 2|d0 d1|
+   *  <= 4 eps + 2 eps^2.
+   */
+  if (x < log(DBL_EPSILON/8)) {
+    /*
+     * Avoid overflow in e^{-x}.  When x < log(eps/4), we
+     * we further have x < logit(eps/4), so that
+     * logistic(x) < eps/4.  Hence the relative error of
+     * logistic(x) - 1/2 from -1/2 is bounded by eps/2, and
+     * so the relative error of -1/2 from logistic(x) - 1/2
+     * is bounded by eps.
+     */
+    return -0.5;
+  } else {
+    return -expm1(-x)/(2*(1 + exp(-x)));
+  }
+}
+
+/**
+ * Compute the log of the sum of the exps.  Caller should arrange the
+ * array in descending order to minimize error because I don't want to
+ * deal with using temporary space and the one caller in this file
+ * arranges that anyway.
+ *
+ * Warning: This implementation does not handle infinite or NaN inputs
+ * sensibly, because I don't need that here at the moment.  (NaN, or
+ * -inf and +inf together, should yield NaN; +inf and finite should
+ * yield +inf; otherwise all -inf should be ignored because exp(-inf) =
+ * 0.)
+ */
+static double
+logsumexp(double *A, size_t n)
+{
+  double maximum, sum;
+  size_t i;
+
+  if (n == 0)
+    return log(0);
+
+  maximum = A[0];
+  for (i = 1; i < n; i++) {
+    if (A[i] > maximum)
+      maximum = A[i];
+  }
+
+  sum = 0;
+  for (i = n; i --> 0;)
+    sum += exp(A[i] - maximum);
+
+  return log(sum) + maximum;
+}
+
+/**
+ * Compute log(1 - e^x).  Defined only for negative x so that e^x < 1.
+ * This is the complement of a probability in log space.
+ */
+static double
+log1mexp(double x)
+{
+
+  /*
+   * We want to compute log on [0, 1/2) but log1p on [1/2, +inf),
+   * so partition x at -log(2) = log(1/2).
+   */
+  if (-log(2) < x)
+    return log(-expm1(x));
+  else
+    return log1p(-exp(x));
+}
+
+/*
+ * Tests of numerical errors in computing logit, logistic, and the
+ * various cdfs, sfs, icdfs, and isfs.
+ */
+
+#define arraycount(A) (sizeof(A)/sizeof(A[0]))
+
+/** Return relative error between <b>actual</b> and <b>expected</b>.
+ *  Special cases: If <b>expected</b> is zero or infinite, return 1 if
+ *  <b>actual</b> is equal to <b>expected</b> and 0 if not, since the
+ *  usual notion of relative error is undefined but we only use this
+ *  for testing relerr(e, a) <= bound.  If either is NaN, return NaN,
+ *  which has the property that NaN <= bound is false no matter what
+ *  bound is.
+ *
+ *  Beware: if you test !(relerr(e, a) > bound), then then the result
+ *  is true when a is NaN because NaN > bound is false too.  See
+ *  CHECK_RELERR for correct use to decide when to report failure.
+ */
+static double
+relerr(double expected, double actual)
+{
+  /*
+   * To silence -Wfloat-equal, we have to test for equality using
+   * inequalities: we have (fabs(expected) <= 0) iff (expected == 0),
+   * and (actual <= expected && actual >= expected) iff actual ==
+   * expected whether expected is zero or infinite.
+   */
+  if (fabs(expected) <= 0 || tor_isinf(expected)) {
+    if (actual <= expected && actual >= expected)
+      return 0;
+    else
+      return 1;
+  } else {
+    return fabs((expected - actual)/expected);
+  }
+}
+
+/** Check that relative error of <b>expected</b> and <b>actual</b> is within
+ *  <b>relerr_bound</b>.  Caller must arrange to have i and relerr_bound in
+ *  scope.  */
+#define CHECK_RELERR(expected, actual) do {                                   \
+  double check_expected = (expected);                                         \
+  double check_actual = (actual);                                             \
+  const char *str_expected = #expected;                                       \
+  const char *str_actual = #actual;                                           \
+  double check_relerr = relerr(expected, actual);                             \
+  if (!(relerr(check_expected, check_actual) <= relerr_bound)) {              \
+    log_warn(LD_GENERAL, "%s:%d: case %u: relerr(%s=%.17e, %s=%.17e)"        \
+             " = %.17e > %.17e\n",                                            \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_expected, check_expected,                                    \
+             str_actual, check_actual,                                        \
+             check_relerr, relerr_bound);                                     \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/* Check that a <= b.
+ * Caller must arrange to have i in scope.  */
+#define CHECK_LE(a, b) do {                                                   \
+  double check_a = (a);                                                       \
+  double check_b = (b);                                                       \
+  const char *str_a = #a;                                                     \
+  const char *str_b = #b;                                                     \
+  if (!(check_a <= check_b)) {                                                \
+    log_warn(LD_GENERAL, "%s:%d: case %u: %s=%.17e > %s=%.17e\n",             \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_a, check_a, str_b, check_b);                                 \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/**
+ * Test the logit and logistic functions.  Confirm that they agree with
+ * the cdf, sf, icdf, and isf of the standard Logistic distribution.
+ * Confirm that the sampler for the standard logistic distribution maps
+ * [0, 1] into the right subinterval for the inverse transform, for
+ * this implementation.
+ */
+static void
+test_logit_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    double x;                   /* x = logit(p) */
+    double p;                   /* p = logistic(x) */
+    double phalf;               /* p - 1/2 = logistic(x) - 1/2 */
+  } cases[] = {
+    { -HUGE_VAL, 0, -0.5 },
+    { -1000, 0, -0.5 },
+    { -710, 4.47628622567513e-309, -0.5 },
+    { -708, 3.307553003638408e-308, -0.5 },
+    { -2, .11920292202211755, -.3807970779778824 },
+    { -1.0000001, .2689414017088022, -.23105859829119776 },
+    { -1, .2689414213699951, -.23105857863000487 },
+    { -0.9999999, .26894144103118883, -.2310585589688111 },
+    /* see src/test/prob_distr_mpfr_ref.c for computation */
+    { -4.000000000537333e-5, .49999, -1.0000000000010001e-5 },
+    { -4.000000000533334e-5, .49999, -.00001 },
+    { -4.000000108916878e-9, .499999999, -1.0000000272292198e-9 },
+    { -4e-9, .499999999, -1e-9 },
+    { -4e-16, .5, -1e-16 },
+    { -4e-300, .5, -1e-300 },
+    { 0, .5, 0 },
+    { 4e-300, .5, 1e-300 },
+    { 4e-16, .5, 1e-16 },
+    { 3.999999886872274e-9, .500000001, 9.999999717180685e-10 },
+    { 4e-9, .500000001, 1e-9 },
+    { 4.0000000005333336e-5, .50001, .00001 },
+    { 8.000042667076272e-3, .502, .002 },
+    { 0.9999999, .7310585589688111, .2310585589688111 },
+    { 1, .7310585786300049, .23105857863000487 },
+    { 1.0000001, .7310585982911977, .23105859829119774 },
+    { 2, .8807970779778823, .3807970779778824 },
+    { 708, 1, .5 },
+    { 710, 1, .5 },
+    { 1000, 1, .5 },
+    { HUGE_VAL, 1, .5 },
+  };
+  double relerr_bound = 3e-15; /* >10eps */
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double phalf = cases[i].phalf;
+
+    /*
+     * cdf is logistic, icdf is logit, and symmetry for
+     * sf/isf.
+     */
+    CHECK_RELERR(logistic(x), cdf_logistic(x, 0, 1));
+    CHECK_RELERR(logistic(-x), sf_logistic(x, 0, 1));
+    CHECK_RELERR(logit(p), icdf_logistic(p, 0, 1));
+    CHECK_RELERR(-logit(p), isf_logistic(p, 0, 1));
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2, 0, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2, 0, 2));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0, 2)/2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, 2)/2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x/2, 0, .5));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x/2, 0, .5));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0,.5)*2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, .5)*2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2 + 1, 1, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2 + 1, 1, 2));
+
+    /*
+     * For p near 0 and p near 1/2, the arithmetic of
+     * translating by 1 loses precision.
+     */
+    if (fabs(p) > DBL_EPSILON && fabs(p) < 0.4) {
+      CHECK_RELERR(icdf_logistic(p, 0, 1),
+          (icdf_logistic(p, 1, 2) - 1)/2);
+      CHECK_RELERR(isf_logistic(p, 0, 1),
+          (isf_logistic(p, 1, 2) - 1)/2);
+    }
+
+    CHECK_RELERR(p, logistic(x));
+    CHECK_RELERR(phalf, logistichalf(x));
+
+    /*
+     * On the interior floating-point numbers, either logit or
+     * logithalf had better give the correct answer.
+     *
+     * For probabilities near 0, we can get much finer resolution with
+     * logit, and for probabilities near 1/2, we can get much finer
+     * resolution with logithalf by representing them using p - 1/2.
+     *
+     * E.g., we can write -.00001 for phalf, and .49999 for p, but the
+     * difference 1/2 - .00001 gives 1.0000000000010001e-5 in binary64
+     * arithmetic.  So test logit(.49999) which should give the same
+     * answer as logithalf(-1.0000000000010001e-5), namely
+     * -4.000000000537333e-5, and also test logithalf(-.00001) which
+     * gives -4.000000000533334e-5 instead -- but don't expect
+     * logit(.49999) to give -4.000000000533334e-5 even though it looks
+     * like 1/2 - .00001.
+     *
+     * A naive implementation of logit will just use log(p/(1 - p)) and
+     * give the answer -4.000000000551673e-05 for .49999, which is
+     * wrong in a lot of digits, which happens because log is
+     * ill-conditioned near 1 and thus amplifies whatever relative
+     * error we made in computing p/(1 - p).
+     */
+    if ((0 < p && p < 1) || tor_isinf(x)) {
+      if (phalf >= p - 0.5 && phalf <= p - 0.5)
+        CHECK_RELERR(x, logit(p));
+      if (p >= 0.5 + phalf && p <= 0.5 + phalf)
+        CHECK_RELERR(x, logithalf(phalf));
+    }
+
+    CHECK_RELERR(-phalf, logistichalf(-x));
+    if (fabs(phalf) < 0.5 || tor_isinf(x))
+      CHECK_RELERR(-x, logithalf(-phalf));
+    if (p < 1 || tor_isinf(x)) {
+      CHECK_RELERR(1 - p, logistic(-x));
+      if (p > .75 || tor_isinf(x))
+        CHECK_RELERR(-x, logit(1 - p));
+    } else {
+      CHECK_LE(logistic(-x), 1e-300);
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(logit(p0/(1 + M_E)), sample_logistic(0, 0, p0));
+    CHECK_RELERR(-logit(p0/(1 + M_E)), sample_logistic(1, 0, p0));
+    CHECK_RELERR(logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(0, 1, p0));
+    CHECK_RELERR(-logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(1, 1, p0));
+  }
+
+  if (!ok)
+    printf("fail logit/logistic / logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the LogLogistic distribution.
+ */
+static void
+test_log_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the LogLogistic distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given LogLogistic
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given LogLogistic
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { 1e-15, 1e-15, .999999999999999 },
+    { .1, .09090909090909091, .90909090909090909 },
+    { .25, .2, .8 },
+    { .5, .33333333333333333, .66666666666666667 },
+    { .75, .42857142857142855, .5714285714285714 },
+    { .9999, .49997499874993756, .5000250012500626 },
+    { .99999999, .49999999749999996, .5000000025 },
+    { .999999999999999, .49999999999999994, .5000000000000002 },
+    { 1, .5, .5 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_log_logistic(x, 1, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(p, cdf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, sf_log_logistic(x, 1, 1));
+    CHECK_RELERR(np, sf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(np, sf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(np, sf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, cdf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(np, cdf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(np, cdf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(np, cdf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(np, cdf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(np, cdf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(p, sf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(p, sf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(p, sf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(p, sf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(p, sf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(p, sf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(x/2, icdf_log_logistic(p, .5, 1));
+    CHECK_RELERR(x*2, icdf_log_logistic(p, 2, 1));
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(sqrt(x)/2, icdf_log_logistic(p, .5, 2));
+    CHECK_RELERR(sqrt(x)*2, icdf_log_logistic(p, 2, 2));
+    CHECK_RELERR(sqrt(x), icdf_log_logistic(p, 1, 2));
+    CHECK_RELERR(x*x/2, icdf_log_logistic(p, .5, .5));
+    CHECK_RELERR(x*x*2, icdf_log_logistic(p, 2, .5));
+
+    if (np < .9) {
+      CHECK_RELERR(x, isf_log_logistic(np, 1, 1));
+      CHECK_RELERR(x/2, isf_log_logistic(np, .5, 1));
+      CHECK_RELERR(x*2, isf_log_logistic(np, 2, 1));
+      CHECK_RELERR(sqrt(x), isf_log_logistic(np, 1, 2));
+      CHECK_RELERR(sqrt(x)/2, isf_log_logistic(np, .5, 2));
+      CHECK_RELERR(sqrt(x)*2, isf_log_logistic(np, 2, 2));
+      CHECK_RELERR(x*x, isf_log_logistic(np, 1, .5));
+      CHECK_RELERR(x*x/2, isf_log_logistic(np, .5, .5));
+      CHECK_RELERR(x*x*2, isf_log_logistic(np, 2, .5));
+
+      CHECK_RELERR(1/x, icdf_log_logistic(np, 1, 1));
+      CHECK_RELERR(1/(2*x), icdf_log_logistic(np, .5, 1));
+      CHECK_RELERR(2/x, icdf_log_logistic(np, 2, 1));
+      CHECK_RELERR(1/sqrt(x), icdf_log_logistic(np, 1, 2));
+      CHECK_RELERR(1/(2*sqrt(x)),
+          icdf_log_logistic(np, .5, 2));
+      CHECK_RELERR(2/sqrt(x), icdf_log_logistic(np, 2, 2));
+      CHECK_RELERR(1/(x*x), icdf_log_logistic(np, 1, .5));
+      CHECK_RELERR(1/(2*x*x), icdf_log_logistic(np, .5, .5));
+      CHECK_RELERR(2/(x*x), icdf_log_logistic(np, 2, .5));
+    }
+
+    CHECK_RELERR(1/x, isf_log_logistic(p, 1, 1));
+    CHECK_RELERR(1/(2*x), isf_log_logistic(p, .5, 1));
+    CHECK_RELERR(2/x, isf_log_logistic(p, 2, 1));
+    CHECK_RELERR(1/sqrt(x), isf_log_logistic(p, 1, 2));
+    CHECK_RELERR(1/(2*sqrt(x)), isf_log_logistic(p, .5, 2));
+    CHECK_RELERR(2/sqrt(x), isf_log_logistic(p, 2, 2));
+    CHECK_RELERR(1/(x*x), isf_log_logistic(p, 1, .5));
+    CHECK_RELERR(1/(2*x*x), isf_log_logistic(p, .5, .5));
+    CHECK_RELERR(2/(x*x), isf_log_logistic(p, 2, .5));
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(0.5*p0/(1 - 0.5*p0), sample_log_logistic(0, p0));
+    CHECK_RELERR((1 - 0.5*p0)/(0.5*p0),
+        sample_log_logistic(1, p0));
+  }
+
+  if (!ok)
+    printf("fail log logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, isf of the Weibull distribution.
+ */
+static void
+test_weibull(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the Weibull distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given Weibull
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given Weibull
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { .1, .09516258196404043, .9048374180359595 },
+    { .5, .3934693402873666, .6065306597126334 },
+    { .6931471805599453, .5, .5 },
+    { 1, .6321205588285577, .36787944117144233 },
+    { 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 36, .9999999999999998, 2.319522830243569e-16 },
+    { 37, .9999999999999999, 8.533047625744066e-17 },
+    { 38, 1, 3.1391327920480296e-17 },
+    { 100, 1, 3.720075976020836e-44 },
+    { 708, 1, 3.307553003638408e-308 },
+    { 710, 1, 4.47628622567513e-309 },
+    { 1000, 1, 0 },
+    { HUGE_VAL, 1, 0 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_weibull(x, 1, 1));
+    CHECK_RELERR(p, cdf_weibull(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_weibull(x*2, 2, 1));
+    /* For 0 < x < sqrt(DBL_MIN), x^2 loses lots of bits.  */
+    if (x <= 0 ||
+        sqrt(DBL_MIN) <= x) {
+      CHECK_RELERR(p, cdf_weibull(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x*2, 2, .5));
+    }
+    CHECK_RELERR(p, cdf_weibull(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)*2, 2, 2));
+    CHECK_RELERR(np, sf_weibull(x, 1, 1));
+    CHECK_RELERR(np, sf_weibull(x/2, .5, 1));
+    CHECK_RELERR(np, sf_weibull(x*2, 2, 1));
+    CHECK_RELERR(np, sf_weibull(x*x, 1, .5));
+    CHECK_RELERR(np, sf_weibull(x*x/2, .5, .5));
+    CHECK_RELERR(np, sf_weibull(x*x*2, 2, .5));
+    if (x >= 10) {
+      /*
+       * exp amplifies the error of sqrt(x)^2
+       * proportionally to exp(x); for large inputs
+       * this is significant.
+       */
+      double t = -expm1(-x*(2*DBL_EPSILON + DBL_EPSILON));
+      relerr_bound = t + DBL_EPSILON + t*DBL_EPSILON;
+      if (relerr_bound < 3e-15)
+        /*
+         * The tests are written only to 16
+         * decimal places anyway even if your
+         * `double' is, say, i387 binary80, for
+         * whatever reason.
+         */
+        relerr_bound = 3e-15;
+      CHECK_RELERR(np, sf_weibull(sqrt(x), 1, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)/2, .5, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)*2, 2, 2));
+    }
+
+    if (p <= 0.75) {
+      /*
+       * For p near 1, not enough precision near 1 to
+       * recover x.
+       */
+      CHECK_RELERR(x, icdf_weibull(p, 1, 1));
+      CHECK_RELERR(x/2, icdf_weibull(p, .5, 1));
+      CHECK_RELERR(x*2, icdf_weibull(p, 2, 1));
+    }
+    if (p >= 0.25 && !tor_isinf(x) && np > 0) {
+      /*
+       * For p near 0, not enough precision in np
+       * near 1 to recover x.  For 0, isf gives inf,
+       * even if p is precise enough for the icdf to
+       * work.
+       */
+      CHECK_RELERR(x, isf_weibull(np, 1, 1));
+      CHECK_RELERR(x/2, isf_weibull(np, .5, 1));
+      CHECK_RELERR(x*2, isf_weibull(np, 2, 1));
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(3*sqrt(-log(p0/2)), sample_weibull(0, p0, 3, 2));
+    CHECK_RELERR(3*sqrt(-log1p(-p0/2)),
+        sample_weibull(1, p0, 3, 2));
+  }
+
+  if (!ok)
+    printf("fail Weibull cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the generalized Pareto
+ * distribution.
+ */
+static void
+test_genpareto(void *arg)
+{
+  (void) arg;
+
+  struct {
+    /* xi is the 'xi' parameter of the generalized Pareto distribution, and the
+     * rest are the same as in the above tests */
+    double xi, x, p, np;
+  } cases[] = {
+    { 0, 0, 0, 1 },
+    { 1e-300, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-300, .1, .09516258196404043, .9048374180359595 },
+    { 1e-300, 1, .6321205588285577, .36787944117144233 },
+    { 1e-300, 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 1e-200, 1e-16, 9.999999999999999e-17, .9999999999999999 },
+    { 1e-16, 1e-200, 9.999999999999998e-201, 1 },
+    { 1e-16, 1e-16, 1e-16, 1 },
+    { 1e-16, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-16, .1, .09516258196404043, .9048374180359595 },
+    { 1e-16, 1, .6321205588285577, .36787944117144233 },
+    { 1e-16, 10, .9999546000702375, 4.539992976248509e-5 },
+    { 1e-10, 1e-6, 9.999995000001667e-7, .9999990000005 },
+    { 1e-8, 1e-8, 9.999999950000001e-9, .9999999900000001 },
+    { 1, 1e-300, 1e-300, 1 },
+    { 1, 1e-16, 1e-16, .9999999999999999 },
+    { 1, .1, .09090909090909091, .9090909090909091 },
+    { 1, 1, .5, .5 },
+    { 1, 10, .9090909090909091, .0909090909090909 },
+    { 1, 100, .9900990099009901, .0099009900990099 },
+    { 1, 1000, .999000999000999, 9.990009990009992e-4 },
+    { 10, 1e-300, 1e-300, 1 },
+    { 10, 1e-16, 9.999999999999995e-17, .9999999999999999 },
+    { 10, .1, .06696700846319258, .9330329915368074 },
+    { 10, 1, .21320655780322778, .7867934421967723 },
+    { 10, 10, .3696701667040189, .6303298332959811 },
+    { 10, 100, .49886285755007337, .5011371424499267 },
+    { 10, 1000, .6018968102992647, .3981031897007353 },
+  };
+  double xi_array[] = { -1.5, -1, -1e-30, 0, 1e-30, 1, 1.5 };
+  size_t i, j;
+  double relerr_bound = 3e-15;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double xi = cases[i].xi;
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(p, cdf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(p, cdf_genpareto(x/2, 0, .5, xi));
+    CHECK_RELERR(np, sf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(np, sf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(np, sf_genpareto(x/2, 0, .5, xi));
+
+    if (p < .5) {
+      CHECK_RELERR(x, icdf_genpareto(p, 0, 1, xi));
+      CHECK_RELERR(x*2, icdf_genpareto(p, 0, 2, xi));
+      CHECK_RELERR(x/2, icdf_genpareto(p, 0, .5, xi));
+    }
+    if (np < .5) {
+      CHECK_RELERR(x, isf_genpareto(np, 0, 1, xi));
+      CHECK_RELERR(x*2, isf_genpareto(np, 0, 2, xi));
+      CHECK_RELERR(x/2, isf_genpareto(np, 0, .5, xi));
+    }
+  }
+
+  for (i = 0; i < arraycount(xi_array); i++) {
+    for (j = 0; j <= 100; j++) {
+      double p0 = (j == 0 ? 2*DBL_MIN : (double)j/100);
+
+      /* This is actually a check against 0, but we do <= so that the compiler
+         does not raise a -Wfloat-equal */
+      if (fabs(xi_array[i]) <= 0) {
+        /*
+         * When xi == 0, the generalized Pareto
+         * distribution reduces to an
+         * exponential distribution.
+         */
+        CHECK_RELERR(-log(p0/2),
+            sample_genpareto(0, p0, 0));
+        CHECK_RELERR(-log1p(-p0/2),
+            sample_genpareto(1, p0, 0));
+      } else {
+        CHECK_RELERR(expm1(-xi_array[i]*log(p0/2))/xi_array[i],
+            sample_genpareto(0, p0, xi_array[i]));
+        CHECK_RELERR((j == 0 ? DBL_MIN :
+                expm1(-xi_array[i]*log1p(-p0/2))/xi_array[i]),
+            sample_genpareto(1, p0, xi_array[i]));
+      }
+
+      CHECK_RELERR(isf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(0, p0, xi_array[i]));
+      CHECK_RELERR(icdf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(1, p0, xi_array[i]));
+    }
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the deterministic sampler for uniform distribution on [a, b].
+ *
+ * This currently only tests whether the outcome lies within [a, b].
+ */
+static void
+test_uniform_interval(void *arg)
+{
+  (void) arg;
+  struct {
+    /* Sample from a uniform distribution with parameters 'a' and 'b', using
+     * 't' as the sampling index. */
+    double t, a, b;
+  } cases[] = {
+    { 0, 0, 0 },
+    { 0, 0, 1 },
+    { 0, 1.0000000000000007, 3.999999999999995 },
+    { 0, 4000, 4000 },
+    { 0.42475836677491291, 4000, 4000 },
+    { 0, -DBL_MAX, DBL_MAX },
+    { 0.25, -DBL_MAX, DBL_MAX },
+    { 0.5, -DBL_MAX, DBL_MAX },
+  };
+  size_t i = 0;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double t = cases[i].t;
+    double a = cases[i].a;
+    double b = cases[i].b;
+
+    CHECK_LE(a, sample_uniform_interval(t, a, b));
+    CHECK_LE(sample_uniform_interval(t, a, b), b);
+
+    CHECK_LE(a, sample_uniform_interval(1 - t, a, b));
+    CHECK_LE(sample_uniform_interval(1 - t, a, b), b);
+
+    CHECK_LE(sample_uniform_interval(t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(t, -b, -a));
+
+    CHECK_LE(sample_uniform_interval(1 - t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(1 - t, -b, -a));
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/********************** Stochastic tests ****************************/
+
+/*
+ * Psi test, sometimes also called G-test.  The psi test statistic,
+ * suitably scaled, has chi^2 distribution, but the psi test tends to
+ * have better statistical power in practice to detect deviations than
+ * the chi^2 test does.  (The chi^2 test statistic is the first term of
+ * the Taylor expansion of the psi test statistic.)  The psi test is
+ * generic, for any CDF; particular distributions might have higher-
+ * power tests to distinguish them from predictable deviations or bugs.
+ *
+ * We choose the psi critical value so that a single psi test has
+ * probability below alpha = 1% of spuriously failing even if all the
+ * code is correct.  But the false positive rate for a suite of n tests
+ * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n.  For n = 10,
+ * this is about 10%, and for n = 100 it is well over 50%.
+ *
+ * Given that these tests will run with every CI job, we want to drive down the
+ * false positive rate. We can drive it down by running each test four times,
+ * and accepting it if it passes at least once; in that case, it is as if we
+ * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test,
+ * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14
+ * jobs, then the chance of a CI build failing is 1.39999903326e-06, which
+ * means that a CI build will break with probability 50% after about 495106
+ * builds.
+ *
+ * The critical value for a chi^2 distribution with 100 degrees of
+ * freedom and false positive rate alpha = 1% was taken from:
+ *
+ *  NIST/SEMATECH e-Handbook of Statistical Methods, Section
+ *  1.3.6.7.4 `Critical Values of the Chi-Square Distribution',
+ *  <http://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm>,
+ *  retrieved 2018-10-28.
+ */
+
+static const size_t NSAMPLES = 100000;
+/* Number of chances we give to the test to succeed. */
+static const unsigned NTRIALS = 4;
+/* Number of times we want the test to pass per NTRIALS. */
+static const unsigned NPASSES_MIN = 1;
+
+#define PSI_DF 100                          /* degrees of freedom */
+static const double PSI_CRITICAL = 135.807; /* critical value, alpha = .01 */
+
+/**
+ * Perform a psi test on an array of sample counts, C, adding up to N
+ * samples, and an array of log expected probabilities, logP,
+ * representing the null hypothesis for the distribution of samples
+ * counted.  Return false if the psi test rejects the null hypothesis,
+ * true if otherwise.
+ */
+static bool
+psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N)
+{
+  double psi = 0;
+  double c = 0;                 /* Kahan compensation */
+  double t, u;
+  size_t i;
+
+  for (i = 0; i < PSI_DF; i++) {
+    /*
+     * c*log(c/(n*p)) = (1/n) * f*log(f/p) where f = c/n is
+     * the frequency, and f*log(f/p) ---> 0 as f ---> 0, so
+     * this is a reasonable choice.  Further, any mass that
+     * _fails_ to turn up in this bin will inflate another
+     * bin instead, so we don't really lose anything by
+     * ignoring empty bins even if they have high
+     * probability.
+     */
+    if (C[i] == 0)
+      continue;
+    t = C[i]*(log((double)C[i]/N) - logP[i]) - c;
+    u = psi + t;
+    c = (u - psi) - t;
+    psi = u;
+  }
+  psi *= 2;
+
+  return psi <= PSI_CRITICAL;
+}
+
+static bool
+test_stochastic_geometric_impl(double p)
+{
+  const struct geometric geometric = {
+    .base = GEOMETRIC(geometric),
+    .p = p,
+  };
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  unsigned i;
+  size_t j;
+
+  /* Compute logP[i] = Geom(i + 1; p).  */
+  for (i = 0; i < PSI_DF - 1; i++)
+    logP[i] = logpmf_geometric(i + 1, p);
+
+  /* Compute logP[n-1] = log (1 - (P[0] + P[1] + ... + P[n-2])).  */
+  logP[PSI_DF - 1] = log1mexp(logsumexp(logP, PSI_DF - 1));
+
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+
+    for (j = 0; j < NSAMPLES; j++) {
+      double n_tmp = dist_sample(&geometric.base);
+
+      /* Must be an integer.  (XXX -Wfloat-equal)  */
+      tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp);
+
+      /* Must be a positive integer.  */
+      tor_assert(n_tmp >= 1);
+
+      /* Probability of getting a value in the billions is negligible.  */
+      tor_assert(n_tmp <= (double)UINT_MAX);
+
+      unsigned n = (unsigned) n_tmp;
+
+      if (n > PSI_DF)
+        n = PSI_DF;
+      C[n - 1]++;
+    }
+
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", "geometric"); */
+    return true;
+  } else {
+    printf("fail %s sampler\n", "geometric");
+    return false;
+  }
+}
+
+/**
+ * Divide the support of <b>dist</b> into histogram bins in <b>logP</b>. Start
+ * at the 1st percentile and ending at the 99th percentile. Pick the bin
+ * boundaries using linear interpolation so that they are uniformly spaced.
+ *
+ * In each bin logP[i] we insert the expected log-probability that a sampled
+ * value will fall into that bin. We will use this as the null hypothesis of
+ * the psi test.
+ *
+ * Set logP[i] = log(CDF(x_i) - CDF(x_{i-1})), where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_cdfs(const struct dist *dist, double lo, double hi, double *logP, size_t n)
+{
+#define CDF(x)  dist_cdf(dist, x)
+#define SF(x)   dist_sf(dist, x)
+  const double w = (hi - lo)/(n - 2);
+  double halfway = dist_icdf(dist, 0.5);
+  double x_0, x_1;
+  size_t i;
+  size_t n2 = ceil_to_size_t((halfway - lo)/w);
+
+  tor_assert(lo <= halfway);
+  tor_assert(halfway <= hi);
+  tor_assert(n2 <= n);
+
+  x_1 = lo;
+  logP[0] = log(CDF(x_1) - 0); /* 0 = CDF(-inf) */
+  for (i = 1; i < n2; i++) {
+    x_0 = x_1;
+    /* do the linear interpolation */
+    x_1 = (i <= n/2 ? lo + i*w : hi - (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[i] = log(CDF(x_1) - CDF(x_0));
+  }
+  x_0 = hi;
+  logP[n - 1] = log(SF(x_0) - 0); /* 0 = SF(+inf) = 1 - CDF(+inf) */
+
+  /* In this loop we are filling out the high part of the array. We are using
+   * SF because in these cases the CDF is near 1 where precision is lower. So
+   * instead we are using SF near 0 where the precision is higher. We have
+   * SF(t) = 1 - CDF(t).  */
+  for (i = 1; i < n - n2; i++) {
+    x_1 = x_0;
+    /* do the linear interpolation */
+    x_0 = (i <= n/2 ? hi - i*w : lo + (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[n - i - 1] = log(SF(x_0) - SF(x_1));
+  }
+#undef SF
+#undef CDF
+}
+
+/**
+ * Draw NSAMPLES samples from dist, counting the number of samples x in
+ * the ith bin C[i] if x_{i-1} <= x < x_i, where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_samples(const struct dist *dist, double lo, double hi, size_t *C, size_t n)
+{
+  const double w = (hi - lo)/(n - 2);
+  size_t i;
+
+  for (i = 0; i < NSAMPLES; i++) {
+    double x = dist_sample(dist);
+    size_t bin;
+
+    if (x < lo)
+      bin = 0;
+    else if (x < hi)
+      bin = 1 + floor_to_size_t((x - lo)/w);
+    else
+      bin = n - 1;
+    tor_assert(bin < n);
+    C[bin]++;
+  }
+}
+
+/**
+ * Carry out a Psi test on <b>dist</b>.
+ *
+ * Sample NSAMPLES from dist, putting them in bins from -inf to lo to
+ * hi to +inf, and apply up to two psi tests.  True if at least one psi
+ * test passes; false if not.  False positive rate should be bounded by
+ * 0.01^2 = 0.0001.
+ */
+static bool
+test_psi_dist_sample(const struct dist *dist)
+{
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  double lo = dist_icdf(dist, 1/(double)(PSI_DF + 2));
+  double hi = dist_isf(dist, 1/(double)(PSI_DF + 2));
+
+  /* Create the null hypothesis in logP */
+  bin_cdfs(dist, lo, hi, logP, PSI_DF);
+
+  /* Now run the test */
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+    bin_samples(dist, lo, hi, C, PSI_DF);
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  /* Did we fail or succeed? */
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", dist_name(dist));*/
+    return true;
+  } else {
+    printf("fail %s sampler\n", dist_name(dist));
+    return false;
+  }
+}
+
+static void
+write_stochastic_warning(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    printf("\n"
+         "NOTE: This is a stochastic test, and we expect it to fail from\n"
+         "time to time, with some low probability. If you see it fail more\n"
+         "than one trial in 100, though, please tell us.\n\n");
+  }
+}
+
+static void
+test_stochastic_uniform(void *arg)
+{
+  (void) arg;
+
+  const struct uniform uniform01 = {
+    .base = UNIFORM(uniform01),
+    .a = 0,
+    .b = 1,
+  };
+  const struct uniform uniform_pos = {
+    .base = UNIFORM(uniform_pos),
+    .a = 1.23,
+    .b = 4.56,
+  };
+  const struct uniform uniform_neg = {
+    .base = UNIFORM(uniform_neg),
+    .a = -10,
+    .b = -1,
+  };
+  const struct uniform uniform_cross = {
+    .base = UNIFORM(uniform_cross),
+    .a = -1.23,
+    .b = 4.56,
+  };
+  const struct uniform uniform_subnormal = {
+    .base = UNIFORM(uniform_subnormal),
+    .a = 4e-324,
+    .b = 4e-310,
+  };
+  const struct uniform uniform_subnormal_cross = {
+    .base = UNIFORM(uniform_subnormal_cross),
+    .a = -4e-324,
+    .b = 4e-310,
+  };
+  bool ok = true, tests_failed = true;
+
+  testing_enable_reproducible_rng();
+
+  ok &= test_psi_dist_sample(&uniform01.base);
+  ok &= test_psi_dist_sample(&uniform_pos.base);
+  ok &= test_psi_dist_sample(&uniform_neg.base);
+  ok &= test_psi_dist_sample(&uniform_cross.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal_cross.base);
+
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static bool
+test_stochastic_logistic_impl(double mu, double sigma)
+{
+  const struct logistic dist = {
+    .base = LOGISTIC(dist),
+    .mu = mu,
+    .sigma = sigma,
+  };
+
+  /* XXX Consider some fancier logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_log_logistic_impl(double alpha, double beta)
+{
+  const struct log_logistic dist = {
+    .base = LOG_LOGISTIC(dist),
+    .alpha = alpha,
+    .beta = beta,
+  };
+
+  /* XXX Consider some fancier log logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_weibull_impl(double lambda, double k)
+{
+  const struct weibull dist = {
+    .base = WEIBULL(dist),
+    .lambda = lambda,
+    .k = k,
+  };
+
+/*
+ * XXX Consider applying a Tiku-Singh test:
+ *
+ *    M.L. Tiku and M. Singh, `Testing the two-parameter
+ *    Weibull distribution', Communications in Statistics --
+ *    Theory and Methods A10(9), 1981, 907--918.
+ *https://www.tandfonline.com/doi/pdf/10.1080/03610928108828082?needAccess=true
+ */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_genpareto_impl(double mu, double sigma, double xi)
+{
+  const struct genpareto dist = {
+    .base = GENPARETO(dist),
+    .mu = mu,
+    .sigma = sigma,
+    .xi = xi,
+  };
+
+  /* XXX Consider some fancier GPD test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static void
+test_stochastic_genpareto(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_genpareto_impl(0, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, -1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(-1, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(1, 2, 0.25);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_geometric(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_geometric_impl(0.1);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.5);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.9);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(1);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_logistic(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_logistic_impl(0, 1);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(0, 1e-16);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(-10, 100);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_log_logistic(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_log_logistic_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(M_E, 1e-1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_weibull(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_weibull_impl(1, 0.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 2);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(10, 1);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+  UNMOCK(crypto_rand);
+}
+
+struct testcase_t prob_distr_tests[] = {
+  { "logit_logistics", test_logit_logistic, TT_FORK, NULL, NULL },
+  { "log_logistic", test_log_logistic, TT_FORK, NULL, NULL },
+  { "weibull", test_weibull, TT_FORK, NULL, NULL },
+  { "genpareto", test_genpareto, TT_FORK, NULL, NULL },
+  { "uniform_interval", test_uniform_interval, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
+
+struct testcase_t slow_stochastic_prob_distr_tests[] = {
+  { "stochastic_genpareto", test_stochastic_genpareto, TT_FORK, NULL, NULL },
+  { "stochastic_geometric", test_stochastic_geometric, TT_FORK, NULL, NULL },
+  { "stochastic_uniform", test_stochastic_uniform, TT_FORK, NULL, NULL },
+  { "stochastic_logistic", test_stochastic_logistic, TT_FORK, NULL, NULL },
+  { "stochastic_log_logistic", test_stochastic_log_logistic, TT_FORK, NULL,
+    NULL },
+  { "stochastic_weibull", test_stochastic_weibull, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process.c b/src/test/test_process.c
new file mode 100644
index 0000000000..7836312761
--- /dev/null
+++ b/src/test/test_process.c
@@ -0,0 +1,669 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process.c
+ * \brief Test cases for the Process API.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/process/env.h"
+
+#define PROCESS_PRIVATE
+#include "lib/process/process.h"
+#define PROCESS_UNIX_PRIVATE
+#include "lib/process/process_unix.h"
+#define PROCESS_WIN32_PRIVATE
+#include "lib/process/process_win32.h"
+
+static const char *stdout_read_buffer;
+static const char *stderr_read_buffer;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static int
+process_mocked_read_stdout(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stdout_read_buffer != NULL) {
+    buf_add_string(buffer, stdout_read_buffer);
+    stdout_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static int
+process_mocked_read_stderr(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stderr_read_buffer != NULL) {
+    buf_add_string(buffer, stderr_read_buffer);
+    stderr_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static void
+process_mocked_write_stdin(process_t *process, buf_t *buffer)
+{
+  const size_t size = buf_datalen(buffer);
+
+  if (size == 0)
+    return;
+
+  char *data = tor_malloc_zero(size + 1);
+  process_data_t *process_data = process_get_data(process);
+
+  buf_get_bytes(buffer, data, size);
+  smartlist_add(process_data->stdin_data, data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+static void
+test_default_values(void *arg)
+{
+  (void)arg;
+  process_t *process = process_new("/path/to/nothing");
+
+  /* We are not running by default. */
+  tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process));
+
+  /* We use the line protocol by default. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  /* We don't set any custom data by default. */
+  tt_ptr_op(NULL, OP_EQ, process_get_data(process));
+
+  /* Our command was given to the process_t's constructor in process_new(). */
+  tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process));
+
+  /* Make sure we are listed in the list of proccesses. */
+  tt_assert(smartlist_contains(process_get_all_processes(),
+                               process));
+
+  /* Default PID is 0. */
+  tt_u64_op(0, OP_EQ, process_get_pid(process));
+
+  /* Our arguments should be empty. */
+  tt_int_op(0, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+ done:
+  process_free(process);
+}
+
+static void
+test_environment(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("");
+  process_environment_t *env = NULL;
+
+  process_set_environment(process, "E", "F");
+  process_set_environment(process, "C", "D");
+  process_set_environment(process, "A", "B");
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "A=B\0C=D\0E=F\0", 12);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "A=B");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "C=D");
+  tt_str_op(env->unixoid_environment_block[2], OP_EQ,
+            "E=F");
+  tt_ptr_op(env->unixoid_environment_block[3], OP_EQ,
+            NULL);
+  process_environment_free(env);
+
+  /* Reset our environment. */
+  smartlist_t *new_env = smartlist_new();
+  smartlist_add(new_env, (char *)"FOO=bar");
+  smartlist_add(new_env, (char *)"HELLO=world");
+
+  process_reset_environment(process, new_env);
+  smartlist_free(new_env);
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "FOO=bar\0HELLO=world\0", 20);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "FOO=bar");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "HELLO=world");
+  tt_ptr_op(env->unixoid_environment_block[2], OP_EQ,
+            NULL);
+
+ done:
+  process_environment_free(env);
+  process_free(process);
+}
+
+static void
+test_stringified_types(void *arg)
+{
+  (void)arg;
+
+  /* process_protocol_t values. */
+  tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW));
+  tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE));
+
+  /* process_status_t values. */
+  tt_str_op("not running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_NOT_RUNNING));
+  tt_str_op("running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_RUNNING));
+  tt_str_op("error", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_ERROR));
+
+ done:
+  return;
+}
+
+static void
+test_line_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\r\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_multi(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Onion Onion Onion");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "A B C D");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Foo bar baz");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Onion Onion Onion");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_partial(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout this is a partial line ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr this is a partial line ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should NOT be ready. */
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+                          "Hello stdout this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+                          "Hello stderr this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_raw_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_protocol(process, PROCESS_PROTOCOL_RAW);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the raw protocol. */
+  tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout\n");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Hello, again, stdout\nThis contains multiple lines");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr\n");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Hello, again, stderr\nThis contains multiple lines");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_write_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  MOCK(process_write_stdin, process_mocked_write_stdin);
+
+  process_write(process, (uint8_t *)"Hello world\n", 12);
+  process_notify_event_stdin(process);
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data));
+
+  process_printf(process, "Hello %s !\n", "moon");
+  process_notify_event_stdin(process);
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data));
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_write_stdin);
+}
+
+static void
+test_exit_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Our default is 0. */
+  tt_u64_op(0, OP_EQ, process_data->exit_code);
+
+  /* Fake that we are a running process. */
+  process_set_status(process, PROCESS_STATUS_RUNNING);
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Fake an exit. */
+  process_notify_event_exit(process, 1337);
+
+  /* Check if our state changed and if our callback fired. */
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+  tt_u64_op(1337, OP_EQ, process_data->exit_code);
+
+ done:
+  process_set_data(process, process_data);
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_argv_simple(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("/bin/cat");
+  char **argv = NULL;
+
+  /* Setup some arguments. */
+  process_append_argument(process, "foo");
+  process_append_argument(process, "bar");
+  process_append_argument(process, "baz");
+
+  /* Check the number of elements. */
+  tt_int_op(3, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+  /* Let's try to convert it into a Unix style char **argv. */
+  argv = process_get_argv(process);
+
+  /* Check our values. */
+  tt_str_op(argv[0], OP_EQ, "/bin/cat");
+  tt_str_op(argv[1], OP_EQ, "foo");
+  tt_str_op(argv[2], OP_EQ, "bar");
+  tt_str_op(argv[3], OP_EQ, "baz");
+  tt_ptr_op(argv[4], OP_EQ, NULL);
+
+ done:
+  tor_free(argv);
+  process_free(process);
+}
+
+static void
+test_unix(void *arg)
+{
+  (void)arg;
+#ifndef _WIN32
+  process_t *process = process_new("");
+
+  /* On Unix all processes should have a Unix process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_unix_process(process));
+
+ done:
+  process_free(process);
+#endif /* !defined(_WIN32) */
+}
+
+static void
+test_win32(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  process_t *process = process_new("");
+  char *joined_argv = NULL;
+
+  /* On Win32 all processes should have a Win32 process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_win32_process(process));
+
+  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
+   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
+   * will try to only generate simple cases for the child process to parse;
+   * i.e. we never embed quoted strings in arguments. */
+
+  const char *argvs[][4] = {
+    {"a", "bb", "CCC", NULL}, // Normal
+    {NULL, NULL, NULL, NULL}, // Empty argument list
+    {"", NULL, NULL, NULL}, // Empty argument
+    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
+    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
+    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
+    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
+    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
+    { NULL } // Terminator
+  };
+
+  const char *cmdlines[] = {
+    "a bb CCC",
+    "",
+    "\"\"",
+    "\\\"a b\\\"b CCC\\\"",
+    "\"a\tbc\" \"dd  dd\" E",
+    "a\\\\\\b \"de fg\" H",
+    "a\\\\\\\"b \\c D\\",
+    "\"a\\\\b c\" d E",
+    NULL // Terminator
+  };
+
+  int i;
+
+  for (i=0; cmdlines[i]!=NULL; i++) {
+    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
+    joined_argv = tor_join_win_cmdline(argvs[i]);
+    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
+    tor_free(joined_argv);
+  }
+
+ done:
+  tor_free(joined_argv);
+  process_free(process);
+#endif /* defined(_WIN32) */
+}
+
+struct testcase_t process_tests[] = {
+  { "default_values", test_default_values, TT_FORK, NULL, NULL },
+  { "environment", test_environment, TT_FORK, NULL, NULL },
+  { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL },
+  { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL },
+  { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL },
+  { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL },
+  { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL },
+  { "write_simple", test_write_simple, TT_FORK, NULL, NULL },
+  { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL },
+  { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL },
+  { "unix", test_unix, TT_FORK, NULL, NULL },
+  { "win32", test_win32, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c
new file mode 100644
index 0000000000..7dc9abde31
--- /dev/null
+++ b/src/test/test_process_descs.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "test/test.h"
+
+static void
+test_process_descs_versions(void *arg)
+{
+  (void)arg;
+  struct {
+    const char *version;
+    bool should_reject;
+  } cases[] = {
+    // a very old version: reject.
+    { "Tor 0.1.2.3-alpha", true },
+    // a non-tor program: don't reject.
+    { "Wombat 0.1.2.3-alpha", false },
+    // a slightly old version: reject
+    { "Tor 0.2.9.4-alpha", true },
+    // a slightly old version: just new enough to support.
+    { "Tor 0.2.9.5-alpha", false },
+    // a newer 0.2.9 version: supported.
+    { "Tor 0.2.9.100", false },
+    // some unsupported versions: reject.
+    { "Tor 0.3.0.0-alpha-dev", true },
+    { "Tor 0.3.0.2-alpha", true },
+    { "Tor 0.3.0.5", true },
+    { "Tor 0.3.1.4", true },
+    { "Tor 0.3.2.4", true },
+    { "Tor 0.3.3.4", true },
+    { "Tor 0.3.4.1-alpha", true },
+    { "Tor 0.3.4.100", true },
+    { "Tor 0.3.5.1-alpha", true },
+    { "Tor 0.3.5.6-rc", true},
+    // new enough to be supported
+    { "Tor 0.3.5.7", false },
+    { "Tor 0.3.5.8", false },
+    { "Tor 0.4.0.1-alpha", false },
+    { "Tor 0.4.1.5", false },
+    // Very far in the future
+    { "Tor 100.100.1.5", false },
+  };
+  size_t n_cases = ARRAY_LENGTH(cases);
+
+  for (unsigned i = 0; i < n_cases; ++i) {
+    const char *msg = NULL;
+    bool rejected = dirserv_rejects_tor_version(cases[i].version, &msg);
+    tt_int_op(rejected, OP_EQ, cases[i].should_reject);
+    tt_int_op(msg == NULL, OP_EQ, rejected == false);
+  }
+
+ done:
+  ;
+}
+
+#define T(name,flags)                                   \
+  { #name, test_process_descs_##name, (flags), NULL, NULL }
+
+struct testcase_t process_descs_tests[] = {
+  T(versions,0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c
new file mode 100644
index 0000000000..f311e8b293
--- /dev/null
+++ b/src/test/test_process_slow.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process_slow.c
+ * \brief Slow test cases for the Process API.
+ */
+
+#define MAINLOOP_PRIVATE
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/process/process.h"
+#include "lib/process/waitpid.h"
+#include "test/test.h"
+
+#ifndef BUILDDIR
+#define BUILDDIR "."
+#endif
+
+#ifdef _WIN32
+#define TEST_PROCESS "test-process.exe"
+#else
+#define TEST_PROCESS BUILDDIR "/src/test/test-process"
+#endif /* defined(_WIN32) */
+
+/** Timer that ticks once a second and stop the event loop after 5 ticks. */
+static periodic_timer_t *main_loop_timeout_timer;
+
+/** How many times have our timer ticked? */
+static int timer_tick_count;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+  bool did_exit;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  process_status_t status;
+
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+  process_data->did_exit = true;
+
+  /* Check if our process is still running? */
+  status = process_get_status(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+#ifdef _WIN32
+static const char *
+get_win32_test_binary_path(void)
+{
+  static char buffer[MAX_PATH];
+
+  /* Get the absolute path of our binary: \path\to\test-slow.exe. */
+  GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer));
+
+  /* Find our process name. */
+  char *offset = strstr(buffer, "test-slow.exe");
+  tt_ptr_op(offset, OP_NE, NULL);
+
+  /* Change test-slow.exe to test-process.exe. */
+  memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS));
+
+  return buffer;
+ done:
+  return NULL;
+}
+#endif /* defined(_WIN32) */
+
+static void
+main_loop_timeout_cb(periodic_timer_t *timer, void *data)
+{
+  /* Sanity check. */
+  tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer);
+  tt_ptr_op(data, OP_NE, NULL);
+
+  /* Our process data. */
+  process_data_t *process_data = data;
+
+  /* Our process did exit. */
+  if (process_data->did_exit)
+    tor_shutdown_event_loop_and_exit(0);
+
+  /* Have we been called 10 times we exit the main loop. */
+  timer_tick_count++;
+
+  tt_int_op(timer_tick_count, OP_LT, 10);
+
+#ifndef _WIN32
+  /* Call waitpid callbacks. */
+  notify_pending_waitpid_callbacks();
+#endif
+
+  return;
+ done:
+  /* Exit with an error. */
+  tor_shutdown_event_loop_and_exit(-1);
+}
+
+static void
+run_main_loop(process_data_t *process_data)
+{
+  int ret;
+
+  /* Wake up after 1 seconds. */
+  static const struct timeval interval = {1, 0};
+
+  timer_tick_count = 0;
+  main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(),
+                                               &interval,
+                                               main_loop_timeout_cb,
+                                               process_data);
+
+  /* Run our main loop. */
+  ret = run_main_loop_until_done();
+
+  /* Clean up our main loop timeout timer. */
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  periodic_timer_free(main_loop_timeout_timer);
+}
+
+static void
+test_callbacks(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Set environment variable. */
+  process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!");
+
+  /* Add some arguments. */
+  process_append_argument(process, "This is the first one");
+  process_append_argument(process, "Second one");
+  process_append_argument(process, "Third: Foo bar baz");
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Write some lines to stdin. */
+  process_printf(process, "Hi process!\r\n");
+  process_printf(process, "Can you read more than one line?\n");
+  process_printf(process, "Can you read partial ...");
+  process_printf(process, " lines?\r\n");
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* We returned. Let's see what our event loop said. */
+  tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12);
+  tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3);
+  tt_assert(process_data->did_exit);
+  tt_u64_op(process_data->exit_code, OP_EQ, 0);
+
+  /* Check stdout output. */
+  char argv0_expected[256];
+  tor_snprintf(argv0_expected, sizeof(argv0_expected),
+               "argv[0] = '%s'", filename);
+
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            argv0_expected);
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "argv[1] = 'This is the first one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "argv[2] = 'Second one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "argv[3] = 'Third: Foo bar baz'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ,
+            "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ,
+            "Output on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ,
+            "Partial line on stdout ...end of partial line on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ,
+            "Read line from stdin: 'Hi process!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ,
+            "Read line from stdin: 'Can you read more than one line?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ,
+            "Read line from stdin: 'Can you read partial ... lines?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ,
+            "We are done for here, thank you!");
+
+  /* Check stderr output. */
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Output on stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Partial line on stderr ...end of partial line on stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_callbacks_terminate(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Zap our process. */
+  bool success;
+
+  success = process_terminate(process);
+  tt_assert(success);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Check if we did exit. */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_nonexistent_executable(void *arg)
+{
+  (void)arg;
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new("binary-does-not-exist");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_exec(process);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Ensure that the exit callback was actually called even though the binary
+   * did not exist.
+   */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+struct testcase_t slow_process_tests[] = {
+  { "callbacks", test_callbacks, 0, NULL, NULL },
+  { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL },
+  { "nonexistent_executable", test_nonexistent_executable, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c
index 08990c0b6a..f9339e8dd3 100644
--- a/src/test/test_proto_http.c
+++ b/src/test/test_proto_http.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 #include "test/log_test_helpers.h"
 
diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c
index af9cf7eee2..18669a7772 100644
--- a/src/test/test_proto_misc.c
+++ b/src/test/test_proto_misc.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_cell.h"
diff --git a/src/test/test_protover.c b/src/test/test_protover.c
index 63c508bd13..0254501165 100644
--- a/src/test/test_protover.c
+++ b/src/test/test_protover.c
@@ -22,7 +22,7 @@ test_protover_parse(void *arg)
   tt_skip();
  done:
   ;
-#else
+#else /* !defined(HAVE_RUST) */
   char *re_encoded = NULL;
 
   const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900";
@@ -89,7 +89,7 @@ test_protover_parse(void *arg)
     SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(elts);
   tor_free(re_encoded);
-#endif
+#endif /* defined(HAVE_RUST) */
 }
 
 static void
@@ -133,7 +133,7 @@ test_protover_parse_fail(void *arg)
                            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
   tt_ptr_op(elts, OP_EQ, NULL);
 
-#endif
+#endif /* defined(HAVE_RUST) */
  done:
   ;
 }
@@ -335,7 +335,7 @@ test_protover_all_supported(void *arg)
                  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
                  "aaaaaaaaaaaa=1-65536", &msg));
   tor_end_capture_bugs_();
-#endif
+#endif /* !defined(HAVE_RUST) */
 
  done:
   tor_end_capture_bugs_();
@@ -459,7 +459,7 @@ test_protover_supported_protocols(void *arg)
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
                                             PROTOVER_LINKAUTH_V1));
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   /* Latest LinkAuth is not exposed in the headers. */
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
diff --git a/src/test/test_pt.c b/src/test/test_pt.c
index 1f9786648a..8f3ce03c42 100644
--- a/src/test/test_pt.c
+++ b/src/test/test_pt.c
@@ -7,22 +7,25 @@
 #define PT_PRIVATE
 #define UTIL_PRIVATE
 #define STATEFILE_PRIVATE
-#define CONTROL_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define PROCESS_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/transports.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/statefile.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/process/process.h"
 
 #include "app/config/or_state_st.h"
 
+#include "test/log_test_helpers.h"
+
 static void
 reset_mp(managed_proxy_t *mp)
 {
@@ -288,41 +291,35 @@ test_pt_get_extrainfo_string(void *arg)
   tor_free(s);
 }
 
-#ifdef _WIN32
-#define STDIN_HANDLE HANDLE*
-#else
-#define STDIN_HANDLE int
-#endif
-
-static smartlist_t *
-tor_get_lines_from_handle_replacement(STDIN_HANDLE handle,
-                                      enum stream_status *stream_status_out)
+static int
+process_read_stdout_replacement(process_t *process, buf_t *buffer)
 {
+  (void)process;
   static int times_called = 0;
-  smartlist_t *retval_sl = smartlist_new();
-
-  (void) handle;
-  (void) stream_status_out;
 
   /* Generate some dummy CMETHOD lines the first 5 times. The 6th
      time, send 'CMETHODS DONE' to finish configuring the proxy. */
-  if (times_called++ != 5) {
-    smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d",
+  times_called++;
+
+  if (times_called <= 5) {
+    buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n",
                            times_called, times_called);
-  } else {
-    smartlist_add_strdup(retval_sl, "SMETHODS DONE");
+  } else if (times_called <= 6) {
+    buf_add_string(buffer, "SMETHODS DONE\n");
+  } else if (times_called <= 7) {
+    buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something "
+                           "bad happened. What do we do!?\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n");
+  } else if (times_called <= 8) {
+    buf_add_string(buffer, "STATUS TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\n");
   }
 
-  return retval_sl;
-}
-
-/* NOP mock */
-static void
-tor_process_handle_destroy_replacement(process_handle_t *process_handle,
-                                       int also_terminate_process)
-{
-  (void) process_handle;
-  (void) also_terminate_process;
+  return (int)buf_datalen(buffer);
 }
 
 static or_state_t *dummy_state = NULL;
@@ -355,12 +352,9 @@ test_pt_configure_proxy(void *arg)
   managed_proxy_t *mp = NULL;
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
-  MOCK(tor_get_lines_from_handle,
-       tor_get_lines_from_handle_replacement);
-  MOCK(tor_process_handle_destroy,
-       tor_process_handle_destroy_replacement);
+  MOCK(process_read_stdout, process_read_stdout_replacement);
   MOCK(get_or_state,
        get_or_state_replacement);
   MOCK(queue_control_event_string,
@@ -372,24 +366,34 @@ test_pt_configure_proxy(void *arg)
   mp->conf_state = PT_PROTO_ACCEPTING_METHODS;
   mp->transports = smartlist_new();
   mp->transports_to_launch = smartlist_new();
-  mp->process_handle = tor_malloc_zero(sizeof(process_handle_t));
   mp->argv = tor_malloc_zero(sizeof(char*)*2);
   mp->argv[0] = tor_strdup("<testcase>");
   mp->is_server = 1;
 
+  /* Configure the process. */
+  mp->process = process_new("");
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_data(mp->process, mp);
+
   /* Test the return value of configure_proxy() by calling it some
      times while it is uninitialized and then finally finalizing its
      configuration. */
   for (i = 0 ; i < 5 ; i++) {
+    /* force a read from our mocked stdout reader. */
+    process_notify_event_stdout(mp->process);
+    /* try to configure our proxy. */
     retval = configure_proxy(mp);
     /* retval should be zero because proxy hasn't finished configuring yet */
     tt_int_op(retval, OP_EQ, 0);
     /* check the number of registered transports */
-    tt_assert(smartlist_len(mp->transports) == i+1);
+    tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1);
     /* check that the mp is still waiting for transports */
     tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS);
   }
 
+  /* Get the SMETHOD DONE written to the process. */
+  process_notify_event_stdout(mp->process);
+
   /* this last configure_proxy() should finalize the proxy configuration. */
   retval = configure_proxy(mp);
   /* retval should be 1 since the proxy finished configuring */
@@ -412,6 +416,49 @@ test_pt_configure_proxy(void *arg)
   tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ,
             "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n");
 
+  /* Get the log message out. */
+  setup_full_capture_of_logs(LOG_ERR);
+  process_notify_event_stdout(mp->process);
+  expect_single_log_msg_containing("Oh noes, something bad happened");
+  teardown_capture_of_logs();
+
+  tt_int_op(controlevent_n, OP_EQ, 10);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10);
+  tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=error "
+            "MESSAGE=\"Oh noes, "
+            "something bad happened. What do we do!?\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=warning "
+            "MESSAGE=\"warning msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=notice "
+            "MESSAGE=\"notice msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=info "
+            "MESSAGE=\"info msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=debug "
+            "MESSAGE=\"debug msg\"\r\n");
+
+  /* Get the STATUS messages out. */
+  process_notify_event_stdout(mp->process);
+
+  tt_int_op(controlevent_n, OP_EQ, 13);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13);
+
+  tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\r\n");
+
   { /* check that the transport info were saved properly in the tor state */
     config_line_t *transport_in_state = NULL;
     smartlist_t *transport_info_sl = smartlist_new();
@@ -434,9 +481,9 @@ test_pt_configure_proxy(void *arg)
   }
 
  done:
+  teardown_capture_of_logs();
   or_state_free(dummy_state);
-  UNMOCK(tor_get_lines_from_handle);
-  UNMOCK(tor_process_handle_destroy);
+  UNMOCK(process_read_stdout);
   UNMOCK(get_or_state);
   UNMOCK(queue_control_event_string);
   if (controlevent_msgs) {
@@ -449,7 +496,7 @@ test_pt_configure_proxy(void *arg)
     smartlist_free(mp->transports);
   }
   smartlist_free(mp->transports_to_launch);
-  tor_free(mp->process_handle);
+  process_free(mp->process);
   tor_free(mp->argv[0]);
   tor_free(mp->argv);
   tor_free(mp);
diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c
new file mode 100644
index 0000000000..76bdbf1891
--- /dev/null
+++ b/src/test/test_ptr_slow.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/ptr_helpers.h"
+
+#include <stdint.h>
+#include <limits.h>
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_int_voidptr_roundtrip(int a)
+{
+  intptr_t ap = (intptr_t)a;
+  void *b = cast_intptr_to_voidstar(ap);
+  intptr_t c = cast_voidstar_to_intptr(b);
+  void *d = cast_intptr_to_voidstar(c);
+
+  tt_assert(ap == c);
+  tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_int_voidstar_interop(void *arg)
+{
+  int a;
+  (void)arg;
+
+  for (a = -1024; a <= 1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MIN; a <= INT_MIN+1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MAX-1024; a < INT_MAX; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_int_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_uint_voidptr_roundtrip(unsigned int a)
+{
+ uintptr_t ap = (uintptr_t)a;
+ void *b = cast_uintptr_to_voidstar(ap);
+ uintptr_t c = cast_voidstar_to_uintptr(b);
+ void *d = cast_uintptr_to_voidstar(c);
+
+ tt_assert(ap == c);
+ tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_uint_voidstar_interop(void *arg)
+{
+  unsigned int a;
+  (void)arg;
+
+  for (a = 0; a <= 1024; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  for (a = UINT_MAX-1024; a < UINT_MAX; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_uint_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+struct testcase_t slow_ptr_tests[] = {
+  { .name = "int_voidstar_interop",
+    .fn = test_int_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  { .name = "uint_voidstar_interop",
+    .fn = test_uint_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c
new file mode 100644
index 0000000000..021323fbf1
--- /dev/null
+++ b/src/test/test_pubsub_build.c
@@ -0,0 +1,578 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+#define PUBSUB_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "test/log_test_helpers.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_int_fmt(msg_aux_data_t aux)
+{
+  int val = (int) aux.u64;
+  char *r=NULL;
+  tor_asprintf(&r, "%d", val);
+  return r;
+}
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+
+static dispatch_typefns_t intfns = {
+  .fmt_fn = ex_int_fmt
+};
+
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+DECLARE_MESSAGE_INT(bunch_of_coconuts, int, int);
+DECLARE_PUBLISH(bunch_of_coconuts);
+DECLARE_SUBSCRIBE(bunch_of_coconuts, coconut_recipient_cb);
+
+DECLARE_MESSAGE(yes_we_have_no, string, char *);
+DECLARE_PUBLISH(yes_we_have_no);
+DECLARE_SUBSCRIBE(yes_we_have_no, absent_item_cb);
+
+static void
+coconut_recipient_cb(const msg_t *m, int n_coconuts)
+{
+  (void)m;
+  (void)n_coconuts;
+}
+
+static void
+absent_item_cb(const msg_t *m, const char *fruitname)
+{
+  (void)m;
+  (void)fruitname;
+}
+
+#define FLAG_SKIP 99999
+
+static void
+seed_dispatch_builder(pubsub_builder_t *b,
+                      unsigned fl1, unsigned fl2, unsigned fl3, unsigned fl4)
+{
+  pubsub_connector_t *c = NULL;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    DISPATCH_REGISTER_TYPE(c, int, &intfns);
+    if (fl1 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, fl1);
+    if (fl2 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, yes_we_have_no, fl2);
+    pubsub_connector_free(c);
+  }
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2"));
+    DISPATCH_REGISTER_TYPE(c, string, &stringfns);
+    if (fl3 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, yes_we_have_no, fl3);
+    if (fl4 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, bunch_of_coconuts, fl4);
+    pubsub_connector_free(c);
+  }
+}
+
+static void
+seed_pubsub_builder_basic(pubsub_builder_t *b)
+{
+  seed_dispatch_builder(b, 0, 0, 0, 0);
+}
+
+/* Regular builder with valid types and messages.
+ */
+static void
+test_pubsub_build_types_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  pubsub_items_t *items = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  dispatcher = pubsub_builder_finalize(b, &items);
+  b = NULL;
+  tt_assert(dispatcher);
+  tt_assert(items);
+  tt_int_op(smartlist_len(items->items), OP_EQ, 4);
+
+  // Make sure that the bindings got build correctly.
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, dispatcher);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+  tt_int_op(dispatcher->n_types, OP_GE, 2);
+  tt_assert(dispatcher->typefns);
+
+  tt_assert(dispatcher->typefns[get_msg_type_id("int")].fmt_fn == ex_int_fmt);
+  tt_assert(dispatcher->typefns[get_msg_type_id("string")].fmt_fn ==
+            ex_str_fmt);
+
+  // Now clear the bindings, like we would do before freeing the
+  // the dispatcher.
+  pubsub_items_clear_bindings(items);
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, NULL);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  pubsub_items_free(items);
+}
+
+/* We fail if the same type is defined in two places with different functions.
+ */
+static void
+test_pubsub_build_types_decls_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    // Extra declaration of int: we don't allow this.
+    DISPATCH_REGISTER_TYPE(c, int, &stringfns);
+    pubsub_connector_free(c);
+  }
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+  // expect_log_msg_containing("(int) declared twice"); // XXXX
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* If a message ID exists but nobody is publishing or subscribing to it,
+ * that's okay. */
+static void
+test_pubsub_build_unused_message(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  // This message isn't actually generated by anyone, but that will be fine:
+  // we just log it at info.
+  get_message_id("unused");
+  setup_capture_of_logs(LOG_INFO);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+  expect_log_msg_containing(
+     "Nobody is publishing or subscribing to message");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Publishing or subscribing to a message with no subscribers / publishers
+ * should fail and warn. */
+static void
+test_pubsub_build_missing_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, FLAG_SKIP, FLAG_SKIP);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing(
+       "Message \"bunch_of_coconuts\" has publishers, but no subscribers.");
+  expect_log_msg_containing(
+       "Message \"yes_we_have_no\" has subscribers, but no publishers.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Make sure that a stub publisher or subscriber prevents an error from
+ * happening even if there are no other publishers/subscribers for a message
+ */
+static void
+test_pubsub_build_stub_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, DISP_FLAG_STUB, DISP_FLAG_STUB);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // no subscribers
+  tt_ptr_op(NULL, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* Only one channel per msg id. */
+static void
+test_pubsub_build_channels_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("int"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent channels.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Only one type per msg id. */
+static void
+test_pubsub_build_types_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent message types.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* The same module can't publish and subscribe the same message */
+static void
+test_pubsub_build_pubsub_same(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    // already publishing this.
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is published "
+                            "and subscribed by the same subsystem \"sys1\".");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* More than one subsystem may publish or subscribe, and that's okay. */
+static void
+test_pubsub_build_pubsub_multi(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_add_pub_(c, &btmp, get_channel_id("main"),
+                    get_message_id("yes_we_have_no"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 2 subscribers.
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(2, OP_EQ, ent->n_enabled);
+  tt_int_op(2, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+static void
+some_other_coconut_hook(const msg_t *m)
+{
+  (void)m;
+}
+
+/* Subscribe hooks should be build correctly when there are a bunch of
+ * them. */
+static void
+test_pubsub_build_sub_many(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  char *sysname = NULL;
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  int i;
+  for (i = 1; i < 100; ++i) {
+    tor_asprintf(&sysname, "system%d",i);
+    c = pubsub_connector_for_subsystem(b, get_subsys_id(sysname));
+    if (i % 7) {
+      DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    } else {
+      pubsub_add_sub_(c, some_other_coconut_hook,
+                      get_channel_id("main"),
+                      get_message_id("bunch_of_coconuts"),
+                      get_msg_type_id("int"),
+                      0 /* flags */,
+                      "somewhere.c", 22);
+    }
+    pubsub_connector_free(c);
+    tor_free(sysname);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(100, OP_EQ, ent->n_enabled);
+  tt_int_op(100, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[76].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[77].fn, OP_EQ, some_other_coconut_hook);
+  tt_ptr_op(ent->rcv[78].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  tor_free(sysname);
+}
+
+/* It's fine to declare the excl flag. */
+static void
+test_pubsub_build_excl_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  // Try one excl/excl pair and one excl/non pair.
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, 0,
+                        DISP_FLAG_EXCL, DISP_FLAG_EXCL);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]->n_enabled);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* but if you declare the excl flag, you need to mean it. */
+static void
+test_pubsub_build_excl_bad(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, DISP_FLAG_EXCL,
+                        0, 0);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, 0);
+    DISPATCH_ADD_SUB_(c, main, yes_we_have_no, 0);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("has multiple publishers, but at least one is "
+                            "marked as exclusive.");
+  expect_log_msg_containing("has multiple subscribers, but at least one is "
+                            "marked as exclusive.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+#define T(name, flags)                                          \
+  { #name, test_pubsub_build_ ## name , (flags), NULL, NULL }
+
+struct testcase_t pubsub_build_tests[] = {
+  T(types_ok, TT_FORK),
+  T(types_decls_conflict, TT_FORK),
+  T(unused_message, TT_FORK),
+  T(missing_pubsub, TT_FORK),
+  T(stub_pubsub, TT_FORK),
+  T(channels_conflict, TT_FORK),
+  T(types_conflict, TT_FORK),
+  T(pubsub_same, TT_FORK),
+  T(pubsub_multi, TT_FORK),
+  T(sub_many, TT_FORK),
+  T(excl_ok, TT_FORK),
+  T(excl_bad, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_msg.c b/src/test/test_pubsub_msg.c
new file mode 100644
index 0000000000..73c7c9f540
--- /dev/null
+++ b/src/test/test_pubsub_msg.c
@@ -0,0 +1,305 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+// We're using the lowest-level publish/subscribe logic here, to avoid the
+// pubsub_macros.h macros and just test the dispatch core.  We'll use a string
+// type for everything.
+
+#define DECLARE_MESSAGE(suffix)                         \
+  static pub_binding_t pub_binding_##suffix;            \
+  static int msg_received_##suffix = 0;                 \
+  static void recv_msg_##suffix(const msg_t *m) {       \
+    (void)m;                                            \
+    ++msg_received_##suffix;                            \
+  }                                                     \
+  EAT_SEMICOLON
+
+#define ADD_PUBLISH(binding_suffix, subsys, channel, msg, flags)        \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_pub_(con, &pub_binding_##binding_suffix,                 \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define ADD_SUBSCRIBE(hook_suffix, subsys, channel, msg, flags)         \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_sub_(con, recv_msg_##hook_suffix,                        \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define SEND(binding_suffix, val)                          \
+  STMT_BEGIN {                                             \
+    msg_aux_data_t data_;                                  \
+    data_.ptr = tor_strdup(val);                           \
+    pubsub_pub_(&pub_binding_##binding_suffix, data_);     \
+  } STMT_END
+
+DECLARE_MESSAGE(msg1);
+DECLARE_MESSAGE(msg2);
+DECLARE_MESSAGE(msg3);
+DECLARE_MESSAGE(msg4);
+DECLARE_MESSAGE(msg5);
+
+static smartlist_t *strings_received = NULL;
+static void
+recv_msg_copy_string(const msg_t *m)
+{
+  const char *s = m->aux_data__.ptr;
+  smartlist_add(strings_received, tor_strdup(s));
+}
+
+static void *
+setup_dispatcher(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  pubsub_connector_t *con;
+
+  {
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id("types"));
+    pubsub_connector_register_type_(con,
+                                    get_msg_type_id("string"),
+                                    &stringfns,
+                                    "nowhere.c", 99);
+    pubsub_connector_free(con);
+  }
+  // message1 has one publisher and one subscriber.
+  ADD_PUBLISH(msg1, sys1, main, message1, 0);
+  ADD_SUBSCRIBE(msg1, sys2, main, message1, 0);
+
+  // message2 has a publisher and a stub subscriber.
+  ADD_PUBLISH(msg2, sys1, main, message2, 0);
+  ADD_SUBSCRIBE(msg2, sys2, main, message2, DISP_FLAG_STUB);
+
+  // message3 has a publisher and three subscribers.
+  ADD_PUBLISH(msg3, sys1, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys2, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys3, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys4, main, message3, 0);
+
+  // message4 has one publisher and two subscribers, but it's on another
+  // channel.
+  ADD_PUBLISH(msg4, sys2, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys1, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys3, other, message4, 0);
+
+  // message5 has a huge number of recipients.
+  ADD_PUBLISH(msg5, sys3, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys4, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys5, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys6, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys7, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys8, main, message5, 0);
+  for (int i = 0; i < 1000-5; ++i) {
+    char *sys;
+    tor_asprintf(&sys, "xsys-%d", i);
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id(sys));
+    pubsub_add_sub_(con, recv_msg_copy_string,
+                    get_channel_id("main"),
+                    get_message_id("message5"),
+                    get_msg_type_id("string"), 0, "here", 100);
+    pubsub_connector_free(con);
+    tor_free(sys);
+  }
+
+  return pubsub_builder_finalize(builder, NULL);
+}
+
+static int
+cleanup_dispatcher(const struct testcase_t *testcase, void *dispatcher_)
+{
+  (void)testcase;
+  dispatch_t *dispatcher = dispatcher_;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+static const struct testcase_setup_t dispatcher_setup = {
+  setup_dispatcher, cleanup_dispatcher
+};
+
+static void
+test_pubsub_msg_minimal(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  SEND(msg1, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(1, OP_EQ, msg_received_msg1); // we got the message!
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_send_to_stub(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg2);
+  SEND(msg2, "hello silence");
+  tt_int_op(0, OP_EQ, msg_received_msg2); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(0, OP_EQ, msg_received_msg2); // doesn't arrive -- stub hook.
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_cancel_msgs(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  for (int i = 0; i < 100; ++i) {
+    SEND(msg1, "hello world");
+  }
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 10));
+  tt_int_op(10, OP_EQ, msg_received_msg1); // we got the message 10 times.
+
+  // At this point, the dispatcher will be freed with queued, undelivered
+  // messages.
+ done:
+  ;
+}
+
+struct alertfn_target {
+  dispatch_t *d;
+  channel_id_t ch;
+  int count;
+};
+static void
+alertfn_generic(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  struct alertfn_target *t = arg;
+  tt_ptr_op(d, OP_EQ, t->d);
+  tt_int_op(ch, OP_EQ, t->ch);
+  ++t->count;
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_alertfns(void *arg)
+{
+  dispatch_t *d = arg;
+  struct alertfn_target ch1_a = { d, get_channel_id("main"), 0 };
+  struct alertfn_target ch2_a = { d, get_channel_id("other"), 0 };
+
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("main"),
+                                  alertfn_generic, &ch1_a));
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("other"),
+                                  alertfn_generic, &ch2_a));
+
+  SEND(msg3, "hello");
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg3, "world");
+  tt_int_op(ch1_a.count, OP_EQ, 1); // only the first message sends an alert
+  tt_int_op(ch2_a.count, OP_EQ, 0); // no alert for 'other'
+
+  SEND(msg4, "worse things happen in C");
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+  // flush the first (main) channel...
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(6, OP_EQ, msg_received_msg3); // 3 subscribers, 2 instances.
+
+  // now that the main channel is flushed, sending another message on it
+  // starts another alert.
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg1, "plover");
+  tt_int_op(ch1_a.count, OP_EQ, 2);
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+/* try more than N_FAST_FNS hooks on msg5 */
+static void
+test_pubsub_msg_many_hooks(void *arg)
+{
+  dispatch_t *d = arg;
+  strings_received = smartlist_new();
+
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  SEND(msg5, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  tt_int_op(0, OP_EQ, smartlist_len(strings_received));
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 100000));
+  tt_int_op(5, OP_EQ, msg_received_msg5);
+  tt_int_op(995, OP_EQ, smartlist_len(strings_received));
+
+ done:
+  SMARTLIST_FOREACH(strings_received, char *, s, tor_free(s));
+  smartlist_free(strings_received);
+}
+
+#define T(name)                                                 \
+  { #name, test_pubsub_msg_ ## name , TT_FORK,                \
+      &dispatcher_setup, NULL }
+
+struct testcase_t pubsub_msg_tests[] = {
+  T(minimal),
+  T(send_to_stub),
+  T(cancel_msgs),
+  T(alertfns),
+  T(many_hooks),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py
index 30a587858f..c9b9200b2d 100644
--- a/src/test/test_rebind.py
+++ b/src/test/test_rebind.py
@@ -22,9 +22,10 @@ def skip(msg):
 
 def try_connecting_to_socksport():
     socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    if socks_socket.connect_ex(('127.0.0.1', socks_port)):
+    e = socks_socket.connect_ex(('127.0.0.1', socks_port))
+    if e:
         tor_process.terminate()
-        fail('Cannot connect to SOCKSPort')
+        fail('Cannot connect to SOCKSPort: error ' + os.strerror(e))
     socks_socket.close()
 
 def wait_for_log(s):
diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh
index ea2012957e..d6d9d86668 100755
--- a/src/test/test_rebind.sh
+++ b/src/test/test_rebind.sh
@@ -7,18 +7,21 @@ if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
   if test "$APPVEYOR" = 'True'; then
-    echo "This test is disabled on Windows CI, as it requires firewall examptions. Skipping." >&2
+    echo "This test is disabled on Windows CI, as it requires firewall exemptions. Skipping." >&2
     exit 77
   fi
 fi
 
-exitcode=0
-
 tmpdir=
-clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; }
+clean () {
+  if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then
+    rm -rf "$tmpdir"
+  fi
+}
+
 trap clean EXIT HUP INT TERM
 
-tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`"
+tmpdir="$(mktemp -d -t tor_rebind_test.XXXXXX)"
 if [ -z "$tmpdir" ]; then
   echo >&2 mktemp failed
   exit 2
diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c
index 0623583511..c65279fb25 100644
--- a/src/test/test_relaycell.c
+++ b/src/test/test_relaycell.c
@@ -17,6 +17,7 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/connection_edge.h"
+#include "core/or/sendme.h"
 #include "core/or/relay.h"
 #include "test/test.h"
 #include "test/log_test_helpers.h"
@@ -812,7 +813,11 @@ test_circbw_relay(void *arg)
   ASSERT_UNCOUNTED_BW();
 
   /* Sendme on circuit with non-full window: counted */
-  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "");
+  /* Recording a cell, the window is updated after decryption so off by one in
+   * order to record and then we process it with the proper window. */
+  circ->cpath->package_window = 901;
+  sendme_record_cell_digest_on_circ(TO_CIRCUIT(circ), circ->cpath);
   circ->cpath->package_window = 900;
   connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
                                      circ->cpath);
diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c
index fe6889e521..4bbf07c3ec 100644
--- a/src/test/test_relaycrypt.c
+++ b/src/test/test_relaycrypt.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define CRYPT_PATH_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/circuitbuild.h"
 #define CIRCUITLIST_PRIVATE
@@ -10,7 +12,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
-
+#include "core/or/crypt_path.h"
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -49,10 +51,10 @@ testing_circuitset_setup(const struct testcase_t *testcase)
   cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
   for (i=0; i<3; ++i) {
     crypt_path_t *hop = tor_malloc_zero(sizeof(*hop));
-    relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
-                      0, 0);
+    relay_crypto_init(&hop->pvt_crypto, KEY_MATERIAL[i],
+                      sizeof(KEY_MATERIAL[i]), 0, 0);
     hop->state = CPATH_STATE_OPEN;
-    onion_append_to_cpath(&cs->origin_circ->cpath, hop);
+    cpath_extend_linked_list(&cs->origin_circ->cpath, hop);
     tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev);
   }
 
diff --git a/src/test/test_rng.c b/src/test/test_rng.c
new file mode 100644
index 0000000000..dcf08fff1d
--- /dev/null
+++ b/src/test/test_rng.c
@@ -0,0 +1,59 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Example usage:
+ *
+ * ./src/test/test-rng --emit | dieharder -g 200 -a
+ *
+ * Remember, dieharder can tell you that your RNG is completely broken, but if
+ * your RNG is not _completely_ broken, dieharder cannot tell you whether your
+ * RNG is actually secure.
+ */
+
+#include "orconfig.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+int
+main(int argc, char **argv)
+{
+  uint8_t buf[0x123];
+
+  if (argc != 2 || strcmp(argv[1], "--emit")) {
+    fprintf(stderr, "If you want me to fill stdout with a bunch of random "
+            "bytes, you need to say --emit.\n");
+    return 1;
+  }
+
+  if (crypto_seed_rng() < 0) {
+    fprintf(stderr, "Can't seed RNG.\n");
+    return 1;
+  }
+
+#if 0
+  while (1) {
+    crypto_rand(buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+#endif /* 0 */
+
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  while (1) {
+    crypto_fast_rng_getbytes(rng, buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+}
diff --git a/src/test/test_router.c b/src/test/test_router.c
index 601881a124..5477ab51e9 100644
--- a/src/test/test_router.c
+++ b/src/test/test_router.c
@@ -7,16 +7,25 @@
  * \brief Unittests for code in router.c
  **/
 
+#define CONFIG_PRIVATE
+#define ROUTER_PRIVATE
+
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerstatus_st.h"
 #include "feature/relay/router.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/encoding/confline.h"
 
 /* Test suite stuff */
 #include "test/test.h"
@@ -91,6 +100,9 @@ test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
   router = (routerinfo_t*)router_get_my_routerinfo();
   tt_ptr_op(router, !=, NULL);
 
+  /* The real router_get_my_routerinfo() looks up onion_curve25519_pkey using
+   * get_current_curve25519_keypair(), but we don't initialise static data in
+   * this test. */
   router->onion_curve25519_pkey = &ntor_keypair.pubkey;
 
   /* Generate our server descriptor and ensure that the substring
@@ -231,11 +243,254 @@ test_router_check_descriptor_bandwidth_changed(void *arg)
   UNMOCK(we_are_hibernating);
 }
 
+static networkstatus_t *mock_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void)now;
+  return mock_ns;
+}
+
+static routerstatus_t *mock_rs = NULL;
+static const routerstatus_t *
+mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+{
+  (void)ns;
+  (void)digest;
+  return mock_rs;
+}
+
+static void
+test_router_mark_if_too_old(void *arg)
+{
+  (void)arg;
+  time_t now = approx_time();
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+  MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry);
+
+  routerstatus_t rs;
+  networkstatus_t ns;
+  memset(&rs, 0, sizeof(rs));
+  memset(&ns, 0, sizeof(ns));
+  mock_ns = &ns;
+  mock_ns->valid_after = now-3600;
+  mock_rs = &rs;
+  mock_rs->published_on = now - 10;
+
+  // no reason to mark this time.
+  desc_clean_since = now-10;
+  desc_dirty_reason = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+
+  // Doesn't appear in consensus?  Still don't mark it.
+  mock_ns = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+  mock_ns = &ns;
+
+  // No new descriptor in a long time?  Mark it.
+  desc_clean_since = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor");
+
+  // Version in consensus published a long time ago?  We won't mark it
+  // if it's been clean for only a short time.
+  desc_clean_since = now - 10;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now - 10);
+
+  // ... but if it's been clean a while, we mark.
+  desc_clean_since = now - 2 * 3600;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "version listed in consensus is quite old");
+
+  // same deal if we're marked stale.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 10;
+  mock_rs->is_staledesc = 1;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "listed as stale in consensus");
+
+  // same deal if we're absent from the consensus.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "not listed in consensus");
+
+ done:
+  UNMOCK(networkstatus_get_live_consensus);
+  UNMOCK(networkstatus_vote_find_entry);
+}
+
+static node_t fake_node;
+static const node_t *
+mock_node_get_by_nickname(const char *name, unsigned flags)
+{
+  (void)flags;
+  if (!strcasecmp(name, "crumpet"))
+    return &fake_node;
+  else
+    return NULL;
+}
+
+static void
+test_router_get_my_family(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  smartlist_t *sl = NULL;
+  char *join = NULL;
+  // Overwrite the result of router_get_my_identity_digest().  This
+  // happens to be okay, but only for testing.
+  set_server_identity_key_digest_testing(
+                                   (const uint8_t*)"holeinthebottomofthe");
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // No family listed -- so there's no list.
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+#define CLEAR() do {                                    \
+    if (sl) {                                           \
+      SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));  \
+      smartlist_free(sl);                               \
+    }                                                   \
+    tor_free(join);                                     \
+    mock_clean_saved_logs();                            \
+  } while (0)
+
+  // Add a single nice friendly hex member.  This should be enough
+  // to have our own ID added.
+  tt_ptr_op(options->MyFamily, OP_EQ, NULL);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 2);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Add a hex member with a ~.  The ~ part should get removed.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$0123456789abcdef0123456789abcdef01234567~Muffin");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Nickname lookup will fail, so a nickname will appear verbatim.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "BAGEL");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  expect_single_log_msg_containing(
+           "There is a router named \"BAGEL\" in my declared family, but "
+           "I have no descriptor for it.");
+  CLEAR();
+
+  // A bogus digest should fail entirely.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$painauchocolat");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.
+  expect_single_log_msg_containing(
+           "There is a router named \"$painauchocolat\" in my declared "
+           "family, but that isn't a legal digest or nickname. Skipping it.");
+  CLEAR();
+
+  // Let's introduce a node we can look up by nickname
+  memset(&fake_node, 0, sizeof(fake_node));
+  memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN);
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "CRUmpeT");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 5);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.  Some with "$painauchocolat".
+  expect_single_log_msg_containing(
+           "There is a router named \"CRUmpeT\" in my declared "
+           "family, but it wasn't listed by digest. Please consider saying "
+           "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's "
+           "what you meant.");
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+  // Try a singleton list containing only us: It should give us NULL.
+  config_free_lines(options->MyFamily);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$686F6C65696E746865626F74746F6D6F66746865");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+ done:
+  or_options_free(options);
+  teardown_capture_of_logs();
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+#undef CLEAR
+}
+
 #define ROUTER_TEST(name, flags)                          \
   { #name, test_router_ ## name, flags, NULL, NULL }
 
 struct testcase_t router_tests[] = {
   ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK),
   ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK),
+  ROUTER_TEST(mark_if_too_old, TT_FORK),
+  ROUTER_TEST(get_my_family, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c
index 727fa5660f..0c6b533698 100644
--- a/src/test/test_routerkeys.c
+++ b/src/test/test_routerkeys.c
@@ -399,7 +399,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -409,7 +409,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -455,11 +455,11 @@ test_routerkeys_ed_keys_init_all(void *arg)
   options->TestingLinkKeySlop = 2*3600;
 
 #ifdef _WIN32
-  mkdir(dir);
-  mkdir(keydir);
+  tt_int_op(0, OP_EQ, mkdir(dir));
+  tt_int_op(0, OP_EQ, mkdir(keydir));
 #else
-  mkdir(dir, 0700);
-  mkdir(keydir, 0700);
+  tt_int_op(0, OP_EQ, mkdir(dir, 0700));
+  tt_int_op(0, OP_EQ, mkdir(keydir, 0700));
 #endif /* defined(_WIN32) */
 
   options->DataDirectory = dir;
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 95c9176faa..6d596e87ea 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -46,7 +46,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "lib/encoding/confline.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "test/test.h"
 #include "test/test_dir_common.h"
@@ -265,7 +265,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   /* Init SR subsystem. */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -275,7 +277,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   construct_consensus(&consensus_text_md, now);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string(consensus_text_md,
+                                                strlen(consensus_text_md),
+                                                NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -301,7 +305,6 @@ test_router_pick_directory_server_impl(void *arg)
   tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until
                                                       + 24*60*60));
   /* These times are outside the test validity period */
-  tt_assert(networkstatus_consensus_is_bootstrapping(now));
   tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60));
   tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60));
 
@@ -475,7 +478,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -626,7 +631,7 @@ mock_clock_skew_warning(const connection_t *conn, long apparent_skew,
   (void)conn;
   mock_apparent_skew = apparent_skew;
   tt_int_op(trusted, OP_EQ, 1);
-  tt_int_op(domain, OP_EQ, LD_GENERAL);
+  tt_i64_op(domain, OP_EQ, LD_GENERAL);
   tt_str_op(received, OP_EQ, "microdesc flavor consensus");
   tt_str_op(source, OP_EQ, "CONSENSUS");
  done:
@@ -648,7 +653,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -662,7 +669,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
   MOCK(clock_skew_warning, mock_clock_skew_warning);
   /* Caller will call teardown_capture_of_logs() */
   setup_capture_of_logs(LOG_WARN);
-  retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+  retval = networkstatus_set_current_consensus(consensus, strlen(consensus),
+                                               "microdesc", 0,
                                                NULL);
 
  done:
diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c
index c45f0e1595..cc73e6c20a 100644
--- a/src/test/test_routerset.c
+++ b/src/test/test_routerset.c
@@ -1765,7 +1765,7 @@ NS(node_get_by_nickname)(const char *nickname, unsigned flags)
  * Structural test for routerset_get_all_nodes, when the nodelist has no nodes.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
+NS_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 static smartlist_t *NS(mock_smartlist);
 
@@ -1795,7 +1795,7 @@ NS(test_main)(void *arg)
     ;
 }
 
-smartlist_t *
+const smartlist_t *
 NS(nodelist_get_list)(void)
 {
   CALLED(nodelist_get_list)++;
@@ -1811,7 +1811,7 @@ NS(nodelist_get_list)(void)
  * the running_only flag is set, but the nodes are not running.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
+NS_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 static smartlist_t *NS(mock_smartlist);
 static node_t NS(mock_node);
@@ -1844,7 +1844,7 @@ NS(test_main)(void *arg)
     ;
 }
 
-smartlist_t *
+const smartlist_t *
 NS(nodelist_get_list)(void)
 {
   CALLED(nodelist_get_list)++;
diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh
index 00b3e88d37..804d2ada36 100755
--- a/src/test/test_rust.sh
+++ b/src/test/test_rust.sh
@@ -14,11 +14,12 @@ rustc_host=$(rustc -vV | grep host | sed 's/host: //')
 
 for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
     if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
+        # shellcheck disable=SC2086
 	cd "${abs_top_builddir:-../../..}/src/rust" && \
 	    CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \
-	    "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} \
+	    "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \
             --features "test_linking_hack" \
-            --target $rustc_host \
+            --target "$rustc_host" \
 	    ${EXTRA_CARGO_OPTIONS} \
 	    --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1
     fi
diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c
new file mode 100644
index 0000000000..eb402232bc
--- /dev/null
+++ b/src/test/test_sendme.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/* Unit tests for handling different kinds of relay cell */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define SENDME_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/circuitlist.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "lib/crypt_ops/crypto_digest.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_md_consensus = current_ns_consensus =
+    tor_malloc_zero(sizeof(networkstatus_t));
+  current_md_consensus->net_params = smartlist_new();
+  current_md_consensus->routerstatus_list = smartlist_new();
+}
+
+static void
+free_mock_consensus(void)
+{
+  SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r,
+                    tor_free(r));
+  smartlist_free(current_md_consensus->routerstatus_list);
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_v1_record_digest(void *arg)
+{
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  /* Create our dummy circuit. */
+  or_circ = or_circuit_new(1, NULL);
+  /* Points it to the OR circuit now. */
+  circ = TO_CIRCUIT(or_circ);
+
+  /* The package window has to be a multiple of CIRCWINDOW_INCREMENT minus 1
+   * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that
+   * shouldn't be noted. */
+  circ->package_window = CIRCWINDOW_INCREMENT;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(!circ->sendme_last_digests);
+
+  /* This should work now. Package window at CIRCWINDOW_INCREMENT + 1. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(circ->sendme_last_digests);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* Next cell in the package window shouldn't do anything. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* The next CIRCWINDOW_INCREMENT should add one more digest. */
+  circ->package_window = (CIRCWINDOW_INCREMENT * 2) + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 2);
+
+ done:
+  circuit_free_(circ);
+}
+
+static void
+test_v1_consensus_params(void *arg)
+{
+  (void) arg;
+
+  setup_mock_consensus();
+  tt_assert(current_md_consensus);
+
+  /* Both zeroes. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=0");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 0);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Both ones. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 1);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Different values from each other. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Validate is the cell version is coherent with our internal default value
+   * and the one in the consensus. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  /* Minimum acceptable value is 1. */
+  tt_int_op(cell_version_can_be_handled(1), OP_EQ, true);
+  /* Minimum acceptable value is 1 so a cell version of 0 is refused. */
+  tt_int_op(cell_version_can_be_handled(0), OP_EQ, false);
+
+ done:
+  free_mock_consensus();
+}
+
+static void
+test_v1_build_cell(void *arg)
+{
+  uint8_t payload[RELAY_PAYLOAD_SIZE], digest[DIGEST_LEN];
+  ssize_t ret;
+  crypto_digest_t *cell_digest = NULL;
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  or_circ = or_circuit_new(1, NULL);
+  circ = TO_CIRCUIT(or_circ);
+  circ->sendme_last_digests = smartlist_new();
+
+  cell_digest = crypto_digest_new();
+  tt_assert(cell_digest);
+  crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20);
+  crypto_digest_get_digest(cell_digest, (char *) digest, sizeof(digest));
+  smartlist_add(circ->sendme_last_digests, tor_memdup(digest, sizeof(digest)));
+
+  /* SENDME v1 payload is 3 bytes + 20 bytes digest. See spec. */
+  ret = build_cell_payload_v1(digest, payload);
+  tt_int_op(ret, OP_EQ, 23);
+
+  /* Validation. */
+
+  /* An empty payload means SENDME version 0 thus valid. */
+  tt_int_op(sendme_is_valid(circ, payload, 0), OP_EQ, true);
+  /* Current phoney digest should have been popped. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+  /* An unparseable cell means invalid. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, (const uint8_t *) "A", 1), OP_EQ, false);
+  expect_log_msg_containing("Unparseable SENDME cell received. "
+                            "Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* No cell digest recorded for this. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  expect_log_msg_containing("We received a SENDME but we have no cell digests "
+                            "to match. Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* Note the wrong digest in the circuit, cell should fail validation. */
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+  expect_log_msg_containing("SENDME v1 cell digest do not match.");
+  teardown_capture_of_logs();
+
+  /* Record the cell digest into the circuit, cell should validate. */
+  memcpy(or_circ->crypto.sendme_digest, digest, sizeof(digest));
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, true);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+ done:
+  crypto_digest_free(cell_digest);
+  circuit_free_(circ);
+}
+
+static void
+test_cell_payload_pad(void *arg)
+{
+  size_t pad_offset, payload_len, expected_offset;
+
+  (void) arg;
+
+  /* Offset should be 0, not enough room for padding. */
+  payload_len = RELAY_PAYLOAD_SIZE;
+  pad_offset = get_pad_cell_offset(payload_len);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Still no room because we keep 4 extra bytes. */
+  pad_offset = get_pad_cell_offset(payload_len - 4);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* We should have 1 byte of padding. Meaning, the offset should be the
+   * CELL_PAYLOAD_SIZE minus 1 byte. */
+  expected_offset = CELL_PAYLOAD_SIZE - 1;
+  pad_offset = get_pad_cell_offset(payload_len - 5);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Now some arbitrary small payload length. The cell size is header + 10 +
+   * extra 4 bytes we keep so the offset should be there. */
+  expected_offset = RELAY_HEADER_SIZE + 10 + 4;
+  pad_offset = get_pad_cell_offset(10);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Data length of 0. */
+  expected_offset = RELAY_HEADER_SIZE + 4;
+  pad_offset = get_pad_cell_offset(0);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+ done:
+  ;
+}
+
+static void
+test_cell_version_validation(void *arg)
+{
+  (void) arg;
+
+  /* We currently only support up to SENDME_MAX_SUPPORTED_VERSION so we are
+   * going to test the boundaries there. */
+
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION));
+
+  /* Version below our supported should pass. */
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION - 1));
+
+  /* Extra version from our supported should fail. */
+  tt_assert(!cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION + 1));
+
+  /* Simple check for version 0. */
+  tt_assert(cell_version_can_be_handled(0));
+
+  /* We MUST handle the default cell version that we emit or accept. */
+  tt_assert(cell_version_can_be_handled(SENDME_EMIT_MIN_VERSION_DEFAULT));
+  tt_assert(cell_version_can_be_handled(SENDME_ACCEPT_MIN_VERSION_DEFAULT));
+
+ done:
+  ;
+}
+
+/* check our decisions about how much stuff to put into relay cells. */
+static void
+test_package_payload_len(void *arg)
+{
+  (void)arg;
+  /* this is not a real circuit: it only has the fields needed for this
+   * test. */
+  circuit_t *c = tor_malloc_zero(sizeof(circuit_t));
+
+  /* check initial conditions. */
+  circuit_reset_sendme_randomness(c);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE, CIRCWINDOW_INCREMENT / 2);
+  tt_int_op(c->send_randomness_after_n_cells, OP_LT, CIRCWINDOW_INCREMENT);
+
+  /* We have a bunch of cells before we need to send randomness, so the first
+   * few can be packaged full. */
+  int initial = c->send_randomness_after_n_cells;
+  size_t n = connection_edge_get_inbuf_bytes_to_package(10000, 0, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  n = connection_edge_get_inbuf_bytes_to_package(95000, 1, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial isn't set, we won't package a partially full cell at
+   * all. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 0, c);
+  tt_int_op(n, OP_EQ, 0);
+  /* no change in our state, since nothing was sent. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial is set and the partial cell is not going to have
+   * _enough_ randomness, we package it, but we don't consider ourselves to
+   * have sent a sufficiently random cell. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-1);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 3);
+
+  /* Make sure we set have_set_sufficiently_random_cell as appropriate. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-64, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-64);
+  tt_assert(c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 4);
+
+  /* Now let's look at what happens when we get down to zero. Since we have
+   * sent a sufficiently random cell, we will not force this one to have a gap.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* What would happen if we hadn't sent a sufficiently random cell? */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  const size_t reduced_payload_size = RELAY_PAYLOAD_SIZE - 4 - 16;
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* Here is a fun case: if it's time to package a small cell, then
+   * package_partial==0 should mean we accept that many bytes.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(reduced_payload_size, 0, c);
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+
+ done:
+  tor_free(c);
+}
+
+struct testcase_t sendme_tests[] = {
+  { "v1_record_digest", test_v1_record_digest, TT_FORK,
+    NULL, NULL },
+  { "v1_consensus_params", test_v1_consensus_params, TT_FORK,
+    NULL, NULL },
+  { "v1_build_cell", test_v1_build_cell, TT_FORK,
+    NULL, NULL },
+  { "cell_payload_pad", test_cell_payload_pad, TT_FORK,
+    NULL, NULL },
+  { "cell_version_validation", test_cell_version_validation, TT_FORK,
+    NULL, NULL },
+  { "package_payload_len", test_package_payload_len, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index b4fe6eef64..9c8703fa6f 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -58,14 +58,17 @@ trusteddirserver_get_by_v3_auth_digest_m(const char *digest)
 }
 
 /* Setup a minimal dirauth environment by initializing the SR state and
- * making sure the options are set to be an authority directory. */
+ * making sure the options are set to be an authority directory.
+ * You must only call this function once per process. */
 static void
 init_authority_state(void)
 {
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   or_options_t *options = get_options_mutable();
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   tt_assert(mock_cert);
   options->AuthoritativeDir = 1;
   tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
@@ -310,6 +313,7 @@ test_get_start_time_of_current_run(void *arg)
 
     retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC",
                                 &mock_consensus.valid_after);
+    tt_int_op(retval, OP_EQ, 0);
 
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC",
                                 &current_time);
@@ -424,7 +428,9 @@ test_sr_commit(void *arg)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -443,12 +449,12 @@ test_sr_commit(void *arg)
     /* We should have a reveal value. */
     tt_assert(commit_has_reveal_value(our_commit));
     /* We should have a random value. */
-    tt_assert(!tor_mem_is_zero((char *) our_commit->random_number,
+    tt_assert(!fast_mem_is_zero((char *) our_commit->random_number,
                                sizeof(our_commit->random_number)));
     /* Commit and reveal timestamp should be the same. */
     tt_u64_op(our_commit->commit_ts, OP_EQ, our_commit->reveal_ts);
     /* We should have a hashed reveal. */
-    tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal,
+    tt_assert(!fast_mem_is_zero(our_commit->hashed_reveal,
                                sizeof(our_commit->hashed_reveal)));
     /* Do we have a valid encoded commit and reveal. Note the following only
      * tests if the generated values are correct. Their could be a bug in
@@ -835,7 +841,9 @@ test_sr_setup_commits(void)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -1073,73 +1081,99 @@ test_sr_get_majority_srv_from_votes(void *arg)
   smartlist_free(votes);
 }
 
+/* Testing sr_srv_dup(). */
 static void
-test_utils(void *arg)
+test_sr_svr_dup(void *arg)
 {
-  (void) arg;
+  (void)arg;
 
-  /* Testing srv_dup(). */
-  {
-    sr_srv_t *srv = NULL, *dup_srv = NULL;
-    const char *srv_value =
-      "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
-    srv = tor_malloc_zero(sizeof(*srv));
-    srv->num_reveals = 42;
-    memcpy(srv->value, srv_value, sizeof(srv->value));
-    dup_srv = srv_dup(srv);
-    tt_assert(dup_srv);
-    tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
-    tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
-    tor_free(srv);
-    tor_free(dup_srv);
-  }
+  sr_srv_t *srv = NULL, *dup_srv = NULL;
+  const char *srv_value =
+    "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
+  srv = tor_malloc_zero(sizeof(*srv));
+  srv->num_reveals = 42;
+  memcpy(srv->value, srv_value, sizeof(srv->value));
+  dup_srv = sr_srv_dup(srv);
+  tt_assert(dup_srv);
+  tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
+  tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
 
-  /* Testing commitments_are_the_same(). Currently, the check is to test the
-   * value of the encoded commit so let's make sure that actually works. */
-  {
-    /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
-     * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
-     * which comes down to SR_COMMIT_BASE64_LEN + 1. */
-    const char *payload =
-      "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
-      "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
-      "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
-      "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
-    sr_commit_t commit1, commit2;
-    memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
-    memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
-    /* Let's corrupt one of them. */
-    memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
-  }
+ done:
+  tor_free(srv);
+  tor_free(dup_srv);
+}
 
-  /* Testing commit_is_authoritative(). */
-  {
-    crypto_pk_t *k = crypto_pk_new();
-    char digest[DIGEST_LEN];
-    sr_commit_t commit;
+/* Testing commitments_are_the_same(). Currently, the check is to test the
+ * value of the encoded commit so let's make sure that actually works. */
+static void
+test_commitments_are_the_same(void *arg)
+{
+  (void)arg;
+
+  /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
+  * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
+  * which comes down to SR_COMMIT_BASE64_LEN + 1. */
+  const char *payload =
+  "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
+  "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
+  "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
+  "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
+  sr_commit_t commit1, commit2;
+  memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
+  memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
+  /* Let's corrupt one of them. */
+  memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
 
-    tt_assert(!crypto_pk_generate_key(k));
+ done:
+  return;
+}
 
-    tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
-    memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
-    /* Change the pubkey. */
-    memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
-    crypto_pk_free(k);
-  }
+/* Testing commit_is_authoritative(). */
+static void
+test_commit_is_authoritative(void *arg)
+{
+  (void)arg;
 
-  /* Testing get_phase_str(). */
-  {
-    tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
-    tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
-  }
+  crypto_pk_t *k = crypto_pk_new();
+  char digest[DIGEST_LEN];
+  sr_commit_t commit;
+
+  tt_assert(!crypto_pk_generate_key(k));
+
+  tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
+  memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
+  /* Change the pubkey. */
+  memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
+
+ done:
+  crypto_pk_free(k);
+}
+
+static void
+test_get_phase_str(void *arg)
+{
+  (void)arg;
+
+  tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
+  tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
+
+ done:
+  return;
+}
+
+/* Test utils that depend on authority state */
+static void
+test_utils_auth(void *arg)
+{
+  (void)arg;
+  init_authority_state();
 
   /* Testing phase transition */
   {
-    init_authority_state();
     set_sr_phase(SR_PHASE_COMMIT);
     tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1);
     tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0);
@@ -1150,8 +1184,193 @@ test_utils(void *arg)
     tt_int_op(is_phase_transition(42), OP_EQ, 1);
   }
 
+  /* Testing get, set, delete, clean SRVs */
+
+  {
+    /* Just set the previous SRV */
+    test_sr_setup_srv(0);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Delete the SRVs one at a time */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now set the SRVs to NULL instead */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now copy the values across */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the previous: the protocol goes the other way */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_previous_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is the same */
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  {
+    /* Now copy a value onto itself */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Take a copy of the old value */
+    sr_srv_t old_current_srv;
+    memcpy(&old_current_srv, sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the current: the protocol never replaces an SRV with
+     * the same value */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_current_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different between current and previous */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is the same as the old content */
+    tt_mem_op(&old_current_srv, OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  /* I don't think we can say "expect a BUG()" in our tests. */
+#if 0
+  {
+    /* Now copy a value onto itself without sr_srv_dup().
+     * This should fail with a BUG() warning. */
+    test_sr_setup_srv(1);
+    sr_state_set_current_srv(sr_state_get_current_srv());
+    sr_state_set_previous_srv(sr_state_get_previous_srv());
+  }
+#endif /* 0 */
+
  done:
-  return;
+  sr_state_free_all();
 }
 
 static void
@@ -1159,6 +1378,7 @@ test_state_transition(void *arg)
 {
   sr_state_t *state = NULL;
   time_t now = time(NULL);
+  sr_srv_t *cur = NULL;
 
   (void) arg;
 
@@ -1197,44 +1417,47 @@ test_state_transition(void *arg)
 
   /* Test SRV rotation in our state. */
   {
-    const sr_srv_t *cur, *prev;
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     tt_assert(cur);
-    /* After, current srv should be the previous and then set to NULL. */
+    /* After, the previous SRV should be the same as the old current SRV, and
+     * the current SRV should be set to NULL */
     state_rotate_srv();
-    prev = sr_state_get_previous_srv();
-    tt_assert(prev == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
     sr_state_clean_srvs();
+    tor_free(cur);
   }
 
   /* New protocol run. */
   {
-    const sr_srv_t *cur;
     /* Setup some new SRVs so we can confirm that a new protocol run
      * actually makes them rotate and compute new ones. */
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
-    tt_assert(cur);
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     set_sr_phase(SR_PHASE_REVEAL);
     MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
     new_protocol_run(now);
     UNMOCK(get_my_v3_authority_cert);
     /* Rotation happened. */
-    tt_assert(sr_state_get_previous_srv() == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     /* We are going into COMMIT phase so we had to rotate our SRVs. Usually
      * our current SRV would be NULL but a new protocol run should make us
      * compute a new SRV. */
     tt_assert(sr_state_get_current_srv());
     /* Also, make sure we did change the current. */
-    tt_assert(sr_state_get_current_srv() != cur);
+    tt_mem_op(sr_state_get_current_srv(), OP_NE, cur, sizeof(sr_srv_t));
     /* We should have our commitment alone. */
     tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
     tt_int_op(state->n_commit_rounds, OP_EQ, 0);
     /* 46 here since we were at 45 just before. */
     tt_u64_op(state->n_protocol_runs, OP_EQ, 46);
+    tor_free(cur);
   }
 
   /* Cleanup of SRVs. */
@@ -1245,6 +1468,7 @@ test_state_transition(void *arg)
   }
 
  done:
+  tor_free(cur);
   sr_state_free_all();
 }
 
@@ -1440,7 +1664,13 @@ struct testcase_t sr_tests[] = {
   { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL },
   { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes,
     TT_FORK, NULL, NULL },
-  { "utils", test_utils, TT_FORK, NULL, NULL },
+  { "sr_svr_dup", test_sr_svr_dup, TT_FORK, NULL, NULL },
+  { "commitments_are_the_same", test_commitments_are_the_same, TT_FORK, NULL,
+    NULL },
+  { "commit_is_authoritative", test_commit_is_authoritative, TT_FORK, NULL,
+    NULL },
+  { "get_phase_str", test_get_phase_str, TT_FORK, NULL, NULL },
+  { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL },
   { "state_transition", test_state_transition, TT_FORK, NULL, NULL },
   { "state_update", test_state_update, TT_FORK,
     NULL, NULL },
diff --git a/src/test/test_slow.c b/src/test/test_slow.c
index bda67b2d92..d4d5b755a5 100644
--- a/src/test/test_slow.c
+++ b/src/test/test_slow.c
@@ -20,7 +20,9 @@
 
 struct testgroup_t testgroups[] = {
   { "slow/crypto/", slow_crypto_tests },
-  { "slow/util/", slow_util_tests },
+  { "slow/process/", slow_process_tests },
+  { "slow/prob_distr/", slow_stochastic_prob_distr_tests },
+  { "slow/ptr/", slow_ptr_tests },
   END_OF_GROUPS
 };
 
diff --git a/src/test/test_socks.c b/src/test/test_socks.c
index 3686e1036b..6d3ebd2320 100644
--- a/src/test/test_socks.c
+++ b/src/test/test_socks.c
@@ -4,7 +4,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_socks.h"
diff --git a/src/test/test_status.c b/src/test/test_status.c
index 9c47469975..2fb2a7b24f 100644
--- a/src/test/test_status.c
+++ b/src/test/test_status.c
@@ -404,7 +404,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -412,7 +412,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -429,7 +429,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 3:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_circuit_handshake_stats"),
                 OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
@@ -442,13 +442,13 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 4:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_link_protocol_counts"),
                 OP_NE, NULL);
       break;
     case 5:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s");
       tt_str_op(va_arg(ap, char *), OP_EQ,
                 " 0 circuits killed with too many cells.");
@@ -574,7 +574,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
   ++NS(n_msgs);
 
   tt_int_op(severity, OP_EQ, LOG_NOTICE);
-  tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+  tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
   tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
   tt_ptr_op(suffix, OP_EQ, NULL);
   tt_str_op(format, OP_EQ,
@@ -709,7 +709,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -723,7 +723,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_accounting"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -889,7 +889,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -903,7 +903,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1038,7 +1038,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1052,7 +1052,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c
index baddf8d66e..19483713f7 100644
--- a/src/test/test_switch_id.c
+++ b/src/test/test_switch_id.c
@@ -87,7 +87,7 @@ main(int argc, char **argv)
 
   fprintf(stderr, "This test is not supported on your OS.\n");
   return 77;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *username;
   const char *testname;
   if (argc != 3) {
diff --git a/src/test/test_switch_id.sh b/src/test/test_switch_id.sh
index 79c44f2eb1..b13bf7602f 100755
--- a/src/test/test_switch_id.sh
+++ b/src/test/test_switch_id.sh
@@ -1,11 +1,11 @@
 #!/bin/sh
 
-if test "`id -u`" != '0'; then
+if test "$(id -u)" != '0'; then
     echo "This test only works when run as root. Skipping." >&2
     exit 77
 fi
 
-if test "`id -u nobody`" = ""; then
+if test "$(id -u nobody)" = ""; then
     echo "This test requires that your system have a 'nobody' user. Sorry." >&2
     exit 1
 fi
diff --git a/src/test/test_token_bucket.c b/src/test/test_token_bucket.c
new file mode 100644
index 0000000000..31670718d9
--- /dev/null
+++ b/src/test/test_token_bucket.c
@@ -0,0 +1,152 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_bwmgt.c
+ * \brief tests for bandwidth management / token bucket functions
+ */
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "lib/evloop/token_bucket.h"
+
+// an imaginary time, in timestamp units. Chosen so it will roll over.
+static const uint32_t START_TS = UINT32_MAX - 1000;
+static const uint32_t RATE = 10;
+static const uint32_t BURST = 50;
+
+static void
+test_token_bucket_ctr_init(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+  tt_uint_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_adjust(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Increase burst. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST * 2);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST * 2);
+
+  /* Decrease burst but still above bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST + 10);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST + 10);
+
+  /* Decrease burst below bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST - 1);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST - 1);
+
+  /* Change rate. */
+  token_bucket_ctr_adjust(&tb, RATE * 2, BURST);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE * 2);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_dec(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Simple decrement by one. */
+  tt_uint_op(0, OP_EQ, token_bucket_ctr_dec(&tb, 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+
+  /* Down to 0. Becomes empty. */
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST - 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, 0);
+
+  /* Reset and try to underflow. */
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST + 1));
+  tt_int_op(tb.counter.bucket, OP_EQ, -1);
+
+  /* Keep underflowing shouldn't flag the bucket as empty. */
+  tt_uint_op(false, OP_EQ, token_bucket_ctr_dec(&tb, BURST));
+  tt_int_op(tb.counter.bucket, OP_EQ, - (int32_t) (BURST + 1));
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_refill(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Reduce of half the bucket and let a single second go before refill. */
+  token_bucket_ctr_dec(&tb, BURST / 2);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST / 2);
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* No time change, nothing should move. */
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* Add 99 seconds, bucket should be back to a full BURST. */
+  token_bucket_ctr_refill(&tb, START_TS + 99);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 99);
+
+  /* Empty bucket at once. */
+  token_bucket_ctr_dec(&tb, BURST);
+  tt_int_op(tb.counter.bucket, OP_EQ, 0);
+  /* On second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 100);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 100);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE);
+  /* A second second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 101);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 101);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE * 2);
+
+ done:
+  ;
+}
+
+#define TOKEN_BUCKET(name)                                          \
+  { #name, test_token_bucket_ ## name , 0, NULL, NULL }
+
+struct testcase_t token_bucket_tests[] = {
+  TOKEN_BUCKET(ctr_init),
+  TOKEN_BUCKET(ctr_adjust),
+  TOKEN_BUCKET(ctr_dec),
+  TOKEN_BUCKET(ctr_refill),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c
index 853abc4f91..d59b8e001d 100644
--- a/src/test/test_tortls.c
+++ b/src/test/test_tortls.c
@@ -230,7 +230,7 @@ test_tortls_tor_tls_get_error(void *data)
   crypto_pk_free(key2);
   tor_tls_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_x509_cert_get_id_digests(void *ignored)
@@ -347,7 +347,7 @@ test_tortls_server_got_renegotiate(void *ignored)
  done:
   tor_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_evaluate_ecgroup_for_tls(void *ignored)
@@ -598,7 +598,7 @@ struct testcase_t tortls_tests[] = {
   LOCAL_TEST_CASE(get_forced_write_size, 0),
   LOCAL_TEST_CASE(used_v1_handshake, TT_FORK),
   LOCAL_TEST_CASE(server_got_renegotiate, 0),
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   LOCAL_TEST_CASE(evaluate_ecgroup_for_tls, 0),
   LOCAL_TEST_CASE(double_init, TT_FORK),
   LOCAL_TEST_CASE(address, TT_FORK),
diff --git a/src/test/test_tortls.h b/src/test/test_tortls.h
index 1a8b117d0f..4567b9f6a0 100644
--- a/src/test/test_tortls.h
+++ b/src/test/test_tortls.h
@@ -10,4 +10,4 @@ extern const char *notCompletelyValidCertString;
 extern const char *validCertString;
 extern const char *caCertString;
 
-#endif
+#endif /* !defined(TEST_TORTLS_H) */
diff --git a/src/test/test_tortls_openssl.c b/src/test/test_tortls_openssl.c
index f039980a25..93135d0b19 100644
--- a/src/test/test_tortls_openssl.c
+++ b/src/test/test_tortls_openssl.c
@@ -133,7 +133,7 @@ library_init(void)
 #else
   SSL_library_init();
   SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 }
 
 static void
@@ -477,7 +477,7 @@ fake_x509_free(X509 *cert)
     tor_free(cert);
   }
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
diff --git a/src/test/test_util.c b/src/test/test_util.c
index 3a0eb15157..aebefe64c5 100644
--- a/src/test/test_util.c
+++ b/src/test/test_util.c
@@ -6,22 +6,24 @@
 #include "orconfig.h"
 #define COMPAT_PRIVATE
 #define COMPAT_TIME_PRIVATE
-#define CONTROL_PRIVATE
 #define UTIL_PRIVATE
 #define UTIL_MALLOC_PRIVATE
 #define SOCKET_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define PROCESS_WIN32_PRIVATE
 #include "lib/testsupport/testsupport.h"
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "feature/control/control.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/transports.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "test/test.h"
 #include "lib/memarea/memarea.h"
 #include "lib/process/waitpid.h"
+#include "lib/process/process_win32.h"
 #include "test/log_test_helpers.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_zstd.h"
@@ -30,7 +32,6 @@
 #include "lib/fs/winlib.h"
 #include "lib/process/env.h"
 #include "lib/process/pidfile.h"
-#include "lib/process/subprocess.h"
 #include "lib/intmath/weakrng.h"
 #include "lib/thread/numcpus.h"
 #include "lib/math/fp.h"
@@ -39,6 +40,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/socketpair.h"
+#include "lib/malloc/map_anon.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
@@ -58,6 +60,12 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
 
 #ifdef _WIN32
 #include <tchar.h>
@@ -69,6 +77,28 @@
 #define INFINITY_DBL ((double)INFINITY)
 #define NAN_DBL ((double)NAN)
 
+/** Test the tor_isinf() wrapper */
+static void
+test_tor_isinf(void *arg)
+{
+  (void) arg;
+
+  tt_assert(tor_isinf(INFINITY_DBL));
+
+  tt_assert(!tor_isinf(NAN_DBL));
+  tt_assert(!tor_isinf(DBL_EPSILON));
+  tt_assert(!tor_isinf(DBL_MAX));
+  tt_assert(!tor_isinf(DBL_MIN));
+
+  tt_assert(!tor_isinf(0.0));
+  tt_assert(!tor_isinf(0.1));
+  tt_assert(!tor_isinf(3));
+  tt_assert(!tor_isinf(3.14));
+
+ done:
+  ;
+}
+
 /* XXXX this is a minimal wrapper to make the unit tests compile with the
  * changed tor_timegm interface. */
 static time_t
@@ -404,7 +434,6 @@ test_util_time(void *arg)
 
 /* Assume tv_usec is an unsigned integer until proven otherwise */
 #define TV_USEC_MAX UINT_MAX
-#define TOR_USEC_PER_SEC 1000000
 
   /* Overflows in the result type */
 
@@ -2058,14 +2087,14 @@ test_util_strmisc(void *arg)
   /* Test mem_is_zero */
   memset(buf,0,128);
   buf[128] = 'x';
-  tt_assert(tor_mem_is_zero(buf, 10));
-  tt_assert(tor_mem_is_zero(buf, 20));
-  tt_assert(tor_mem_is_zero(buf, 128));
-  tt_assert(!tor_mem_is_zero(buf, 129));
+  tt_assert(fast_mem_is_zero(buf, 10));
+  tt_assert(fast_mem_is_zero(buf, 20));
+  tt_assert(fast_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 129));
   buf[60] = (char)255;
-  tt_assert(!tor_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 128));
   buf[0] = (char)1;
-  tt_assert(!tor_mem_is_zero(buf, 10));
+  tt_assert(!fast_mem_is_zero(buf, 10));
 
   /* Test 'escaped' */
   tt_ptr_op(escaped(NULL), OP_EQ, NULL);
@@ -2179,15 +2208,6 @@ test_util_strmisc(void *arg)
   tt_int_op(strcmp_opt(NULL,  "foo"), OP_LT, 0);
   tt_int_op(strcmp_opt("foo", NULL),  OP_GT, 0);
 
-  /* Test strcmp_len */
-  tt_int_op(strcmp_len("foo", "bar", 3),   OP_GT, 0);
-  tt_int_op(strcmp_len("foo", "bar", 2),   OP_LT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */
-  tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0);  /* Really stop at len */
-  tt_int_op(strcmp_len("blah", "", 4),     OP_GT, 0);
-  tt_int_op(strcmp_len("blah", "", 0),    OP_EQ, 0);
-
  done:
   tor_free(cp_tmp);
 }
@@ -3769,7 +3789,7 @@ test_util_memarea(void *arg)
   tt_int_op(((uintptr_t)p3) % sizeof(void*),OP_EQ, 0);
   tt_assert(!memarea_owns_ptr(area, p3+8192));
   tt_assert(!memarea_owns_ptr(area, p3+30));
-  tt_assert(tor_mem_is_zero(p2, 52));
+  tt_assert(fast_mem_is_zero(p2, 52));
   /* Make sure we don't overalign. */
   p1 = memarea_alloc(area, 1);
   p2 = memarea_alloc(area, 1);
@@ -4050,6 +4070,13 @@ test_util_string_is_utf8(void *ptr)
   tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7));
   tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9));
 
+  /* Test the utf8_no_bom function */
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7));
+  tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect",
+                                            strlen("loveandrespect")));
+
   // Validate exactly 'len' bytes.
   tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2));
   tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6));
@@ -4320,204 +4347,6 @@ test_util_load_win_lib(void *ptr)
 }
 #endif /* defined(_WIN32) */
 
-#ifndef _WIN32
-static void
-clear_hex_errno(char *hex_errno)
-{
-  memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1);
-}
-
-static void
-test_util_exit_status(void *ptr)
-{
-  /* Leave an extra byte for a \0 so we can do string comparison */
-  char hex_errno[HEX_ERRNO_SIZE + 1];
-  int n;
-
-  (void)ptr;
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0, hex_errno);
-  tt_str_op("0/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-#if SIZEOF_INT == 4
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x80000000, hex_errno);
-  tt_str_op("FF/-80000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#elif SIZEOF_INT == 8
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno);
-  tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#endif /* SIZEOF_INT == 4 || ... */
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x7F, 0, hex_errno);
-  tt_str_op("7F/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x08, -0x242, hex_errno);
-  tt_str_op("8/-242\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
- done:
-  ;
-}
-#endif /* !defined(_WIN32) */
-
-#ifndef _WIN32
-static void
-test_util_string_from_pipe(void *ptr)
-{
-  int test_pipe[2] = {-1, -1};
-  int retval = 0;
-  enum stream_status status = IO_STREAM_TERM;
-  ssize_t retlen;
-  char buf[4] = { 0 };
-
-  (void)ptr;
-
-  errno = 0;
-
-  /* Set up a pipe to test on */
-  retval = pipe(test_pipe);
-  tt_int_op(retval, OP_EQ, 0);
-
-  /* Send in a string. */
-  retlen = write(test_pipe[1], "ABC", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "ABC");
-  errno = 0;
-
-  /* Send in a string that contains a nul. */
-  retlen = write(test_pipe[1], "AB\0", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a nul only. */
-  retlen = write(test_pipe[1], "\0", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string that contains a trailing newline. */
-  retlen = write(test_pipe[1], "AB\n", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a newline only. */
-  retlen = write(test_pipe[1], "\n", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string and check that we nul terminate return values. */
-  retlen = write(test_pipe[1], "AAA", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AAA");
-  tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf));
-  errno = 0;
-
-  retlen = write(test_pipe[1], "B", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  memset(buf, '\xff', sizeof(buf));
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "B");
-  tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf));
-  errno = 0;
-
-  /* Send in multiple lines. */
-  retlen = write(test_pipe[1], "A\nB", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "A\nB");
-  errno = 0;
-
-  /* Send in a line and close */
-  retlen = write(test_pipe[1], "AB", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retval = close(test_pipe[1]);
-  tt_int_op(retval, OP_EQ, 0);
-  test_pipe[1] = -1;
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Check for EOF */
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_CLOSED);
-  errno = 0;
-
- done:
-  if (test_pipe[0] != -1)
-    close(test_pipe[0]);
-  if (test_pipe[1] != -1)
-    close(test_pipe[1]);
-}
-
-#endif /* !defined(_WIN32) */
-
 /**
  * Test for format_hex_number_sigsafe()
  */
@@ -4612,57 +4441,6 @@ test_util_format_dec_number(void *ptr)
   return;
 }
 
-/**
- * Test that we can properly format a Windows command line
- */
-static void
-test_util_join_win_cmdline(void *ptr)
-{
-  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
-   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
-   * will try to only generate simple cases for the child process to parse;
-   * i.e. we never embed quoted strings in arguments. */
-
-  const char *argvs[][4] = {
-    {"a", "bb", "CCC", NULL}, // Normal
-    {NULL, NULL, NULL, NULL}, // Empty argument list
-    {"", NULL, NULL, NULL}, // Empty argument
-    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
-    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
-    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
-    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
-    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
-    { NULL } // Terminator
-  };
-
-  const char *cmdlines[] = {
-    "a bb CCC",
-    "",
-    "\"\"",
-    "\\\"a b\\\"b CCC\\\"",
-    "\"a\tbc\" \"dd  dd\" E",
-    "a\\\\\\b \"de fg\" H",
-    "a\\\\\\\"b \\c D\\",
-    "\"a\\\\b c\" d E",
-    NULL // Terminator
-  };
-
-  int i;
-  char *joined_argv = NULL;
-
-  (void)ptr;
-
-  for (i=0; cmdlines[i]!=NULL; i++) {
-    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
-    joined_argv = tor_join_win_cmdline(argvs[i]);
-    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
-    tor_free(joined_argv);
-  }
-
- done:
-  tor_free(joined_argv);
-}
-
 #define MAX_SPLIT_LINE_COUNT 4
 struct split_lines_test_t {
   const char *orig_line; // Line to be split (may contain \0's)
@@ -4670,67 +4448,6 @@ struct split_lines_test_t {
   const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines
 };
 
-/**
- * Test that we properly split a buffer into lines
- */
-static void
-test_util_split_lines(void *ptr)
-{
-  /* Test cases. orig_line of last test case must be NULL.
-   * The last element of split_line[i] must be NULL. */
-  struct split_lines_test_t tests[] = {
-    {"", 0, {NULL}},
-    {"foo", 3, {"foo", NULL}},
-    {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}},
-    {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}},
-    {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}},
-    {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}},
-    {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}},
-    {NULL, 0, { NULL }}
-  };
-
-  int i, j;
-  char *orig_line=NULL;
-  smartlist_t *sl=NULL;
-
-  (void)ptr;
-
-  for (i=0; tests[i].orig_line; i++) {
-    sl = smartlist_new();
-    /* Allocate space for string and trailing NULL */
-    orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1);
-    tor_split_lines(sl, orig_line, tests[i].orig_length);
-
-    j = 0;
-    log_info(LD_GENERAL, "Splitting test %d of length %d",
-             i, tests[i].orig_length);
-    SMARTLIST_FOREACH_BEGIN(sl, const char *, line) {
-      /* Check we have not got too many lines */
-      tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j);
-      /* Check that there actually should be a line here */
-      tt_ptr_op(tests[i].split_line[j], OP_NE, NULL);
-      log_info(LD_GENERAL, "Line %d of test %d, should be <%s>",
-               j, i, tests[i].split_line[j]);
-      /* Check that the line is as expected */
-      tt_str_op(line,OP_EQ, tests[i].split_line[j]);
-      j++;
-    } SMARTLIST_FOREACH_END(line);
-    /* Check that we didn't miss some lines */
-    tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]);
-    tor_free(orig_line);
-    smartlist_free(sl);
-    sl = NULL;
-  }
-
- done:
-  tor_free(orig_line);
-  smartlist_free(sl);
-}
-
 static void
 test_util_di_ops(void *arg)
 {
@@ -5682,6 +5399,13 @@ test_util_socketpair(void *arg)
     tt_skip();
   }
 #endif /* defined(__FreeBSD__) */
+#ifdef ENETUNREACH
+  if (ersatz && socketpair_result == -ENETUNREACH) {
+    /* We can also fail with -ENETUNREACH if we have no network stack at
+     * all. */
+    tt_skip();
+  }
+#endif /* defined(ENETUNREACH) */
   tt_int_op(0, OP_EQ, socketpair_result);
 
   tt_assert(SOCKET_OK(fds[0]));
@@ -5834,6 +5558,18 @@ test_util_ipv4_validation(void *arg)
 }
 
 static void
+test_util_ipv6_validation(void *arg)
+{
+  (void)arg;
+
+  tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e"));
+  tt_assert(!string_is_valid_ipv6_address("11:22::33:44:"));
+
+  done:
+  return;
+}
+
+static void
 test_util_writepid(void *arg)
 {
   (void) arg;
@@ -6387,9 +6123,9 @@ test_util_log_mallinfo(void *arg)
   } else {
     tt_u64_op(mem1, OP_LT, mem2);
   }
-#else
+#else /* !defined(HAVE_MALLINFO) */
   tt_skip();
-#endif
+#endif /* defined(HAVE_MALLINFO) */
  done:
   teardown_capture_of_logs();
   tor_free(log1);
@@ -6397,6 +6133,130 @@ test_util_log_mallinfo(void *arg)
   tor_free(mem);
 }
 
+static void
+test_util_map_anon(void *arg)
+{
+  (void)arg;
+  char *ptr = NULL;
+  size_t sz = 16384;
+  unsigned inherit=0;
+
+  /* Basic checks. */
+  ptr = tor_mmap_anonymous(sz, 0, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 3;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz-2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 3);
+
+  /* Try again, with a private (non-swappable) mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+  /* Now let's test a drop-on-fork mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+}
+
+static void
+test_util_map_anon_nofork(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  /* The operating system doesn't support forking. */
+  tt_skip();
+ done:
+  ;
+#else /* !defined(_WIN32) */
+  /* We have the right OS support.  We're going to try marking the buffer as
+   * either zero-on-fork or as drop-on-fork, whichever is supported.  Then we
+   * will fork and send a byte back to the parent process.  This will either
+   * crash, or send zero. */
+
+  char *ptr = NULL;
+  const char TEST_VALUE = 0xd0;
+  size_t sz = 16384;
+  int pipefd[2] = {-1, -1};
+  unsigned inherit=0;
+
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  memset(ptr, (uint8_t)TEST_VALUE, sz);
+
+  tt_int_op(0, OP_EQ, pipe(pipefd));
+  pid_t child = fork();
+  if (child == 0) {
+    /* We're in the child. */
+    close(pipefd[0]);
+    ssize_t r = write(pipefd[1], &ptr[sz-1], 1); /* This may crash. */
+    close(pipefd[1]);
+    if (r < 0)
+      exit(1);
+    exit(0);
+  }
+  tt_int_op(child, OP_GT, 0);
+  /* In the parent. */
+  close(pipefd[1]);
+  pipefd[1] = -1;
+  char buf[1];
+  ssize_t r = read(pipefd[0], buf, 1);
+
+  if (inherit == INHERIT_RES_ZERO) {
+    // We should be seeing clear-on-fork behavior.
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, 0); // that byte should be zero.
+  } else if (inherit == INHERIT_RES_DROP) {
+    // We should be seeing noinherit behavior.
+    tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed.
+  } else {
+    // noinherit isn't implemented.
+    tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, TEST_VALUE); // that byte should be TEST_VALUE.
+  }
+
+  int ws;
+  waitpid(child, &ws, 0);
+
+#ifndef NOINHERIT_CAN_FAIL
+  /* Only if NOINHERIT_CAN_FAIL should it be possible for us to get
+   * INHERIT_KEEP behavior in this case. */
+  tt_int_op(inherit, OP_NE, INHERIT_RES_KEEP);
+#else
+  if (inherit == INHERIT_RES_KEEP) {
+    /* Call this test "skipped", not "passed", since noinherit wasn't
+     * implemented. */
+    tt_skip();
+  }
+#endif /* !defined(NOINHERIT_CAN_FAIL) */
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+  if (pipefd[0] >= 0) {
+    close(pipefd[0]);
+  }
+  if (pipefd[1] >= 0) {
+    close(pipefd[1]);
+  }
+#endif /* defined(_WIN32) */
+}
+
 #define UTIL_LEGACY(name)                                               \
   { #name, test_util_ ## name , 0, NULL, NULL }
 
@@ -6490,12 +6350,8 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(nowrap_math, 0),
   UTIL_TEST(num_cpus, 0),
   UTIL_TEST_WIN_ONLY(load_win_lib, 0),
-  UTIL_TEST_NO_WIN(exit_status, 0),
-  UTIL_TEST_NO_WIN(string_from_pipe, 0),
   UTIL_TEST(format_hex_number, 0),
   UTIL_TEST(format_dec_number, 0),
-  UTIL_TEST(join_win_cmdline, 0),
-  UTIL_TEST(split_lines, 0),
   UTIL_TEST(n_bits_set, 0),
   UTIL_TEST(eat_whitespace, 0),
   UTIL_TEST(sl_new_from_text_lines, 0),
@@ -6512,6 +6368,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(mathlog, 0),
   UTIL_TEST(fraction, 0),
   UTIL_TEST(weak_random, 0),
+  { "tor_isinf", test_tor_isinf, TT_FORK, NULL, NULL },
   { "socket_ipv4", test_util_socket, TT_FORK, &passthrough_setup,
     (void*)"4" },
   { "socket_ipv6", test_util_socket, TT_FORK,
@@ -6524,6 +6381,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(hostname_validation, 0),
   UTIL_TEST(dest_validation_edgecase, 0),
   UTIL_TEST(ipv4_validation, 0),
+  UTIL_TEST(ipv6_validation, 0),
   UTIL_TEST(writepid, 0),
   UTIL_TEST(get_avail_disk_space, 0),
   UTIL_TEST(touch_file, 0),
@@ -6536,5 +6394,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(htonll, 0),
   UTIL_TEST(get_unquoted_path, 0),
   UTIL_TEST(log_mallinfo, 0),
+  UTIL_TEST(map_anon, 0),
+  UTIL_TEST(map_anon_nofork, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c
index d344d0e95c..2859da66b2 100644
--- a/src/test/test_util_format.c
+++ b/src/test/test_util_format.c
@@ -346,7 +346,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswqmjs";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 10);
     tt_str_op(expected, OP_EQ, dst);
   }
 
@@ -357,7 +357,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswq";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 8);
     tt_mem_op(expected, OP_EQ, dst, strlen(expected));
   }
 
@@ -367,7 +367,7 @@ test_util_format_base32_decode(void *arg)
     ret = base32_decode(dst, real_dstlen, "#abcde", 6);
     tt_int_op(ret, OP_EQ, -1);
     /* Make sure the destination buffer has been zeroed even on error. */
-    tt_int_op(tor_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
+    tt_int_op(fast_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
   }
 
  done:
@@ -392,10 +392,13 @@ test_util_format_encoded_size(void *arg)
 
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0);
     tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
+
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i,
                   BASE64_ENCODE_MULTILINE);
     tt_int_op(strlen(outbuf), OP_EQ,
               base64_encode_size(i, BASE64_ENCODE_MULTILINE));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
   }
 
  done:
@@ -417,4 +420,3 @@ struct testcase_t util_format_tests[] = {
   { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c
deleted file mode 100644
index 29e30eaa11..0000000000
--- a/src/test/test_util_slow.c
+++ /dev/null
@@ -1,396 +0,0 @@
-/* Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#define UTIL_PRIVATE
-#define SUBPROCESS_PRIVATE
-#include "lib/crypt_ops/crypto_cipher.h"
-#include "lib/log/log.h"
-#include "lib/process/subprocess.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/printf.h"
-#include "lib/time/compat_time.h"
-#include "test/test.h"
-
-#include <errno.h>
-#include <string.h>
-
-#ifndef BUILDDIR
-#define BUILDDIR "."
-#endif
-
-#ifdef _WIN32
-#define notify_pending_waitpid_callbacks() STMT_NIL
-#define TEST_CHILD "test-child.exe"
-#define EOL "\r\n"
-#else
-#define TEST_CHILD (BUILDDIR "/src/test/test-child")
-#define EOL "\n"
-#endif /* defined(_WIN32) */
-
-#ifdef _WIN32
-/* I've assumed Windows doesn't have the gap between fork and exec
- * that causes the race condition on unix-like platforms */
-#define MATCH_PROCESS_STATUS(s1,s2)         ((s1) == (s2))
-
-#else /* !(defined(_WIN32)) */
-/* work around a race condition of the timing of SIGCHLD handler updates
- * to the process_handle's fields, and checks of those fields
- *
- * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to
- * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */
-#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING  (PROCESS_STATUS_RUNNING+1)
-#define IS_RUNNING_OR_NOTRUNNING(s)           \
-  ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING)
-/* well, this is ugly */
-#define MATCH_PROCESS_STATUS(s1,s2)           \
-  (  (s1) == (s2)                                         \
-     ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s2))                  \
-     ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s1)))
-
-#endif /* defined(_WIN32) */
-
-/** Helper function for testing tor_spawn_background */
-static void
-run_util_spawn_background(const char *argv[], const char *expected_out,
-                          const char *expected_err, int expected_exit,
-                          int expected_status)
-{
-  int retval, exit_code;
-  ssize_t pos;
-  process_handle_t *process_handle=NULL;
-  char stdout_buf[100], stderr_buf[100];
-  int status;
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  notify_pending_waitpid_callbacks();
-
-  /* the race condition doesn't affect status,
-   * because status isn't updated by the SIGCHLD handler,
-   * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, status));
-  if (status == PROCESS_STATUS_ERROR) {
-    tt_ptr_op(process_handle, OP_EQ, NULL);
-    return;
-  }
-
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* When a spawned process forks, fails, then exits very quickly,
-   * (this typically occurs when exec fails)
-   * there is a race condition between the SIGCHLD handler
-   * updating the process_handle's fields, and this test
-   * checking the process status in those fields.
-   * The SIGCHLD update can occur before or after the code below executes.
-   * This causes intermittent failures in spawn_background_fail(),
-   * typically when the machine is under load.
-   * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */
-
-  /* the race condition affects the change in
-   * process_handle->status from RUNNING to NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status));
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  /* the race condition affects the change in
-   * process_handle->waitpid_cb to NULL,
-   * so we skip the check if expected_status is ambiguous,
-   * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(process_handle->waitpid_cb != NULL
-              || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING);
-#endif /* !defined(_WIN32) */
-
-#ifdef _WIN32
-  tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE);
-#else
-  tt_assert(process_handle->stdout_pipe >= 0);
-  tt_assert(process_handle->stderr_pipe >= 0);
-  tt_assert(process_handle->stdin_pipe >= 0);
-#endif /* defined(_WIN32) */
-
-  /* Check stdout */
-  pos = tor_read_all_from_process_stdout(process_handle, stdout_buf,
-                                         sizeof(stdout_buf) - 1);
-  tt_assert(pos >= 0);
-  stdout_buf[pos] = '\0';
-  tt_int_op(strlen(expected_out),OP_EQ, pos);
-  tt_str_op(expected_out,OP_EQ, stdout_buf);
-
-  notify_pending_waitpid_callbacks();
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-  // TODO: Make test-child exit with something other than 0
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
-  notify_pending_waitpid_callbacks();
-
- done:
-  if (process_handle)
-    tor_process_handle_destroy(process_handle, 1);
-}
-
-/** Check that we can launch a process and read the output */
-static void
-test_util_spawn_background_ok(void *ptr)
-{
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL;
-  const char *expected_err = "ERR"EOL;
-
-  (void)ptr;
-
-  run_util_spawn_background(argv, expected_out, expected_err, 0,
-      PROCESS_STATUS_RUNNING);
-}
-
-/** Check that failing to find the executable works as expected */
-static void
-test_util_spawn_background_fail(void *ptr)
-{
-  const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL};
-  const char *expected_err = "";
-  char expected_out[1024];
-  char code[32];
-#ifdef _WIN32
-  const int expected_status = PROCESS_STATUS_ERROR;
-#else
-  /* TODO: Once we can signal failure to exec, set this to be
-   * PROCESS_STATUS_RUNNING_OR_ERROR */
-  const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING;
-#endif /* defined(_WIN32) */
-
-  memset(expected_out, 0xf0, sizeof(expected_out));
-  memset(code, 0xf0, sizeof(code));
-
-  (void)ptr;
-
-  tor_snprintf(code, sizeof(code), "%x/%x",
-    9 /* CHILD_STATE_FAILEXEC */ , ENOENT);
-  tor_snprintf(expected_out, sizeof(expected_out),
-    "ERR: Failed to spawn background process - code %s\n", code);
-
-  run_util_spawn_background(argv, expected_out, expected_err, 255,
-      expected_status);
-}
-
-/** Test that reading from a handle returns a partial read rather than
- * blocking */
-static void
-test_util_spawn_background_partial_read_impl(int exit_early)
-{
-  const int expected_exit = 0;
-  const int expected_status = PROCESS_STATUS_RUNNING;
-
-  int retval, exit_code;
-  ssize_t pos = -1;
-  process_handle_t *process_handle=NULL;
-  int status;
-  char stdout_buf[100], stderr_buf[100];
-
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL,
-                                 "DONE" EOL,
-                                 NULL };
-  const char *expected_err = "ERR" EOL;
-
-#ifndef _WIN32
-  int eof = 0;
-#endif
-  int expected_out_ctr;
-
-  if (exit_early) {
-    argv[1] = "--hang";
-    expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL;
-  }
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-  tt_int_op(expected_status,OP_EQ, status);
-  tt_assert(process_handle);
-  tt_int_op(expected_status,OP_EQ, process_handle->status);
-
-  /* Check stdout */
-  for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) {
-#ifdef _WIN32
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL);
-#else
-    /* Check that we didn't read the end of file last time */
-    tt_assert(!eof);
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL, &eof);
-#endif /* defined(_WIN32) */
-    log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos);
-
-    /* We would have blocked, keep on trying */
-    if (0 == pos)
-      continue;
-
-    tt_assert(pos > 0);
-    stdout_buf[pos] = '\0';
-    tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf);
-    tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos);
-    expected_out_ctr++;
-  }
-
-  if (exit_early) {
-    tor_process_handle_destroy(process_handle, 1);
-    process_handle = NULL;
-    goto done;
-  }
-
-  /* The process should have exited without writing more */
-#ifdef _WIN32
-  pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                            sizeof(stdout_buf) - 1,
-                            process_handle);
-  tt_int_op(0,OP_EQ, pos);
-#else /* !(defined(_WIN32)) */
-  if (!eof) {
-    /* We should have got all the data, but maybe not the EOF flag */
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1,
-                              process_handle, &eof);
-    tt_int_op(0,OP_EQ, pos);
-    tt_assert(eof);
-  }
-  /* Otherwise, we got the EOF on the last read */
-#endif /* defined(_WIN32) */
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-
-  // TODO: Make test-child exit with something other than 0
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-static void
-test_util_spawn_background_partial_read(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(0);
-}
-
-static void
-test_util_spawn_background_exit_early(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(1);
-}
-
-static void
-test_util_spawn_background_waitpid_notify(void *arg)
-{
-  int retval, exit_code;
-  process_handle_t *process_handle=NULL;
-  int status;
-  int ms_timer;
-
-  const char *argv[] = {TEST_CHILD, "--fast", NULL};
-
-  (void) arg;
-
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* We're not going to look at the stdout/stderr output this time. Instead,
-   * we're testing whether notify_pending_waitpid_calbacks() can report the
-   * process exit (on unix) and/or whether tor_get_exit_code() can notice it
-   * (on windows) */
-
-#ifndef _WIN32
-  ms_timer = 30*1000;
-  tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL);
-  while (process_handle->waitpid_cb && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-    notify_pending_waitpid_callbacks();
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif /* !defined(_WIN32) */
-
-  ms_timer = 30*1000;
-  while (((retval = tor_get_exit_code(process_handle, 0, &exit_code))
-                == PROCESS_EXIT_RUNNING) && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-
-  tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-#undef TEST_CHILD
-#undef EOL
-
-#undef MATCH_PROCESS_STATUS
-
-#ifndef _WIN32
-#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING
-#undef IS_RUNNING_OR_NOTRUNNING
-#endif
-
-#define UTIL_TEST(name, flags)                          \
-  { #name, test_util_ ## name, flags, NULL, NULL }
-
-struct testcase_t slow_util_tests[] = {
-  UTIL_TEST(spawn_background_ok, 0),
-  UTIL_TEST(spawn_background_fail, 0),
-  UTIL_TEST(spawn_background_partial_read, 0),
-  UTIL_TEST(spawn_background_exit_early, 0),
-  UTIL_TEST(spawn_background_waitpid_notify, 0),
-  END_OF_TESTCASES
-};
diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c
new file mode 100644
index 0000000000..c8111ea5df
--- /dev/null
+++ b/src/test/test_voting_flags.c
@@ -0,0 +1,191 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define VOTEFLAGS_PRIVATE
+
+#include "core/or/or.h"
+
+#include "feature/dirauth/voteflags.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "app/config/config.h"
+
+#include "test/test.h"
+
+typedef struct {
+  time_t now;
+  routerinfo_t ri;
+  node_t node;
+
+  routerstatus_t expected;
+} flag_vote_test_cfg_t;
+
+static void
+setup_cfg(flag_vote_test_cfg_t *c)
+{
+  memset(c, 0, sizeof(*c));
+
+  c->now = approx_time();
+
+  c->ri.nickname = (char *) "testing100";
+  strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname));
+
+  memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN);
+  memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN);
+
+  c->ri.cache_info.published_on = c->now - 100;
+  c->expected.published_on = c->now - 100;
+
+  c->ri.addr = 0x7f010105;
+  c->expected.addr = 0x7f010105;
+  c->ri.or_port = 9090;
+  c->expected.or_port = 9090;
+
+  tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6);
+  tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6);
+
+  // By default we have no loaded information about stability or speed,
+  // so we'll default to voting "yeah sure." on these two.
+  c->expected.is_fast = 1;
+  c->expected.is_stable = 1;
+}
+
+static bool
+check_result(flag_vote_test_cfg_t *c)
+{
+  bool result = false;
+  routerstatus_t rs;
+  memset(&rs, 0, sizeof(rs));
+  dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0);
+
+  tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on);
+  tt_str_op(rs.nickname, OP_EQ, c->expected.nickname);
+
+  // identity_digest and descriptor_digest are not set here.
+
+  tt_uint_op(rs.addr, OP_EQ, c->expected.addr);
+  tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port);
+  tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port);
+
+  tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr));
+  tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport);
+
+#define FLAG(flagname) \
+  tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname)
+
+  FLAG(is_authority);
+  FLAG(is_exit);
+  FLAG(is_stable);
+  FLAG(is_fast);
+  FLAG(is_flagged_running);
+  FLAG(is_named);
+  FLAG(is_unnamed);
+  FLAG(is_valid);
+  FLAG(is_possible_guard);
+  FLAG(is_bad_exit);
+  FLAG(is_hs_dir);
+  FLAG(is_v2_dir);
+  FLAG(is_staledesc);
+  FLAG(has_bandwidth);
+  FLAG(has_exitsummary);
+  FLAG(bw_is_unmeasured);
+
+  result = true;
+
+ done:
+  return result;
+}
+
+static void
+test_voting_flags_minimal(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  (void) check_result(cfg);
+}
+
+static void
+test_voting_flags_ipv6(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+
+  tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->ri.ipv6_orport = 9091;
+  // no change in expected results, since we aren't set up with ipv6
+  // connectivity.
+  if (!check_result(cfg))
+    goto done;
+
+  get_options_mutable()->AuthDirHasIPv6Connectivity = 1;
+  // no change in expected results, since last_reachable6 won't be set.
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->node.last_reachable6 = cfg->now - 10;
+  // now that lastreachable6 is set, we expect to see the result.
+  tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->expected.ipv6_orport = 9091;
+  if (!check_result(cfg))
+    goto done;
+ done:
+  ;
+}
+
+static void
+test_voting_flags_staledesc(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  time_t now = cfg->now;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  // no change in expectations for is_staledesc
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.is_staledesc = 1;
+  if (!check_result(cfg))
+    goto done;
+
+ done:
+  ;
+}
+
+static void *
+setup_voting_flags_test(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  setup_cfg(cfg);
+  return cfg;
+}
+
+static int
+teardown_voting_flags_test(const struct testcase_t *testcase, void *arg)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = arg;
+  tor_free(cfg);
+  return 1;
+}
+
+static const struct testcase_setup_t voting_flags_setup = {
+  .setup_fn = setup_voting_flags_test,
+  .cleanup_fn = teardown_voting_flags_test,
+};
+
+#define T(name,flags)                                   \
+  { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL }
+
+struct testcase_t voting_flags_tests[] = {
+  T(minimal, 0),
+  T(ipv6, TT_FORK),
+  // TODO: Add more of these tests.
+  T(staledesc, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c
index c58634da5c..ba478a45a4 100644
--- a/src/test/test_workqueue.c
+++ b/src/test/test_workqueue.c
@@ -63,7 +63,7 @@ mark_handled(int serial)
   tor_assert(! bitarray_is_set(handled, serial));
   bitarray_set(handled, serial);
   tor_mutex_release(&bitmap_mutex);
-#else /* !(defined(TRACK_RESPONSES)) */
+#else /* !defined(TRACK_RESPONSES) */
   (void)serial;
 #endif /* defined(TRACK_RESPONSES) */
 }
diff --git a/src/test/test_workqueue_cancel.sh b/src/test/test_workqueue_cancel.sh
index f7c663171e..e50b884f26 100755
--- a/src/test/test_workqueue_cancel.sh
+++ b/src/test/test_workqueue_cancel.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue -C 1
+"${builddir:-.}/src/test/test_workqueue" -C 1
 
diff --git a/src/test/test_workqueue_efd.sh b/src/test/test_workqueue_efd.sh
index 4d89396819..592841fc91 100755
--- a/src/test/test_workqueue_efd.sh
+++ b/src/test/test_workqueue_efd.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_efd2.sh b/src/test/test_workqueue_efd2.sh
index 7cfff45ff3..4cf1b76cbe 100755
--- a/src/test/test_workqueue_efd2.sh
+++ b/src/test/test_workqueue_efd2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_pipe.sh b/src/test/test_workqueue_pipe.sh
index afcef87853..fc3ef34c6c 100755
--- a/src/test/test_workqueue_pipe.sh
+++ b/src/test/test_workqueue_pipe.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-socketpair
diff --git a/src/test/test_workqueue_pipe2.sh b/src/test/test_workqueue_pipe2.sh
index a20a1427e0..7f19ea880d 100755
--- a/src/test/test_workqueue_pipe2.sh
+++ b/src/test/test_workqueue_pipe2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_socketpair.sh b/src/test/test_workqueue_socketpair.sh
index 76af79746d..1ee1776447 100755
--- a/src/test/test_workqueue_socketpair.sh
+++ b/src/test/test_workqueue_socketpair.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-pipe
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index 2c9c4538b9..06a6f312ad 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -12,6 +12,7 @@
 #include "orconfig.h"
 #include "core/or/or.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -25,6 +26,8 @@
 #include "lib/compress/compress.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
+#include "app/main/subsysmgr.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -86,7 +89,7 @@ setup_directory(void)
                  (int)getpid(), rnd32);
     r = mkdir(temp_dir);
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s",
                (int) getpid(), rnd32);
   r = mkdir(temp_dir, 0700);
@@ -230,17 +233,17 @@ void
 tinytest_prefork(void)
 {
   free_pregenerated_keys();
-  crypto_prefork();
+  subsystems_prefork();
 }
 void
 tinytest_postfork(void)
 {
-  crypto_postfork();
+  subsystems_postfork();
   init_pregenerated_keys();
 }
 
 static void
-log_callback_failure(int severity, uint32_t domain, const char *msg)
+log_callback_failure(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)msg;
   if (severity == LOG_ERR || (domain & LD_BUG)) {
@@ -259,24 +262,15 @@ main(int c, const char **v)
   int loglevel = LOG_ERR;
   int accel_crypto = 0;
 
-  /* We must initialise logs before we call tor_assert() */
-  init_logging(1);
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
 
-  update_approx_time(time(NULL));
   options = options_new();
-  tor_threads_init();
-  tor_compress_init();
-
-  network_init();
-
-  monotime_init();
 
   struct tor_libevent_cfg cfg;
   memset(&cfg, 0, sizeof(cfg));
   tor_libevent_initialize(&cfg);
 
   control_initialize_event_queue();
-  configure_backtrace_handler(get_version());
 
   for (i_out = i = 1; i < c; ++i) {
     if (!strcmp(v[i], "--warn")) {
@@ -301,7 +295,7 @@ main(int c, const char **v)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
   {
@@ -309,9 +303,10 @@ main(int c, const char **v)
     log_severity_list_t s;
     memset(&s, 0, sizeof(s));
     set_log_severity_config(LOG_ERR, LOG_ERR, &s);
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_callback_log(&s, log_callback_failure);
   }
+  flush_log_messages_from_startup();
   init_protocol_warning_severity_level();
 
   options->command = CMD_RUN_UNITTESTS;
@@ -367,8 +362,6 @@ main(int c, const char **v)
 
   free_pregenerated_keys();
 
-  crypto_global_cleanup();
-
   if (have_failed)
     return 1;
   else
diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c
index 0f22d4e01b..727449e6eb 100644
--- a/src/test/testing_rsakeys.c
+++ b/src/test/testing_rsakeys.c
@@ -448,7 +448,8 @@ static int next_key_idx_2048;
 static crypto_pk_t *
 pk_generate_internal(int bits)
 {
-  tor_assert(bits == 2048 || bits == 1024);
+  tor_assertf(bits == 2048 || bits == 1024,
+             "Wrong key size: %d", bits);
 
 #ifdef USE_PREGENERATED_RSA_KEYS
   int *idxp;
@@ -467,7 +468,7 @@ pk_generate_internal(int bits)
   *idxp += crypto_rand_int_range(1,3);
   *idxp %= n_pregen;
   return crypto_pk_dup_key(pregen_array[*idxp]);
-#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */
+#else /* !defined(USE_PREGENERATED_RSA_KEYS) */
   crypto_pk_t *result;
   int res;
   result = crypto_pk_new();
diff --git a/src/test/zero_length_keys.sh b/src/test/zero_length_keys.sh
index 5635bdfd89..1702d11245 100755
--- a/src/test/zero_length_keys.sh
+++ b/src/test/zero_length_keys.sh
@@ -19,7 +19,7 @@
 #   3: a command failed - the test could not be completed
 #
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   echo "Usage: ${0} PATH_TO_TOR [-z|-d|-e]"
   exit 1
 elif [ $# -eq 1 ]; then
@@ -31,7 +31,7 @@ else #[$# -gt 1 ]; then
   shift
 fi
 
-DATA_DIR=`mktemp -d -t tor_zero_length_keys.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_zero_length_keys.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -40,7 +40,7 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 touch "$DATA_DIR"/empty_torrc
 touch "$DATA_DIR"/empty_defaults_torrc
diff --git a/src/tools/include.am b/src/tools/include.am
index f7aa7e0d1e..72dfe6017c 100644
--- a/src/tools/include.am
+++ b/src/tools/include.am
@@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve
 endif
 
 src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c
-src_tools_tor_resolve_LDFLAGS =
+src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_LIBS) \
+        $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\
 	$(rust_ldadd) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@
 
@@ -15,8 +17,11 @@ if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_cov_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_TESTING_LIBS) \
+	$(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@
 endif
 
diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c
index 25113420df..ea96f41dbf 100644
--- a/src/tools/tor-gencert.c
+++ b/src/tools/tor-gencert.c
@@ -31,7 +31,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/err.h>
 
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <errno.h>
 
diff --git a/src/tools/tor-print-ed-signing-cert.c b/src/tools/tor-print-ed-signing-cert.c
index 1f1a01ab5c..43a1d7bcbd 100644
--- a/src/tools/tor-print-ed-signing-cert.c
+++ b/src/tools/tor-print-ed-signing-cert.c
@@ -10,11 +10,13 @@
 #include "lib/cc/torint.h"  /* TOR_PRIdSZ */
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/malloc/malloc.h"
+#include "lib/encoding/time_fmt.h"
 
 int
 main(int argc, char **argv)
 {
   ed25519_cert_t *cert = NULL;
+  char rfc1123_buf[RFC1123_TIME_LEN+1] = "";
 
   if (argc != 2) {
     fprintf(stderr, "Usage:\n");
@@ -59,6 +61,11 @@ main(int argc, char **argv)
 
   printf("Expires at: %s", ctime(&expires_at));
 
+  format_rfc1123_time(rfc1123_buf, expires_at);
+  printf("RFC 1123 timestamp: %s\n", rfc1123_buf);
+
+  printf("UNIX timestamp: %ld\n", (long int)expires_at);
+
   ed25519_cert_free(cert);
 
   return 0;
diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c
index 6a84abe557..5d97696c18 100644
--- a/src/tools/tor-resolve.c
+++ b/src/tools/tor-resolve.c
@@ -15,6 +15,7 @@
 #include "lib/string/util_string.h"
 
 #include "lib/net/socks5_status.h"
+#include "trunnel/socks5.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -49,68 +50,204 @@
 
 static void usage(void) ATTR_NORETURN;
 
+/**
+ * Set <b>out</b> to a pointer to newly allocated buffer containing
+ * SOCKS4a RESOLVE request with <b>username</b> and <b>hostname</b>.
+ * Return number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks4a_resolve_request(uint8_t **out,
+                              const char *username,
+                              const char *hostname)
+{
+  tor_assert(out);
+  tor_assert(username);
+  tor_assert(hostname);
+
+  const char *errmsg = NULL;
+  uint8_t *output = NULL;
+  socks4_client_request_t *rq = socks4_client_request_new();
+
+  socks4_client_request_set_version(rq, 4);
+  socks4_client_request_set_command(rq, CMD_RESOLVE);
+  socks4_client_request_set_port(rq, 0);
+  socks4_client_request_set_addr(rq, 0x00000001u);
+  socks4_client_request_set_username(rq, username);
+  socks4_client_request_set_socks4a_addr_hostname(rq, hostname);
+
+  errmsg = socks4_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks4_client_request_encoded_len(rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encoded_len failed";
+    goto cleanup;
+  }
+
+  output = tor_malloc(encoded_len);
+  memset(output, 0, encoded_len);
+
+  encoded_len = socks4_client_request_encode(output, encoded_len, rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encode failed";
+    goto cleanup;
+  }
+
+  *out = output;
+
+ cleanup:
+  socks4_client_request_free(rq);
+  if (errmsg) {
+    log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg);
+    *out = NULL;
+    tor_free(output);
+  }
+  return errmsg ? -1 : encoded_len;
+}
+
+#define SOCKS5_ATYPE_HOSTNAME 0x03
+#define SOCKS5_ATYPE_IPV4     0x01
+#define SOCKS5_ATYPE_IPV6     0x04
+
+/**
+ * Set <b>out</b> to pointer to newly allocated buffer containing
+ * SOCKS5 RESOLVE/RESOLVE_PTR request with given <b>hostname<b>.
+ * Generate a reverse request if <b>reverse</b> is true.
+ * Return the number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks5_resolve_request(uint8_t **out,
+                             const char *hostname,
+                             int reverse)
+{
+  const char *errmsg = NULL;
+  uint8_t *outbuf = NULL;
+  int is_ip_address;
+  tor_addr_t addr;
+  size_t addrlen;
+  int ipv6;
+  is_ip_address = tor_addr_parse(&addr, hostname) != -1;
+  if (!is_ip_address && reverse) {
+    log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
+    return -1;
+  }
+  ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
+  addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
+  if (addrlen > UINT8_MAX) {
+    log_err(LD_GENERAL, "Hostname is too long!");
+    return -1;
+  }
+
+  socks5_client_request_t *rq = socks5_client_request_new();
+
+  socks5_client_request_set_version(rq, 5);
+  /* RESOLVE_PTR or RESOLVE */
+  socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR :
+                                                  CMD_RESOLVE);
+  socks5_client_request_set_reserved(rq, 0);
+
+  uint8_t atype = SOCKS5_ATYPE_HOSTNAME;
+  if (reverse)
+    atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4;
+
+  socks5_client_request_set_atype(rq, atype);
+
+  switch (atype) {
+    case SOCKS5_ATYPE_IPV4: {
+      socks5_client_request_set_dest_addr_ipv4(rq,
+                                        tor_addr_to_ipv4h(&addr));
+    } break;
+    case SOCKS5_ATYPE_IPV6: {
+      uint8_t *ipv6_array =
+        socks5_client_request_getarray_dest_addr_ipv6(rq);
+
+      tor_assert(ipv6_array);
+
+      memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16);
+    } break;
+
+    case SOCKS5_ATYPE_HOSTNAME: {
+      domainname_t *dn = domainname_new();
+      domainname_set_len(dn, addrlen - 1);
+      domainname_setlen_name(dn, addrlen - 1);
+      char *dn_buf = domainname_getarray_name(dn);
+      memcpy(dn_buf, hostname, addrlen - 1);
+
+      errmsg = domainname_check(dn);
+
+      if (errmsg) {
+        domainname_free(dn);
+        goto cleanup;
+      } else {
+        socks5_client_request_set_dest_addr_domainname(rq, dn);
+      }
+    } break;
+    default:
+      tor_assert_unreached();
+      break;
+  }
+
+  socks5_client_request_set_dest_port(rq, 0);
+
+  errmsg = socks5_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks5_client_request_encoded_len(rq);
+  if (encoded_len < 0) {
+    errmsg = "Cannot predict encoded length";
+    goto cleanup;
+  }
+
+  outbuf = tor_malloc(encoded_len);
+  memset(outbuf, 0, encoded_len);
+
+  encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq);
+  if (encoded_len < 0) {
+    errmsg = "encoding failed";
+    goto cleanup;
+  }
+
+  *out = outbuf;
+
+ cleanup:
+  socks5_client_request_free(rq);
+  if (errmsg) {
+    tor_free(outbuf);
+    log_err(LD_NET, "build_socks5_resolve_request failed with error: %s",
+            errmsg);
+  }
+
+  return errmsg ? -1 : encoded_len;
+}
+
 /** Set *<b>out</b> to a newly allocated SOCKS4a resolve request with
  * <b>username</b> and <b>hostname</b> as provided.  Return the number
  * of bytes in the request. */
 static ssize_t
-build_socks_resolve_request(char **out,
+build_socks_resolve_request(uint8_t **out,
                             const char *username,
                             const char *hostname,
                             int reverse,
                             int version)
 {
-  size_t len = 0;
   tor_assert(out);
   tor_assert(username);
   tor_assert(hostname);
 
+  tor_assert(version == 4 || version == 5);
+
   if (version == 4) {
-    len = 8 + strlen(username) + 1 + strlen(hostname) + 1;
-    *out = tor_malloc(len);
-    (*out)[0] = 4;      /* SOCKS version 4 */
-    (*out)[1] = '\xF0'; /* Command: resolve. */
-    set_uint16((*out)+2, htons(0)); /* port: 0. */
-    set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */
-    memcpy((*out)+8, username, strlen(username)+1);
-    memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1);
+    return build_socks4a_resolve_request(out, username, hostname);
   } else if (version == 5) {
-    int is_ip_address;
-    tor_addr_t addr;
-    size_t addrlen;
-    int ipv6;
-    is_ip_address = tor_addr_parse(&addr, hostname) != -1;
-    if (!is_ip_address && reverse) {
-      log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
-      return -1;
-    }
-    ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
-    addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
-    if (addrlen > UINT8_MAX) {
-      log_err(LD_GENERAL, "Hostname is too long!");
-      return -1;
-    }
-    len = 6 + addrlen;
-    *out = tor_malloc(len);
-    (*out)[0] = 5; /* SOCKS version 5 */
-    (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */
-    (*out)[2] = 0; /* reserved. */
-    if (reverse) {
-      (*out)[3] = ipv6 ? 4 : 1;
-      if (ipv6)
-        memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16);
-      else
-        set_uint32((*out)+4, tor_addr_to_ipv4n(&addr));
-    } else {
-      (*out)[3] = 3;
-      (*out)[4] = (char)(uint8_t)(addrlen - 1);
-      memcpy((*out)+5, hostname, addrlen - 1);
-    }
-    set_uint16((*out)+4+addrlen, 0); /* port */
-  } else {
-    tor_assert(0);
+    return build_socks5_resolve_request(out, hostname, reverse);
   }
 
-  return len;
+  tor_assert_unreached();
+  return -1;
 }
 
 static void
@@ -134,34 +271,50 @@ parse_socks4a_resolve_response(const char *hostname,
                                const char *response, size_t len,
                                tor_addr_t *addr_out)
 {
+  int result = 0;
   uint8_t status;
   tor_assert(response);
   tor_assert(addr_out);
 
-  if (len < RESPONSE_LEN_4) {
+  socks4_server_reply_t *reply;
+
+  ssize_t parsed = socks4_server_reply_parse(&reply,
+                                             (const uint8_t *)response,
+                                             len);
+
+  if (parsed == -1) {
+    log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response");
+    result = -1; goto cleanup;
+  }
+
+  if (parsed == -2) {
     log_warn(LD_PROTOCOL,"Truncated socks response.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  if (((uint8_t)response[0])!=0) { /* version: 0 */
+
+  if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */
     log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  status = (uint8_t)response[1];
-  if (get_uint16(response+2)!=0) { /* port: 0 */
+  if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */
     log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
+  status = socks4_server_reply_get_status(reply);
   if (status != 90) {
     log_warn(LD_NET,"Got status response '%d': socks request failed.", status);
     if (!strcasecmpend(hostname, ".onion")) {
       onion_warning(hostname);
-      return -1;
+      result = -1; goto cleanup;
     }
-    return -1;
+    result = -1; goto cleanup;
   }
 
-  tor_addr_from_ipv4n(addr_out, get_uint32(response+4));
-  return 0;
+  tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply));
+
+ cleanup:
+  socks4_server_reply_free(reply);
+  return result;
 }
 
 /* It would be nice to let someone know what SOCKS5 issue a user may have */
@@ -205,7 +358,7 @@ do_resolve(const char *hostname,
   int s = -1;
   struct sockaddr_storage ss;
   socklen_t socklen;
-  char *req = NULL;
+  uint8_t *req = NULL;
   ssize_t len = 0;
 
   tor_assert(hostname);
@@ -230,23 +383,58 @@ do_resolve(const char *hostname,
   }
 
   if (version == 5) {
-    char method_buf[2];
-    if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) {
+    socks5_client_version_t *v = socks5_client_version_new();
+
+    socks5_client_version_set_version(v, 5);
+    socks5_client_version_set_n_methods(v, 1);
+    socks5_client_version_setlen_methods(v, 1);
+    socks5_client_version_set_methods(v, 0, 0x00);
+
+    tor_assert(!socks5_client_version_check(v));
+    ssize_t encoded_len = socks5_client_version_encoded_len(v);
+    tor_assert(encoded_len > 0);
+
+    uint8_t *buf = tor_malloc(encoded_len);
+    encoded_len = socks5_client_version_encode(buf, encoded_len, v);
+    tor_assert(encoded_len > 0);
+
+    socks5_client_version_free(v);
+
+    if (write_all_to_socket(s, (const char *)buf,
+                            encoded_len) != encoded_len) {
       log_err(LD_NET, "Error sending SOCKS5 method list.");
+      tor_free(buf);
+
       goto err;
     }
-    if (read_all_from_socket(s, method_buf, 2) != 2) {
+
+    tor_free(buf);
+
+    uint8_t method_buf[2];
+
+    if (read_all_from_socket(s, (char *)method_buf, 2) != 2) {
       log_err(LD_NET, "Error reading SOCKS5 methods.");
       goto err;
     }
-    if (method_buf[0] != '\x05') {
-      log_err(LD_NET, "Unrecognized socks version: %u",
-              (unsigned)method_buf[0]);
+
+    socks5_server_method_t *m;
+    ssize_t parsed = socks5_server_method_parse(&m, method_buf,
+                                                sizeof(method_buf));
+
+    if (parsed < 2) {
+      log_err(LD_NET, "Failed to parse SOCKS5 method selection "
+                      "message");
+      socks5_server_method_free(m);
       goto err;
     }
-    if (method_buf[1] != '\x00') {
+
+    uint8_t method = socks5_server_method_get_method(m);
+
+    socks5_server_method_free(m);
+
+    if (method != 0x00) {
       log_err(LD_NET, "Unrecognized socks authentication method: %u",
-              (unsigned)method_buf[1]);
+              (unsigned)method);
       goto err;
     }
   }
@@ -257,7 +445,7 @@ do_resolve(const char *hostname,
     tor_assert(!req);
     goto err;
   }
-  if (write_all_to_socket(s, req, len) != len) {
+  if (write_all_to_socket(s, (const char *)req, len) != len) {
     log_sock_error("sending SOCKS request", s);
     tor_free(req);
     goto err;
@@ -276,55 +464,59 @@ do_resolve(const char *hostname,
       goto err;
     }
   } else {
-    char reply_buf[16];
-    if (read_all_from_socket(s, reply_buf, 4) != 4) {
-      log_err(LD_NET, "Error reading SOCKS5 response.");
+    uint8_t reply_buf[512];
+
+    len = read_all_from_socket(s, (char *)reply_buf,
+                               sizeof(reply_buf));
+
+    socks5_server_reply_t *reply;
+
+    ssize_t parsed = socks5_server_reply_parse(&reply,
+                                               reply_buf,
+                                               len);
+    if (parsed == -1) {
+      log_err(LD_NET, "Failed parsing SOCKS5 response");
       goto err;
     }
-    if (reply_buf[0] != 5) {
-      log_err(LD_NET, "Bad SOCKS5 reply version.");
+
+    if (parsed == -2) {
+      log_err(LD_NET, "Truncated SOCKS5 response");
       goto err;
     }
+
     /* Give a user some useful feedback about SOCKS5 errors */
-    if (reply_buf[1] != 0) {
+    uint8_t reply_field = socks5_server_reply_get_reply(reply);
+    if (reply_field != 0) {
       log_warn(LD_NET,"Got SOCKS5 status response '%u': %s",
-               (unsigned)reply_buf[1],
-               socks5_reason_to_string(reply_buf[1]));
-      if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) {
+               (unsigned)reply_field,
+               socks5_reason_to_string(reply_field));
+      if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) {
         onion_warning(hostname);
       }
+
+      socks5_server_reply_free(reply);
       goto err;
     }
-    if (reply_buf[3] == 1) {
+
+    uint8_t atype = socks5_server_reply_get_atype(reply);
+
+    if (atype == SOCKS5_ATYPE_IPV4) {
       /* IPv4 address */
-      if (read_all_from_socket(s, reply_buf, 4) != 4) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf));
-    } else if (reply_buf[3] == 4) {
+      tor_addr_from_ipv4h(result_addr,
+        socks5_server_reply_get_bind_addr_ipv4(reply));
+    } else if (atype == SOCKS5_ATYPE_IPV6) {
       /* IPv6 address */
-      if (read_all_from_socket(s, reply_buf, 16) != 16) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv6_bytes(result_addr, reply_buf);
-    } else if (reply_buf[3] == 3) {
+      tor_addr_from_ipv6_bytes(result_addr,
+        (const char *)socks5_server_reply_getarray_bind_addr_ipv6(reply));
+    } else if (atype == SOCKS5_ATYPE_HOSTNAME) {
       /* Domain name */
-      size_t result_len;
-      if (read_all_from_socket(s, reply_buf, 1) != 1) {
-        log_err(LD_NET, "Error reading address_length in socks5 response.");
-        goto err;
-      }
-      result_len = *(uint8_t*)(reply_buf);
-      *result_hostname = tor_malloc(result_len+1);
-      if (read_all_from_socket(s, *result_hostname, result_len)
-          != (int) result_len) {
-        log_err(LD_NET, "Error reading hostname in socks5 response.");
-        goto err;
-      }
-      (*result_hostname)[result_len] = '\0';
+      domainname_t *dn =
+        socks5_server_reply_get_bind_addr_domainname(reply);
+
+      *result_hostname = tor_strdup(domainname_getstr_name(dn));
     }
+
+    socks5_server_reply_free(reply);
   }
 
   tor_close_socket(s);
diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c
index 59e6b38384..d96496e90c 100644
--- a/src/trunnel/channelpadding_negotiation.c
+++ b/src/trunnel/channelpadding_negotiation.c
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h
index fcfc232fea..3f96174f68 100644
--- a/src/trunnel/channelpadding_negotiation.h
+++ b/src/trunnel/channelpadding_negotiation.h
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c
new file mode 100644
index 0000000000..547818f2ec
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.c
@@ -0,0 +1,549 @@
+/* circpad_negotiation.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "circpad_negotiation.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int circpadnegotiation_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || circpadnegotiation_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+circpad_negotiate_t *
+circpad_negotiate_new(void)
+{
+  circpad_negotiate_t *val = trunnel_calloc(1, sizeof(circpad_negotiate_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiate_clear(circpad_negotiate_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiate_free(circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiate_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiate_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiate_get_version(const circpad_negotiate_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_command(const circpad_negotiate_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp)
+{
+  return inp->echo_request;
+}
+int
+circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->echo_request = val;
+  return 0;
+}
+const char *
+circpad_negotiate_check(const circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiate_encoded_len(const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiate_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+
+  /* Length of u8 echo_request IN [0, 1] */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiate_clear_errors(circpad_negotiate_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiate_encode(uint8_t *output, const size_t avail, const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiate_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiate_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 echo_request IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->echo_request));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiate_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiate_parse_into(circpad_negotiate_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 echo_request IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->echo_request = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    goto fail;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiate_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiate_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiate_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+circpad_negotiated_t *
+circpad_negotiated_new(void)
+{
+  circpad_negotiated_t *val = trunnel_calloc(1, sizeof(circpad_negotiated_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  val->response = CIRCPAD_RESPONSE_ERR;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiated_clear(circpad_negotiated_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiated_free(circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiated_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiated_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiated_get_version(const circpad_negotiated_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_command(const circpad_negotiated_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_response(const circpad_negotiated_t *inp)
+{
+  return inp->response;
+}
+int
+circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_RESPONSE_ERR || val == CIRCPAD_RESPONSE_OK))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->response = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+const char *
+circpad_negotiated_check(const circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiated_encoded_len(const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiated_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiated_clear_errors(circpad_negotiated_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiated_encode(uint8_t *output, const size_t avail, const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiated_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiated_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->response));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiated_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiated_parse_into(circpad_negotiated_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  CHECK_REMAINING(1, truncated);
+  obj->response = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiated_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiated_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiated_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h
new file mode 100644
index 0000000000..ba9155019e
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.h
@@ -0,0 +1,195 @@
+/* circpad_negotiation.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CIRCPAD_NEGOTIATION_H
+#define TRUNNEL_CIRCPAD_NEGOTIATION_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CIRCPAD_COMMAND_STOP 1
+#define CIRCPAD_COMMAND_START 2
+#define CIRCPAD_RESPONSE_OK 1
+#define CIRCPAD_RESPONSE_ERR 2
+#define CIRCPAD_MACHINE_CIRC_SETUP 1
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATE)
+struct circpad_negotiate_st {
+  uint8_t version;
+  uint8_t command;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  /** If true, send a relay_drop reply.. */
+  uint8_t echo_request;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiate_st circpad_negotiate_t;
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATED)
+struct circpad_negotiated_st {
+  uint8_t version;
+  uint8_t command;
+  uint8_t response;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiated_st circpad_negotiated_t;
+/** Return a newly allocated circpad_negotiate with all elements set
+ * to zero.
+ */
+circpad_negotiate_t *circpad_negotiate_new(void);
+/** Release all storage held by the circpad_negotiate in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void circpad_negotiate_free(circpad_negotiate_t *victim);
+/** Try to parse a circpad_negotiate from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * circpad_negotiate_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiate in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiate_encoded_len(const circpad_negotiate_t *obj);
+/** Try to encode the circpad_negotiate from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiate_encode(uint8_t *output, size_t avail, const circpad_negotiate_t *input);
+/** Check whether the internal state of the circpad_negotiate in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *circpad_negotiate_check(const circpad_negotiate_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiate_clear_errors(circpad_negotiate_t *obj);
+/** Return the value of the version field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_version(const circpad_negotiate_t *inp);
+/** Set the value of the version field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_command(const circpad_negotiate_t *inp);
+/** Set the value of the command field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp);
+/** Set the value of the machine_type field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the echo_request field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp);
+/** Set the value of the echo_request field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val);
+/** Return a newly allocated circpad_negotiated with all elements set
+ * to zero.
+ */
+circpad_negotiated_t *circpad_negotiated_new(void);
+/** Release all storage held by the circpad_negotiated in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void circpad_negotiated_free(circpad_negotiated_t *victim);
+/** Try to parse a circpad_negotiated from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated circpad_negotiated_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiated in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiated_encoded_len(const circpad_negotiated_t *obj);
+/** Try to encode the circpad_negotiated from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiated_encode(uint8_t *output, size_t avail, const circpad_negotiated_t *input);
+/** Check whether the internal state of the circpad_negotiated in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *circpad_negotiated_check(const circpad_negotiated_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiated_clear_errors(circpad_negotiated_t *obj);
+/** Return the value of the version field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_version(const circpad_negotiated_t *inp);
+/** Set the value of the version field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_command(const circpad_negotiated_t *inp);
+/** Set the value of the command field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the response field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_response(const circpad_negotiated_t *inp);
+/** Set the value of the response field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp'
+ */
+uint8_t circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp);
+/** Set the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val);
+
+
+#endif
diff --git a/src/trunnel/circpad_negotiation.trunnel b/src/trunnel/circpad_negotiation.trunnel
new file mode 100644
index 0000000000..abbc929cc5
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.trunnel
@@ -0,0 +1,44 @@
+/* These are the padding negotiation commands */
+const CIRCPAD_COMMAND_STOP = 1;
+const CIRCPAD_COMMAND_START = 2;
+
+/* Responses to commands */
+const CIRCPAD_RESPONSE_OK = 1;
+const CIRCPAD_RESPONSE_ERR = 2;
+
+/* Built-in machine types */
+
+/* 1) Machine that obscures circuit setup */
+const CIRCPAD_MACHINE_CIRC_SETUP = 1;
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiate {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+
+  /** If true, send a relay_drop reply.. */
+  // FIXME-MP-AP: Maybe we just say to transition to the first state
+  // here instead.. Also what about delay before responding?
+  u8 echo_request IN [0,1];
+};
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiated {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+  u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+};
diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c
index 1276c7a505..86b79ef9b6 100644
--- a/src/trunnel/ed25519_cert.c
+++ b/src/trunnel/ed25519_cert.c
@@ -1,4 +1,4 @@
-/* ed25519_cert.c -- generated by Trunnel v1.5.2.
+/* ed25519_cert.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h
index e086c6fced..bd91ce1055 100644
--- a/src/trunnel/ed25519_cert.h
+++ b/src/trunnel/ed25519_cert.h
@@ -1,4 +1,4 @@
-/* ed25519_cert.h -- generated by Trunnel v1.5.2.
+/* ed25519_cert.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel
index 8d6483d558..e424ce5464 100644
--- a/src/trunnel/ed25519_cert.trunnel
+++ b/src/trunnel/ed25519_cert.trunnel
@@ -28,12 +28,6 @@ const LS_IPV6 = 0x01;
 const LS_LEGACY_ID = 0x02;
 const LS_ED25519_ID = 0x03;
 
-// XXX hs_link_specifier_dup() violates the opaqueness of link_specifier_t by
-//  taking its sizeof(). If we ever want to turn on TRUNNEL_OPAQUE, or
-//  if we ever make link_specifier contain other types, we will
-//  need to refactor that function to do the copy by encoding and decoding the
-//  object.
-
 // amended from tor.trunnel
 struct link_specifier {
   u8 ls_type;
diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c
index af223560c1..1f50961d69 100644
--- a/src/trunnel/hs/cell_common.c
+++ b/src/trunnel/hs/cell_common.c
@@ -1,4 +1,4 @@
-/* cell_common.c -- generated by Trunnel v1.5.2.
+/* cell_common.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -28,10 +28,10 @@ int cellcommon_deadcode_dummy__ = 0;
     }                                                            \
   } while (0)
 
-trn_cell_extension_fields_t *
-trn_cell_extension_fields_new(void)
+trn_cell_extension_field_t *
+trn_cell_extension_field_new(void)
 {
-  trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_field_t));
   if (NULL == val)
     return NULL;
   return val;
@@ -40,7 +40,7 @@ trn_cell_extension_fields_new(void)
 /** Release all storage held inside 'obj', but do not free 'obj'.
  */
 static void
-trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear(trn_cell_extension_field_t *obj)
 {
   (void) obj;
   TRUNNEL_DYNARRAY_WIPE(&obj->field);
@@ -48,62 +48,62 @@ trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
 }
 
 void
-trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_free(trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return;
-  trn_cell_extension_fields_clear(obj);
-  trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_field_t));
   trunnel_free_(obj);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp)
 {
   return inp->field_type;
 }
 int
-trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_type = val;
   return 0;
 }
 uint8_t
-trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp)
 {
   return inp->field_len;
 }
 int
-trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_len = val;
   return 0;
 }
 size_t
-trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp)
 {
   return TRUNNEL_DYNARRAY_LEN(&inp->field);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->field, idx);
 }
 
 uint8_t
-trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx)
 {
-  return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx);
+  return trn_cell_extension_field_get_field((trn_cell_extension_field_t*)inp, idx);
 }
 int
-trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt)
+trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt)
+trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->field.n_ == UINT8_MAX)
@@ -117,17 +117,17 @@ trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t el
 }
 
 uint8_t *
-trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp)
 {
   return inp->field.elts_;
 }
 const uint8_t  *
-trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp)
 {
-  return (const uint8_t  *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp);
+  return (const uint8_t  *)trn_cell_extension_field_getarray_field((trn_cell_extension_field_t*)inp);
 }
 int
-trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen)
+trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen)
 {
   uint8_t *newptr;
 #if UINT8_MAX < SIZE_MAX
@@ -147,7 +147,7 @@ trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t
   return -1;
 }
 const char *
-trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_check(const trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return "Object was NULL";
@@ -159,11 +159,11 @@ trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
 }
 
 ssize_t
-trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
 
-  if (NULL != trn_cell_extension_fields_check(obj))
+  if (NULL != trn_cell_extension_field_check(obj))
      return -1;
 
 
@@ -178,24 +178,24 @@ trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
   return result;
 }
 int
-trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj)
 {
   int r = obj->trunnel_error_code_;
   obj->trunnel_error_code_ = 0;
   return r;
 }
 ssize_t
-trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encode(uint8_t *output, const size_t avail, const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
   size_t written = 0;
   uint8_t *ptr = output;
   const char *msg;
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
-  const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj);
+  const ssize_t encoded_len = trn_cell_extension_field_encoded_len(obj);
 #endif
 
-  if (NULL != (msg = trn_cell_extension_fields_check(obj)))
+  if (NULL != (msg = trn_cell_extension_field_check(obj)))
     goto check_failed;
 
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
@@ -252,11 +252,11 @@ trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_
   return result;
 }
 
-/** As trn_cell_extension_fields_parse(), but do not allocate the
+/** As trn_cell_extension_field_parse(), but do not allocate the
  * output object.
  */
 static ssize_t
-trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse_into(trn_cell_extension_field_t *obj, const uint8_t *input, const size_t len_in)
 {
   const uint8_t *ptr = input;
   size_t remaining = len_in;
@@ -290,15 +290,15 @@ trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uin
 }
 
 ssize_t
-trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in)
 {
   ssize_t result;
-  *output = trn_cell_extension_fields_new();
+  *output = trn_cell_extension_field_new();
   if (NULL == *output)
     return -1;
-  result = trn_cell_extension_fields_parse_into(*output, input, len_in);
+  result = trn_cell_extension_field_parse_into(*output, input, len_in);
   if (result < 0) {
-    trn_cell_extension_fields_free(*output);
+    trn_cell_extension_field_free(*output);
     *output = NULL;
   }
   return result;
@@ -322,7 +322,7 @@ trn_cell_extension_clear(trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      trn_cell_extension_field_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   TRUNNEL_DYNARRAY_WIPE(&obj->fields);
@@ -356,66 +356,66 @@ trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp)
   return TRUNNEL_DYNARRAY_LEN(&inp->fields);
 }
 
-struct trn_cell_extension_fields_st *
+struct trn_cell_extension_field_st *
 trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
 }
 
- const struct trn_cell_extension_fields_st *
+ const struct trn_cell_extension_field_st *
 trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx)
 {
   return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx);
 }
 int
-trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
-  trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+  trn_cell_extension_field_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
   if (oldval && oldval != elt)
-    trn_cell_extension_fields_free(oldval);
+    trn_cell_extension_field_free(oldval);
   return trn_cell_extension_set0_fields(inp, idx, elt);
 }
 int
-trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->fields.n_ == UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
-  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {});
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_field_st *, &inp->fields, elt, {});
   return 0;
  trunnel_alloc_failed:
   TRUNNEL_SET_ERROR_CODE(inp);
   return -1;
 }
 
-struct trn_cell_extension_fields_st * *
+struct trn_cell_extension_field_st * *
 trn_cell_extension_getarray_fields(trn_cell_extension_t *inp)
 {
   return inp->fields.elts_;
 }
-const struct trn_cell_extension_fields_st *  const  *
+const struct trn_cell_extension_field_st *  const  *
 trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp)
 {
-  return (const struct trn_cell_extension_fields_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
+  return (const struct trn_cell_extension_field_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
 }
 int
 trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen)
 {
-  struct trn_cell_extension_fields_st * *newptr;
+  struct trn_cell_extension_field_st * *newptr;
 #if UINT8_MAX < SIZE_MAX
   if (newlen > UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
   newptr = trunnel_dynarray_setlen(&inp->fields.allocated_,
                  &inp->fields.n_, inp->fields.elts_, newlen,
-                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free,
+                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_field_free,
                  &inp->trunnel_error_code_);
   if (newlen != 0 && newptr == NULL)
     goto trunnel_alloc_failed;
@@ -437,7 +437,7 @@ trn_cell_extension_check(const trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
+      if (NULL != (msg = trn_cell_extension_field_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
         return msg;
     }
   }
@@ -458,12 +458,12 @@ trn_cell_extension_encoded_len(const trn_cell_extension_t *obj)
   /* Length of u8 num */
   result += 1;
 
-  /* Length of struct trn_cell_extension_fields fields[num] */
+  /* Length of struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result += trn_cell_extension_field_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   return result;
@@ -500,13 +500,13 @@ trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_ex
   trunnel_set_uint8(ptr, (obj->num));
   written += 1; ptr += 1;
 
-  /* Encode struct trn_cell_extension_fields fields[num] */
+  /* Encode struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
       trunnel_assert(written <= avail);
-      result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result = trn_cell_extension_field_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
       if (result < 0)
         goto fail; /* XXXXXXX !*/
       written += result; ptr += result;
@@ -553,18 +553,18 @@ trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, c
   obj->num = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
 
-  /* Parse struct trn_cell_extension_fields fields[num] */
-  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {});
+  /* Parse struct trn_cell_extension_field fields[num] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_field_t *, &obj->fields, obj->num, {});
   {
-    trn_cell_extension_fields_t * elt;
+    trn_cell_extension_field_t * elt;
     unsigned idx;
     for (idx = 0; idx < obj->num; ++idx) {
-      result = trn_cell_extension_fields_parse(&elt, ptr, remaining);
+      result = trn_cell_extension_field_parse(&elt, ptr, remaining);
       if (result < 0)
         goto relay_fail;
       trunnel_assert((size_t)result <= remaining);
       remaining -= result; ptr += result;
-      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);});
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_field_t *, &obj->fields, elt, {trn_cell_extension_field_free(elt);});
     }
   }
   trunnel_assert(ptr + remaining == input + len_in);
diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h
index e08eedfdb3..beb65e015f 100644
--- a/src/trunnel/hs/cell_common.h
+++ b/src/trunnel/hs/cell_common.h
@@ -1,4 +1,4 @@
-/* cell_common.h -- generated by Trunnel v1.5.2.
+/* cell_common.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -8,112 +8,112 @@
 #include <stdint.h>
 #include "trunnel.h"
 
-#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS)
-struct trn_cell_extension_fields_st {
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELD)
+struct trn_cell_extension_field_st {
   uint8_t field_type;
   uint8_t field_len;
   TRUNNEL_DYNARRAY_HEAD(, uint8_t) field;
   uint8_t trunnel_error_code_;
 };
 #endif
-typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t;
+typedef struct trn_cell_extension_field_st trn_cell_extension_field_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION)
 struct trn_cell_extension_st {
   uint8_t num;
-  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_field_st *) fields;
   uint8_t trunnel_error_code_;
 };
 #endif
 typedef struct trn_cell_extension_st trn_cell_extension_t;
-/** Return a newly allocated trn_cell_extension_fields with all
+/** Return a newly allocated trn_cell_extension_field with all
  * elements set to zero.
  */
-trn_cell_extension_fields_t *trn_cell_extension_fields_new(void);
-/** Release all storage held by the trn_cell_extension_fields in
+trn_cell_extension_field_t *trn_cell_extension_field_new(void);
+/** Release all storage held by the trn_cell_extension_field in
  * 'victim'. (Do nothing if 'victim' is NULL.)
  */
-void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim);
-/** Try to parse a trn_cell_extension_fields from the buffer in
+void trn_cell_extension_field_free(trn_cell_extension_field_t *victim);
+/** Try to parse a trn_cell_extension_field from the buffer in
  * 'input', using up to 'len_in' bytes from the input buffer. On
  * success, return the number of bytes consumed and set *output to the
- * newly allocated trn_cell_extension_fields_t. On failure, return -2
+ * newly allocated trn_cell_extension_field_t. On failure, return -2
  * if the input appears truncated, and -1 if the input is otherwise
  * invalid.
  */
-ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in);
 /** Return the number of bytes we expect to need to encode the
- * trn_cell_extension_fields in 'obj'. On failure, return a negative
+ * trn_cell_extension_field in 'obj'. On failure, return a negative
  * value. Note that this value may be an overestimate, and can even be
  * an underestimate for certain unencodeable objects.
  */
-ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj);
-/** Try to encode the trn_cell_extension_fields from 'input' into the
+ssize_t trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj);
+/** Try to encode the trn_cell_extension_field from 'input' into the
  * buffer at 'output', using up to 'avail' bytes of the output buffer.
  * On success, return the number of bytes used. On failure, return -2
  * if the buffer was not long enough, and -1 if the input was invalid.
  */
-ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input);
-/** Check whether the internal state of the trn_cell_extension_fields
+ssize_t trn_cell_extension_field_encode(uint8_t *output, size_t avail, const trn_cell_extension_field_t *input);
+/** Check whether the internal state of the trn_cell_extension_field
  * in 'obj' is consistent. Return NULL if it is, and a short message
  * if it is not.
  */
-const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj);
+const char *trn_cell_extension_field_check(const trn_cell_extension_field_t *obj);
 /** Clear any errors that were set on the object 'obj' by its setter
  * functions. Return true iff errors were cleared.
  */
-int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj);
+int trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj);
 /** Return the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the length of the dynamic array holding the field field of
- * the trn_cell_extension_fields_t in 'inp'.
+ * the trn_cell_extension_field_t in 'inp'.
  */
-size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp);
+size_t trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp'.
+ * field of the trn_cell_extension_field_t in 'inp'.
  */
-uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx);
+uint8_t trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp', so that it will
+ * field of the trn_cell_extension_field_t in 'inp', so that it will
  * hold the value 'elt'.
  */
-int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt);
+int trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt);
 /** Append a new element 'elt' to the dynamic array field field of the
- * trn_cell_extension_fields_t in 'inp'.
+ * trn_cell_extension_field_t in 'inp'.
  */
-int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt);
+int trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field field of
  * 'inp'.
  */
-uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t * trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-const uint8_t  * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp);
+const uint8_t  * trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp);
 /** Change the length of the variable-length array field field of
  * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen);
+int trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_extension with all elements set
  * to zero.
  */
@@ -166,32 +166,32 @@ size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp'.
  */
-struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
+struct trn_cell_extension_field_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
- const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
+ const struct trn_cell_extension_field_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp', so that it will hold
  * the value 'elt'. Free the previous value, if any.
  */
-int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** As trn_cell_extension_set_fields, but does not free the previous
  * value.
  */
-int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** Append a new element 'elt' to the dynamic array field fields of
  * the trn_cell_extension_t in 'inp'.
  */
-int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt);
 /** Return a pointer to the variable-length array field fields of
  * 'inp'.
  */
-struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
+struct trn_cell_extension_field_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
-const struct trn_cell_extension_fields_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
+const struct trn_cell_extension_field_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
 /** Change the length of the variable-length array field fields of
  * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
  * elements. Return 0 on success; return -1 and set the error code on
diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel
index 1aa6999de7..7e99cbfa66 100644
--- a/src/trunnel/hs/cell_common.trunnel
+++ b/src/trunnel/hs/cell_common.trunnel
@@ -1,6 +1,6 @@
 /* This file contains common data structure that cells use. */
 
-struct trn_cell_extension_fields {
+struct trn_cell_extension_field {
   u8 field_type;
   u8 field_len;
   u8 field[field_len];
@@ -8,5 +8,5 @@ struct trn_cell_extension_fields {
 
 struct trn_cell_extension {
   u8 num;
-  struct trn_cell_extension_fields fields[num];
+  struct trn_cell_extension_field fields[num];
 };
diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c
index ae3b7b1bc8..f31404c55f 100644
--- a/src/trunnel/hs/cell_establish_intro.c
+++ b/src/trunnel/hs/cell_establish_intro.c
@@ -1,4 +1,4 @@
-/* cell_establish_intro.c -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -36,6 +36,185 @@ ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
 ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
 const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
 int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+trn_cell_extension_dos_param_t *
+trn_cell_extension_dos_param_new(void)
+{
+  trn_cell_extension_dos_param_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_param_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_param_clear(trn_cell_extension_dos_param_t *obj)
+{
+  (void) obj;
+}
+
+void
+trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_param_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_param_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->type;
+}
+int
+trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val)
+{
+  inp->type = val;
+  return 0;
+}
+uint64_t
+trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->value;
+}
+int
+trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val)
+{
+  inp->value = val;
+  return 0;
+}
+const char *
+trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_param_check(obj))
+     return -1;
+
+
+  /* Length of u8 type */
+  result += 1;
+
+  /* Length of u64 value */
+  result += 8;
+  return result;
+}
+int
+trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_param_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_param_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_param_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->type));
+  written += 1; ptr += 1;
+
+  /* Encode u64 value */
+  trunnel_assert(written <= avail);
+  if (avail - written < 8)
+    goto truncated;
+  trunnel_set_uint64(ptr, trunnel_htonll(obj->value));
+  written += 8; ptr += 8;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_param_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_extension_dos_param_parse_into(trn_cell_extension_dos_param_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 type */
+  CHECK_REMAINING(1, truncated);
+  obj->type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u64 value */
+  CHECK_REMAINING(8, truncated);
+  obj->value = trunnel_ntohll(trunnel_get_uint64(ptr));
+  remaining -= 8; ptr += 8;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_param_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_param_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_param_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_establish_intro_t *
 trn_cell_establish_intro_new(void)
 {
@@ -561,6 +740,296 @@ trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_
   }
   return result;
 }
+trn_cell_extension_dos_t *
+trn_cell_extension_dos_new(void)
+{
+  trn_cell_extension_dos_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_clear(trn_cell_extension_dos_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trn_cell_extension_dos_param_free(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->params);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->params);
+}
+
+void
+trn_cell_extension_dos_free(trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp)
+{
+  return inp->n_params;
+}
+int
+trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val)
+{
+  inp->n_params = val;
+  return 0;
+}
+size_t
+trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->params);
+}
+
+struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+}
+
+ const struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return trn_cell_extension_dos_get_params((trn_cell_extension_dos_t*)inp, idx);
+}
+int
+trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  trn_cell_extension_dos_param_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+  if (oldval && oldval != elt)
+    trn_cell_extension_dos_param_free(oldval);
+  return trn_cell_extension_dos_set0_params(inp, idx, elt);
+}
+int
+trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->params, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->params.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_dos_param_st *, &inp->params, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct trn_cell_extension_dos_param_st * *
+trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp)
+{
+  return inp->params.elts_;
+}
+const struct trn_cell_extension_dos_param_st *  const  *
+trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp)
+{
+  return (const struct trn_cell_extension_dos_param_st *  const  *)trn_cell_extension_dos_getarray_params((trn_cell_extension_dos_t*)inp);
+}
+int
+trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen)
+{
+  struct trn_cell_extension_dos_param_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->params.allocated_,
+                 &inp->params.n_, inp->params.elts_, newlen,
+                 sizeof(inp->params.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_dos_param_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->params.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      if (NULL != (msg = trn_cell_extension_dos_param_check(TRUNNEL_DYNARRAY_GET(&obj->params, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->params) != obj->n_params)
+    return "Length mismatch for params";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_params */
+  result += 1;
+
+  /* Length of struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      result += trn_cell_extension_dos_param_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  return result;
+}
+int
+trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_params */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_params));
+  written += 1; ptr += 1;
+
+  /* Encode struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trunnel_assert(written <= avail);
+      result = trn_cell_extension_dos_param_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_extension_dos_parse_into(trn_cell_extension_dos_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_params */
+  CHECK_REMAINING(1, truncated);
+  obj->n_params = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct trn_cell_extension_dos_param params[n_params] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_dos_param_t *, &obj->params, obj->n_params, {});
+  {
+    trn_cell_extension_dos_param_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_params; ++idx) {
+      result = trn_cell_extension_dos_param_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_dos_param_t *, &obj->params, elt, {trn_cell_extension_dos_param_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_intro_established_t *
 trn_cell_intro_established_new(void)
 {
diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h
index ccaef5488c..1924d9cab6 100644
--- a/src/trunnel/hs/cell_establish_intro.h
+++ b/src/trunnel/hs/cell_establish_intro.h
@@ -1,4 +1,4 @@
-/* cell_establish_intro.h -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -10,6 +10,17 @@
 
 struct trn_cell_extension_st;
 #define TRUNNEL_SHA3_256_LEN 32
+#define TRUNNEL_CELL_EXTENSION_TYPE_DOS 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC 2
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS_PARAM)
+struct trn_cell_extension_dos_param_st {
+  uint8_t type;
+  uint64_t value;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_param_st trn_cell_extension_dos_param_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO)
 struct trn_cell_establish_intro_st {
   const uint8_t *start_cell;
@@ -26,6 +37,14 @@ struct trn_cell_establish_intro_st {
 };
 #endif
 typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS)
+struct trn_cell_extension_dos_st {
+  uint8_t n_params;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_dos_param_st *) params;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_st trn_cell_extension_dos_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED)
 struct trn_cell_intro_established_st {
   struct trn_cell_extension_st *extensions;
@@ -33,6 +52,62 @@ struct trn_cell_intro_established_st {
 };
 #endif
 typedef struct trn_cell_intro_established_st trn_cell_intro_established_t;
+/** Return a newly allocated trn_cell_extension_dos_param with all
+ * elements set to zero.
+ */
+trn_cell_extension_dos_param_t *trn_cell_extension_dos_param_new(void);
+/** Release all storage held by the trn_cell_extension_dos_param in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *victim);
+/** Try to parse a trn_cell_extension_dos_param from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_extension_dos_param_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos_param in 'obj'. On failure, return a
+ * negative value. Note that this value may be an overestimate, and
+ * can even be an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj);
+/** Try to encode the trn_cell_extension_dos_param from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t trn_cell_extension_dos_param_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_param_t *input);
+/** Check whether the internal state of the
+ * trn_cell_extension_dos_param in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj);
+/** Return the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val);
+/** Return the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint64_t trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val);
 /** Return a newly allocated trn_cell_establish_intro with all
  * elements set to zero.
  */
@@ -216,6 +291,90 @@ const uint8_t  * trn_cell_establish_intro_getconstarray_sig(const trn_cell_estab
  * -1 and set the error code on 'inp' on failure.
  */
 int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_extension_dos with all elements
+ * set to zero.
+ */
+trn_cell_extension_dos_t *trn_cell_extension_dos_new(void);
+/** Release all storage held by the trn_cell_extension_dos in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_free(trn_cell_extension_dos_t *victim);
+/** Try to parse a trn_cell_extension_dos from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_extension_dos_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj);
+/** Try to encode the trn_cell_extension_dos from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_dos_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_t *input);
+/** Check whether the internal state of the trn_cell_extension_dos in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj);
+/** Return the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp);
+/** Set the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the params field of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+size_t trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+ const struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp', so that it will
+ * hold the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** As trn_cell_extension_dos_set_params, but does not free the
+ * previous value.
+ */
+int trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** Append a new element 'elt' to the dynamic array field params of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+int trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt);
+/** Return a pointer to the variable-length array field params of
+ * 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * * trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_dos_param_st *  const  * trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp);
+/** Change the length of the variable-length array field params of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_intro_established with all
  * elements set to zero.
  */
diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel
index 011ee62a15..e30938f6c2 100644
--- a/src/trunnel/hs/cell_establish_intro.trunnel
+++ b/src/trunnel/hs/cell_establish_intro.trunnel
@@ -39,3 +39,26 @@ struct trn_cell_intro_established {
   /* Extension(s). Reserved fields. */
   struct trn_cell_extension extensions;
 };
+
+/*
+ * ESTABLISH_INTRO cell extensions.
+ */
+
+const TRUNNEL_CELL_EXTENSION_TYPE_DOS = 0x01;
+
+/* DoS Parameter types. */
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC = 0x01;
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC = 0x02;
+
+/*
+ * DoS Parameters Extension. See proposal 305 for more details.
+ */
+struct trn_cell_extension_dos_param {
+  u8 type;
+  u64 value;
+};
+
+struct trn_cell_extension_dos {
+  u8 n_params;
+  struct trn_cell_extension_dos_param params[n_params];
+};
diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c
index 53b3d299f2..016c9fa8d6 100644
--- a/src/trunnel/hs/cell_introduce1.c
+++ b/src/trunnel/hs/cell_introduce1.c
@@ -1,4 +1,4 @@
-/* cell_introduce1.c -- generated by Trunnel v1.5.2.
+/* cell_introduce1.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h
index 986a531ca7..8dabff3cb5 100644
--- a/src/trunnel/hs/cell_introduce1.h
+++ b/src/trunnel/hs/cell_introduce1.h
@@ -1,4 +1,4 @@
-/* cell_introduce1.h -- generated by Trunnel v1.5.2.
+/* cell_introduce1.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c
index 53cb609138..1204e93cfc 100644
--- a/src/trunnel/hs/cell_rendezvous.c
+++ b/src/trunnel/hs/cell_rendezvous.c
@@ -1,4 +1,4 @@
-/* cell_rendezvous.c -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h
index 39e14da25b..5a8c2ff52a 100644
--- a/src/trunnel/hs/cell_rendezvous.h
+++ b/src/trunnel/hs/cell_rendezvous.h
@@ -1,4 +1,4 @@
-/* cell_rendezvous.h -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/include.am b/src/trunnel/include.am
index 03c1753e96..6c3a5ff06b 100644
--- a/src/trunnel/include.am
+++ b/src/trunnel/include.am
@@ -11,7 +11,9 @@ TRUNNELINPUTS = \
 	src/trunnel/link_handshake.trunnel \
 	src/trunnel/pwbox.trunnel \
 	src/trunnel/channelpadding_negotiation.trunnel \
-	src/trunner/socks5.trunnel
+	src/trunnel/sendme_cell.trunnel \
+	src/trunnel/socks5.trunnel \
+	src/trunnel/circpad_negotiation.trunnel
 
 TRUNNELSOURCES = \
 	src/ext/trunnel/trunnel.c \
@@ -23,7 +25,10 @@ TRUNNELSOURCES = \
 	src/trunnel/hs/cell_introduce1.c \
 	src/trunnel/hs/cell_rendezvous.c \
 	src/trunnel/channelpadding_negotiation.c \
-	src/trunnel/socks5.c
+	src/trunnel/sendme_cell.c                    \
+	src/trunnel/socks5.c \
+	src/trunnel/netinfo.c \
+	src/trunnel/circpad_negotiation.c
 
 TRUNNELHEADERS = \
 	src/ext/trunnel/trunnel.h		\
@@ -37,7 +42,10 @@ TRUNNELHEADERS = \
 	src/trunnel/hs/cell_introduce1.h \
 	src/trunnel/hs/cell_rendezvous.h \
 	src/trunnel/channelpadding_negotiation.h \
-	src/trunnel/socks5.h
+	src/trunnel/sendme_cell.h                    \
+	src/trunnel/socks5.h                    \
+	src/trunnel/netinfo.h \
+	src/trunnel/circpad_negotiation.h
 
 src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES)
 src_trunnel_libor_trunnel_a_CPPFLAGS = \
diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c
index 03ead31c62..76db4b0e29 100644
--- a/src/trunnel/link_handshake.c
+++ b/src/trunnel/link_handshake.c
@@ -1,4 +1,4 @@
-/* link_handshake.c -- generated by Trunnel v1.5.2.
+/* link_handshake.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h
index 6a23483adc..0c7ac36b1b 100644
--- a/src/trunnel/link_handshake.h
+++ b/src/trunnel/link_handshake.h
@@ -1,4 +1,4 @@
-/* link_handshake.h -- generated by Trunnel v1.5.2.
+/* link_handshake.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c
new file mode 100644
index 0000000000..d7d0cddc89
--- /dev/null
+++ b/src/trunnel/netinfo.c
@@ -0,0 +1,723 @@
+/* netinfo.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "netinfo.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int netinfo_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+netinfo_addr_t *
+netinfo_addr_new(void)
+{
+  netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_addr_clear(netinfo_addr_t *obj)
+{
+  (void) obj;
+}
+
+void
+netinfo_addr_free(netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_addr_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_addr_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+netinfo_addr_get_addr_type(const netinfo_addr_t *inp)
+{
+  return inp->addr_type;
+}
+int
+netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->addr_type = val;
+  return 0;
+}
+uint8_t
+netinfo_addr_get_len(const netinfo_addr_t *inp)
+{
+  return inp->len;
+}
+int
+netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->len = val;
+  return 0;
+}
+uint32_t
+netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp)
+{
+  return inp->addr_ipv4;
+}
+int
+netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val)
+{
+  inp->addr_ipv4 = val;
+  return 0;
+}
+size_t
+netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->addr_ipv6[idx];
+}
+
+uint8_t
+netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx)
+{
+  return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx);
+}
+int
+netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->addr_ipv6[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp)
+{
+  return inp->addr_ipv6;
+}
+const uint8_t  *
+netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp)
+{
+  return (const uint8_t  *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp);
+}
+const char *
+netinfo_addr_check(const netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+netinfo_addr_encoded_len(const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_addr_check(obj))
+     return -1;
+
+
+  /* Length of u8 addr_type */
+  result += 1;
+
+  /* Length of u8 len */
+  result += 1;
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+
+      /* Length of u32 addr_ipv4 */
+      result += 4;
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+
+      /* Length of u8 addr_ipv6[16] */
+      result += 16;
+      break;
+
+    default:
+      break;
+  }
+  return result;
+}
+int
+netinfo_addr_clear_errors(netinfo_addr_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_addr_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_len = NULL;
+
+  if (NULL != (msg = netinfo_addr_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 addr_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->addr_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 len */
+  backptr_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->len));
+  written += 1; ptr += 1;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union addr[addr_type] */
+    trunnel_assert(written <= avail);
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Encode u32 addr_ipv4 */
+        trunnel_assert(written <= avail);
+        if (avail - written < 4)
+          goto truncated;
+        trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4));
+        written += 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Encode u8 addr_ipv6[16] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 16)
+          goto truncated;
+        memcpy(ptr, obj->addr_ipv6, 16);
+        written += 16; ptr += 16;
+        break;
+
+      default:
+        break;
+    }
+    /* Write the length field back to len */
+    trunnel_assert(written >= written_before_union);
+#if UINT8_MAX < SIZE_MAX
+    if (written - written_before_union > UINT8_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint8(backptr_len, (written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_addr_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 addr_type */
+  CHECK_REMAINING(1, truncated);
+  obj->addr_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 len */
+  CHECK_REMAINING(1, truncated);
+  obj->len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->len, truncated);
+    remaining_after = remaining - obj->len;
+    remaining = obj->len;
+
+    /* Parse union addr[addr_type] */
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Parse u32 addr_ipv4 */
+        CHECK_REMAINING(4, fail);
+        obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr));
+        remaining -= 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Parse u8 addr_ipv6[16] */
+        CHECK_REMAINING(16, fail);
+        memcpy(obj->addr_ipv6, ptr, 16);
+        remaining -= 16; ptr += 16;
+        break;
+
+      default:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_addr_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_addr_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_addr_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+netinfo_cell_t *
+netinfo_cell_new(void)
+{
+  netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_cell_clear(netinfo_cell_t *obj)
+{
+  (void) obj;
+  netinfo_addr_free(obj->other_addr);
+  obj->other_addr = NULL;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs);
+}
+
+void
+netinfo_cell_free(netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_cell_t));
+  trunnel_free_(obj);
+}
+
+uint32_t
+netinfo_cell_get_timestamp(const netinfo_cell_t *inp)
+{
+  return inp->timestamp;
+}
+int
+netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val)
+{
+  inp->timestamp = val;
+  return 0;
+}
+struct netinfo_addr_st *
+netinfo_cell_get_other_addr(netinfo_cell_t *inp)
+{
+  return inp->other_addr;
+}
+const struct netinfo_addr_st *
+netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp)
+{
+  return netinfo_cell_get_other_addr((netinfo_cell_t*) inp);
+}
+int
+netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  if (inp->other_addr && inp->other_addr != val)
+    netinfo_addr_free(inp->other_addr);
+  return netinfo_cell_set0_other_addr(inp, val);
+}
+int
+netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  inp->other_addr = val;
+  return 0;
+}
+uint8_t
+netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp)
+{
+  return inp->n_my_addrs;
+}
+int
+netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val)
+{
+  inp->n_my_addrs = val;
+  return 0;
+}
+size_t
+netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs);
+}
+
+struct netinfo_addr_st *
+netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+}
+
+ const struct netinfo_addr_st *
+netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx)
+{
+  return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx);
+}
+int
+netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+  if (oldval && oldval != elt)
+    netinfo_addr_free(oldval);
+  return netinfo_cell_set0_my_addrs(inp, idx, elt);
+}
+int
+netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt);
+  return 0;
+}
+int
+netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->my_addrs.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct netinfo_addr_st * *
+netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp)
+{
+  return inp->my_addrs.elts_;
+}
+const struct netinfo_addr_st *  const  *
+netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp)
+{
+  return (const struct netinfo_addr_st *  const  *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp);
+}
+int
+netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen)
+{
+  struct netinfo_addr_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_,
+                 &inp->my_addrs.n_, inp->my_addrs.elts_, newlen,
+                 sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->my_addrs.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+netinfo_cell_check(const netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = netinfo_addr_check(obj->other_addr)))
+      return msg;
+  }
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs)
+    return "Length mismatch for my_addrs";
+  return NULL;
+}
+
+ssize_t
+netinfo_cell_encoded_len(const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_cell_check(obj))
+     return -1;
+
+
+  /* Length of u32 timestamp */
+  result += 4;
+
+  /* Length of struct netinfo_addr other_addr */
+  result += netinfo_addr_encoded_len(obj->other_addr);
+
+  /* Length of u8 n_my_addrs */
+  result += 1;
+
+  /* Length of struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  return result;
+}
+int
+netinfo_cell_clear_errors(netinfo_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_cell_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = netinfo_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u32 timestamp */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp));
+  written += 4; ptr += 4;
+
+  /* Encode struct netinfo_addr other_addr */
+  trunnel_assert(written <= avail);
+  result = netinfo_addr_encode(ptr, avail - written, obj->other_addr);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 n_my_addrs */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_my_addrs));
+  written += 1; ptr += 1;
+
+  /* Encode struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      trunnel_assert(written <= avail);
+      result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u32 timestamp */
+  CHECK_REMAINING(4, truncated);
+  obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse struct netinfo_addr other_addr */
+  result = netinfo_addr_parse(&obj->other_addr, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 n_my_addrs */
+  CHECK_REMAINING(1, truncated);
+  obj->n_my_addrs = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct netinfo_addr my_addrs[n_my_addrs] */
+  TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {});
+  {
+    netinfo_addr_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_my_addrs; ++idx) {
+      result = netinfo_addr_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h
new file mode 100644
index 0000000000..37c2ae3c2d
--- /dev/null
+++ b/src/trunnel/netinfo.h
@@ -0,0 +1,226 @@
+/* netinfo.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_NETINFO_H
+#define TRUNNEL_NETINFO_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define NETINFO_ADDR_TYPE_IPV4 4
+#define NETINFO_ADDR_TYPE_IPV6 6
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR)
+struct netinfo_addr_st {
+  uint8_t addr_type;
+  uint8_t len;
+  uint32_t addr_ipv4;
+  uint8_t addr_ipv6[16];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_addr_st netinfo_addr_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL)
+struct netinfo_cell_st {
+  uint32_t timestamp;
+  struct netinfo_addr_st *other_addr;
+  uint8_t n_my_addrs;
+  TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_cell_st netinfo_cell_t;
+/** Return a newly allocated netinfo_addr with all elements set to
+ * zero.
+ */
+netinfo_addr_t *netinfo_addr_new(void);
+/** Release all storage held by the netinfo_addr in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_addr_free(netinfo_addr_t *victim);
+/** Try to parse a netinfo_addr from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_addr_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_addr in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj);
+/** Try to encode the netinfo_addr from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input);
+/** Check whether the internal state of the netinfo_addr in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_addr_check(const netinfo_addr_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_addr_clear_errors(netinfo_addr_t *obj);
+/** Return the value of the addr_type field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp);
+/** Set the value of the addr_type field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the len field of the netinfo_addr_t in 'inp'
+ */
+uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp);
+/** Set the value of the len field of the netinfo_addr_t in 'inp' to
+ * 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp);
+/** Set the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val);
+/** Return the (constant) length of the array holding the addr_ipv6
+ * field of the netinfo_addr_t in 'inp'.
+ */
+size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp'.
+ */
+uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the
+ * value 'elt'.
+ */
+int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'.
+ */
+uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+const uint8_t  * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp);
+/** Return a newly allocated netinfo_cell with all elements set to
+ * zero.
+ */
+netinfo_cell_t *netinfo_cell_new(void);
+/** Release all storage held by the netinfo_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_cell_free(netinfo_cell_t *victim);
+/** Try to parse a netinfo_cell from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj);
+/** Try to encode the netinfo_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input);
+/** Check whether the internal state of the netinfo_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_cell_check(const netinfo_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_cell_clear_errors(netinfo_cell_t *obj);
+/** Return the value of the timestamp field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp);
+/** Set the value of the timestamp field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val);
+/** Return the value of the other_addr field of the netinfo_cell_t in
+ * 'inp'
+ */
+struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp);
+/** As netinfo_cell_get_other_addr, but take and return a const
+ * pointer
+ */
+const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp);
+/** Set the value of the other_addr field of the netinfo_cell_t in
+ * 'inp' to 'val'. Free the old value if any. Steals the referenceto
+ * 'val'.Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** As netinfo_cell_set_other_addr, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** Return the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp);
+/** Set the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the my_addrs field
+ * of the netinfo_cell_t in 'inp'.
+ */
+size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp'.
+ */
+struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+ const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the
+ * value 'elt'. Free the previous value, if any.
+ */
+int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** As netinfo_cell_set_my_addrs, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** Append a new element 'elt' to the dynamic array field my_addrs of
+ * the netinfo_cell_t in 'inp'.
+ */
+int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt);
+/** Return a pointer to the variable-length array field my_addrs of
+ * 'inp'.
+ */
+struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+const struct netinfo_addr_st *  const  * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp);
+/** Change the length of the variable-length array field my_addrs of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel
new file mode 100644
index 0000000000..2c4b7a7591
--- /dev/null
+++ b/src/trunnel/netinfo.trunnel
@@ -0,0 +1,24 @@
+// Warning: make sure these values are consistent with RESOLVED_TYPE_*
+// constants in Tor code and numbers in Section 6.4 of tor-spec.txt.
+
+const NETINFO_ADDR_TYPE_IPV4 = 4;
+const NETINFO_ADDR_TYPE_IPV6 = 6;
+
+struct netinfo_addr {
+  u8 addr_type;
+  u8 len;
+  union addr[addr_type] with length len {
+    NETINFO_ADDR_TYPE_IPV4: u32 ipv4;
+    NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16];
+    default: ignore;
+  };
+
+}
+
+struct netinfo_cell {
+  u32 timestamp;
+  struct netinfo_addr other_addr;
+  u8 n_my_addrs;
+  struct netinfo_addr my_addrs[n_my_addrs];
+}
+
diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c
index c356515d36..c159a5e687 100644
--- a/src/trunnel/pwbox.c
+++ b/src/trunnel/pwbox.c
@@ -1,4 +1,4 @@
-/* pwbox.c -- generated by Trunnel v1.5.2.
+/* pwbox.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h
index a9a421408a..36d595f4ef 100644
--- a/src/trunnel/pwbox.h
+++ b/src/trunnel/pwbox.h
@@ -1,4 +1,4 @@
-/* pwbox.h -- generated by Trunnel v1.5.2.
+/* pwbox.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/sendme_cell.c b/src/trunnel/sendme_cell.c
new file mode 100644
index 0000000000..b9f8fe967f
--- /dev/null
+++ b/src/trunnel/sendme_cell.c
@@ -0,0 +1,347 @@
+/* sendme_cell.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "sendme_cell.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int sendmecell_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || sendmecell_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+sendme_cell_t *
+sendme_cell_new(void)
+{
+  sendme_cell_t *val = trunnel_calloc(1, sizeof(sendme_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+sendme_cell_clear(sendme_cell_t *obj)
+{
+  (void) obj;
+}
+
+void
+sendme_cell_free(sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  sendme_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(sendme_cell_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+sendme_cell_get_version(const sendme_cell_t *inp)
+{
+  return inp->version;
+}
+int
+sendme_cell_set_version(sendme_cell_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint16_t
+sendme_cell_get_data_len(const sendme_cell_t *inp)
+{
+  return inp->data_len;
+}
+int
+sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val)
+{
+  inp->data_len = val;
+  return 0;
+}
+size_t
+sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp)
+{
+  (void)inp;  return TRUNNEL_SENDME_V1_DIGEST_LEN;
+}
+
+uint8_t
+sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  return inp->data_v1_digest[idx];
+}
+
+uint8_t
+sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx)
+{
+  return sendme_cell_get_data_v1_digest((sendme_cell_t*)inp, idx);
+}
+int
+sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  inp->data_v1_digest[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp)
+{
+  return inp->data_v1_digest;
+}
+const uint8_t  *
+sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp)
+{
+  return (const uint8_t  *)sendme_cell_getarray_data_v1_digest((sendme_cell_t*)inp);
+}
+const char *
+sendme_cell_check(const sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0 || obj->version == 1))
+    return "Integer out of bounds";
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+      break;
+
+    default:
+        return "Bad tag for union";
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+sendme_cell_encoded_len(const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != sendme_cell_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0, 1] */
+  result += 1;
+
+  /* Length of u16 data_len */
+  result += 2;
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+
+      /* Length of u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+      result += TRUNNEL_SENDME_V1_DIGEST_LEN;
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+  return result;
+}
+int
+sendme_cell_clear_errors(sendme_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+sendme_cell_encode(uint8_t *output, const size_t avail, const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = sendme_cell_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_data_len = NULL;
+
+  if (NULL != (msg = sendme_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u16 data_len */
+  backptr_data_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->data_len));
+  written += 2; ptr += 2;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union data[version] */
+    trunnel_assert(written <= avail);
+    switch (obj->version) {
+
+      case 0:
+        break;
+
+      case 1:
+
+        /* Encode u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        trunnel_assert(written <= avail);
+        if (avail - written < TRUNNEL_SENDME_V1_DIGEST_LEN)
+          goto truncated;
+        memcpy(ptr, obj->data_v1_digest, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        written += TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        trunnel_assert(0);
+        break;
+    }
+    /* Write the length field back to data_len */
+    trunnel_assert(written >= written_before_union);
+#if UINT16_MAX < SIZE_MAX
+    if (written - written_before_union > UINT16_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint16(backptr_data_len, trunnel_htons(written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As sendme_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+sendme_cell_parse_into(sendme_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0 || obj->version == 1))
+    goto fail;
+
+  /* Parse u16 data_len */
+  CHECK_REMAINING(2, truncated);
+  obj->data_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->data_len, truncated);
+    remaining_after = remaining - obj->data_len;
+    remaining = obj->data_len;
+
+    /* Parse union data[version] */
+    switch (obj->version) {
+
+      case 0:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+
+      case 1:
+
+        /* Parse u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        CHECK_REMAINING(TRUNNEL_SENDME_V1_DIGEST_LEN, fail);
+        memcpy(obj->data_v1_digest, ptr, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        remaining -= TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        goto fail;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = sendme_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = sendme_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    sendme_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/sendme_cell.h b/src/trunnel/sendme_cell.h
new file mode 100644
index 0000000000..45efb9f10d
--- /dev/null
+++ b/src/trunnel/sendme_cell.h
@@ -0,0 +1,101 @@
+/* sendme_cell.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_SENDME_CELL_H
+#define TRUNNEL_SENDME_CELL_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define TRUNNEL_SENDME_V1_DIGEST_LEN 20
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_CELL)
+struct sendme_cell_st {
+  uint8_t version;
+  uint16_t data_len;
+  uint8_t data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct sendme_cell_st sendme_cell_t;
+/** Return a newly allocated sendme_cell with all elements set to
+ * zero.
+ */
+sendme_cell_t *sendme_cell_new(void);
+/** Release all storage held by the sendme_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void sendme_cell_free(sendme_cell_t *victim);
+/** Try to parse a sendme_cell from the buffer in 'input', using up to
+ * 'len_in' bytes from the input buffer. On success, return the number
+ * of bytes consumed and set *output to the newly allocated
+ * sendme_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * sendme_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t sendme_cell_encoded_len(const sendme_cell_t *obj);
+/** Try to encode the sendme_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t sendme_cell_encode(uint8_t *output, size_t avail, const sendme_cell_t *input);
+/** Check whether the internal state of the sendme_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *sendme_cell_check(const sendme_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int sendme_cell_clear_errors(sendme_cell_t *obj);
+/** Return the value of the version field of the sendme_cell_t in
+ * 'inp'
+ */
+uint8_t sendme_cell_get_version(const sendme_cell_t *inp);
+/** Set the value of the version field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_version(sendme_cell_t *inp, uint8_t val);
+/** Return the value of the data_len field of the sendme_cell_t in
+ * 'inp'
+ */
+uint16_t sendme_cell_get_data_len(const sendme_cell_t *inp);
+/** Set the value of the data_len field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the
+ * data_v1_digest field of the sendme_cell_t in 'inp'.
+ */
+size_t sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp'.
+ */
+uint8_t sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+uint8_t sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SENDME_V1_DIGEST_LEN-element array
+ * field data_v1_digest of 'inp'.
+ */
+uint8_t * sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+const uint8_t  * sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp);
+
+
+#endif
diff --git a/src/trunnel/sendme_cell.trunnel b/src/trunnel/sendme_cell.trunnel
new file mode 100644
index 0000000000..300963e679
--- /dev/null
+++ b/src/trunnel/sendme_cell.trunnel
@@ -0,0 +1,19 @@
+/* This file contains the SENDME cell definition. */
+
+/* v1 digest length in bytes. */
+const TRUNNEL_SENDME_V1_DIGEST_LEN = 20;
+
+/* SENDME cell declaration. */
+struct sendme_cell {
+  /* Version field. */
+  u8 version IN [0x00, 0x01];
+
+  /* Length of data contained in this cell. */
+  u16 data_len;
+
+  /* The data content depends on the version. */
+  union data[version] with length data_len {
+    0x00: ignore;
+    0x01: u8 v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  };
+}
diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c
index 9e5f6fcfed..f32862e353 100644
--- a/src/trunnel/socks5.c
+++ b/src/trunnel/socks5.c
@@ -1,4 +1,4 @@
-/* socks5.c -- generated by Trunnel v1.5.2.
+/* socks5.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -1694,7 +1694,6 @@ socks4_server_reply_new(void)
   socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t));
   if (NULL == val)
     return NULL;
-  val->version = 4;
   return val;
 }
 
@@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp)
 int
 socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val)
 {
-  if (! ((val == 4))) {
+  if (! ((val == 0 || val == 4))) {
      TRUNNEL_SET_ERROR_CODE(inp);
      return -1;
   }
@@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj)
     return "Object was NULL";
   if (obj->trunnel_error_code_)
     return "A set function failed on this object";
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     return "Integer out of bounds";
   return NULL;
 }
@@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj)
      return -1;
 
 
-  /* Length of u8 version IN [4] */
+  /* Length of u8 version IN [0, 4] */
   result += 1;
 
   /* Length of u8 status */
@@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser
   trunnel_assert(encoded_len >= 0);
 #endif
 
-  /* Encode u8 version IN [4] */
+  /* Encode u8 version IN [0, 4] */
   trunnel_assert(written <= avail);
   if (avail - written < 1)
     goto truncated;
@@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input,
   ssize_t result = 0;
   (void)result;
 
-  /* Parse u8 version IN [4] */
+  /* Parse u8 version IN [0, 4] */
   CHECK_REMAINING(1, truncated);
   obj->version = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     goto fail;
 
   /* Parse u8 status */
diff --git a/src/trunnel/socks5.h b/src/trunnel/socks5.h
index d3bea152e7..23ac64faba 100644
--- a/src/trunnel/socks5.h
+++ b/src/trunnel/socks5.h
@@ -1,4 +1,4 @@
-/* socks5.h -- generated by Trunnel v1.5.2.
+/* socks5.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel
index b86ec03b9d..17fa2ed996 100644
--- a/src/trunnel/socks5.trunnel
+++ b/src/trunnel/socks5.trunnel
@@ -86,7 +86,7 @@ struct socks4_client_request {
 }
 
 struct socks4_server_reply {
-   u8 version IN [4];
+   u8 version IN [0,4];
    u8 status;
    u16 port;
    u32 addr;
diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h
index c4118fce4c..80da371560 100644
--- a/src/trunnel/trunnel-local.h
+++ b/src/trunnel/trunnel-local.h
@@ -14,5 +14,6 @@
 #define trunnel_reallocarray tor_reallocarray
 #define trunnel_assert tor_assert
 #define trunnel_memwipe(mem, len) memwipe((mem), 0, (len))
+#define trunnel_abort tor_abort_
 
 #endif
diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h
index ec7a5b2429..38464f1c59 100644
--- a/src/win32/orconfig.h
+++ b/src/win32/orconfig.h
@@ -218,7 +218,7 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.3.5.11-dev"
+#define VERSION "0.4.2.8-dev"