1184 files changed, 76086 insertions, 24324 deletions

diff --git a/.gitignore b/.gitignore
index f1ce903a11..77610b3193 100644
--- a/.gitignore
+++ b/.gitignore
@@ -158,14 +158,20 @@ uptime-*.json
 
 # /src/lib
 /src/lib/libcurve25519_donna.a
+/src/lib/libtor-buf.a
+/src/lib/libtor-buf-testing.a
 /src/lib/libtor-compress.a
 /src/lib/libtor-compress-testing.a
+/src/lib/libtor-confmgt.a
+/src/lib/libtor-confmgt-testing.a
 /src/lib/libtor-container.a
 /src/lib/libtor-container-testing.a
 /src/lib/libtor-crypt-ops.a
 /src/lib/libtor-crypt-ops-testing.a
 /src/lib/libtor-ctime.a
 /src/lib/libtor-ctime-testing.a
+/src/lib/libtor-dispatch.a
+/src/lib/libtor-dispatch-testing.a
 /src/lib/libtor-encoding.a
 /src/lib/libtor-encoding-testing.a
 /src/lib/libtor-evloop.a
@@ -198,6 +204,8 @@ uptime-*.json
 /src/lib/libtor-osinfo-testing.a
 /src/lib/libtor-process.a
 /src/lib/libtor-process-testing.a
+/src/lib/libtor-pubsub.a
+/src/lib/libtor-pubsub-testing.a
 /src/lib/libtor-sandbox.a
 /src/lib/libtor-sandbox-testing.a
 /src/lib/libtor-string.a
@@ -213,6 +221,8 @@ uptime-*.json
 /src/lib/libtor-tls.a
 /src/lib/libtor-tls-testing.a
 /src/lib/libtor-trace.a
+/src/lib/libtor-version.a
+/src/lib/libtor-version-testing.a
 /src/lib/libtor-wallclock.a
 /src/lib/libtor-wallclock-testing.a
 
@@ -240,20 +250,22 @@ uptime-*.json
 /src/test/test
 /src/test/test-slow
 /src/test/test-bt-cl
-/src/test/test-child
+/src/test/test-process
 /src/test/test-memwipe
 /src/test/test-ntor-cl
 /src/test/test-hs-ntor-cl
+/src/test/test-rng
 /src/test/test-switch-id
 /src/test/test-timers
 /src/test/test_workqueue
 /src/test/test.exe
 /src/test/test-slow.exe
 /src/test/test-bt-cl.exe
-/src/test/test-child.exe
+/src/test/test-process.exe
 /src/test/test-ntor-cl.exe
 /src/test/test-hs-ntor-cl.exe
 /src/test/test-memwipe.exe
+/src/test/test-rng.exe
 /src/test/test-switch-id.exe
 /src/test/test-timers.exe
 /src/test/test_workqueue.exe
diff --git a/.travis.yml b/.travis.yml
index 8bc9898f8f..3a160840a7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,6 +29,8 @@ env:
     - HARDENING_OPTIONS="--enable-expensive-hardening"
     ## We turn off asciidoc by default, because it's slow
     - ASCIIDOC_OPTIONS="--disable-asciidoc"
+    ## Our default rust version is the minimum supported version
+    - RUST_VERSION="1.31.0"
     ## Turn off tor's sandbox in chutney, until we fix sandbox errors that are
     ## triggered by Ubuntu Xenial and Bionic. See #32722.
     - CHUTNEY_TOR_SANDBOX="0"
@@ -70,7 +72,7 @@ matrix:
     - env: NSS_OPTIONS="--enable-nss" C_DIALECT_OPTIONS="-std=gnu99"
 
     ## We include a single coverage build with the best options for coverage
-    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS=""
+    - env: COVERAGE_OPTIONS="--enable-coverage" HARDENING_OPTIONS="" TOR_TEST_RNG_SEED="636f766572616765"
 
     ## We clone our stem repo and run `make test-stem`
     - env: TEST_STEM="yes" SKIP_MAKE_CHECK="yes"
@@ -79,7 +81,7 @@ matrix:
     ## concurrent macOS jobs. We're not actively developing Rust, so it is
     ## the lowest priority.
     ## We run rust on macOS, because we have seen macOS rust failures before
-    #- env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+    #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
     #  compiler: clang
     #  os: osx
 
@@ -96,7 +98,7 @@ matrix:
 
     ## Since this job is disabled, there's not much point having an exception
     ## for it
-    #- env: RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
+    #- env: RUST_VERSION="nightly" RUST_OPTIONS="--enable-rust --enable-cargo-online-mode"
     #  compiler: clang
     #  os: osx
 
@@ -125,6 +127,7 @@ addons:
       - libscrypt-dev
       - libseccomp-dev
       - libzstd-dev
+      - shellcheck
       ## Conditional build dependencies
       ## Always installed, so we don't need sudo
       - asciidoc
@@ -153,6 +156,7 @@ addons:
       - pkg-config
       ## Optional build dependencies
       - ccache
+      - shellcheck
       ## Conditional build dependencies
       ## Always installed, because manual brew installs are hard to get right
       - asciidoc
@@ -183,8 +187,8 @@ install:
   - if [[ "$ASCIIDOC_OPTIONS" == "" ]] && [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export XML_CATALOG_FILES="/usr/local/etc/xml/catalog"; fi
   ## If we're using Rust, download rustup
   - if [[ "$RUST_OPTIONS" != "" ]]; then curl -Ssf -o rustup.sh https://sh.rustup.rs; fi
-  ## Install the nightly channels of rustc and cargo and setup our toolchain environment
-  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain nightly; fi
+  ## Install the stable channels of rustc and cargo and setup our toolchain environment
+  - if [[ "$RUST_OPTIONS" != "" ]]; then sh rustup.sh -y --default-toolchain $RUST_VERSION; fi
   - if [[ "$RUST_OPTIONS" != "" ]]; then source $HOME/.cargo/env; fi
   ## If we're testing rust builds in offline-mode, then set up our vendored dependencies
   - if [[ "$TOR_RUST_DEPENDENCIES" == "true" ]]; then export TOR_RUST_DEPENDENCIES=$PWD/src/ext/rust/crates; fi
@@ -209,6 +213,10 @@ install:
   - if [[ "$CHUTNEY" != "" ]]; then pushd "$CHUTNEY_PATH"; git log -1 ; popd ; fi
   ## If we're running stem, show the stem version and commit
   - if [[ "$TEST_STEM" != "" ]]; then pushd stem; python -c "from stem import stem; print(stem.__version__);"; git log -1; popd; fi
+  ## We don't want Tor tests to depend on default configuration file at
+  ## ~/.torrc. So we put some random bytes in there, to make sure we get build
+  ## failures in case Tor is reading it during CI jobs.
+  - dd ibs=1 count=1024 if=/dev/urandom > ~/.torrc
 
 script:
   # Skip test_rebind on macOS
diff --git a/ChangeLog b/ChangeLog
index cdf7249059..5b86efe7bd 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,2723 @@
+Changes in version 0.4.2.2-alpha - 2019-10-07
+  This release fixes several bugs from the previous alpha release, and
+  from earlier versions. It also includes a change in authorities, so
+  that they begin to reject the currently unsupported release series.
+
+  o Major features (directory authorities):
+    - Directory authorities now reject relays running all currently
+      deprecated release series. The currently supported release series
+      are: 0.2.9, 0.3.5, 0.4.0, 0.4.1, and 0.4.2. Closes ticket 31549.
+
+  o Major bugfixes (embedded Tor):
+    - Avoid a possible crash when restarting Tor in embedded mode and
+      enabling a different set of publish/subscribe messages. Fixes bug
+      31898; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (torrc parsing):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any
+      config options (but does contain comments or whitespace). Fixes
+      bug 31408; bugfix on 0.3.1.1-alpha.
+
+  o Minor features (auto-formatting scripts):
+    - When annotating C macros, never generate a line that our check-
+      spaces script would reject. Closes ticket 31759.
+    - When annotating C macros, try to remove cases of double-negation.
+      Closes ticket 31779.
+
+  o Minor features (continuous integration):
+    - When building on Appveyor and Travis, pass the "-k" flag to make,
+      so that we are informed of all compilation failures, not just the
+      first one or two. Closes ticket 31372.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2
+      Country database. Closes ticket 31931.
+
+  o Minor features (maintenance scripts):
+    - Add a Coccinelle script to detect bugs caused by incrementing or
+      decrementing a variable inside a call to log_debug(). Since
+      log_debug() is a macro whose arguments are conditionally
+      evaluated, it is usually an error to do this. One such bug was
+      30628, in which SENDME cells were miscounted by a decrement
+      operator inside a log_debug() call. Closes ticket 30743.
+
+  o Minor features (onion services v3):
+    - Assist users who try to setup v2 client authorization in v3 onion
+      services by pointing them to the right documentation. Closes
+      ticket 28966.
+
+  o Minor bugfixes (Appveyor continuous integration):
+    - Avoid spurious errors when Appveyor CI fails before the install
+      step. Fixes bug 31884; bugfix on 0.3.4.2-alpha.
+
+  o Minor bugfixes (best practices tracker):
+    - When listing overbroad exceptions, do not also list problems, and
+      do not list insufficiently broad exceptions. Fixes bug 31338;
+      bugfix on 0.4.2.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix the MAPADDRESS controller command to accept one or more
+      arguments. Previously, it required two or more arguments, and
+      ignored the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Add a missing check for HAVE_PTHREAD_H, because the backtrace code
+      uses mutexes. Fixes bug 31614; bugfix on 0.2.5.2-alpha.
+    - Disable backtrace signal handlers when shutting down tor. Fixes
+      bug 31614; bugfix on 0.2.5.2-alpha.
+    - Rate-limit our the logging message about the obsolete .exit
+      notation. Previously, there was no limit on this warning, which
+      could potentially be triggered many times by a hostile website.
+      Fixes bug 31466; bugfix on 0.2.2.1-alpha.
+    - When initialising log domain masks, only set known log domains.
+      Fixes bug 31854; bugfix on 0.2.1.1-alpha.
+
+  o Minor bugfixes (logging, protocol violations):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (modules):
+    - Explain what the optional Directory Authority module is, and what
+      happens when it is disabled. Fixes bug 31825; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (multithreading):
+    - Avoid some undefined behaviour when freeing mutexes. Fixes bug
+      31736; bugfix on 0.0.7.
+
+  o Minor bugfixes (relay):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug
+      30916; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (tests, SunOS):
+    - Avoid a map_anon_nofork test failure due to a signed/unsigned
+      integer comparison. Fixes bug 31897; bugfix on 0.4.1.1-alpha.
+
+  o Code simplification and refactoring:
+    - Refactor connection_control_process_inbuf() to reduce the size of
+      a practracker exception. Closes ticket 31840.
+    - Refactor the microdescs_parse_from_string() function into smaller
+      pieces, for better comprehensibility. Closes ticket 31675.
+    - Use SEVERITY_MASK_IDX() to find the LOG_* mask indexes in the unit
+      tests and fuzzers, rather than using hard-coded values. Closes
+      ticket 31334.
+    - Interface for function `decrypt_desc_layer` cleaned up. Closes
+      ticket 31589.
+
+  o Documentation:
+    - Document the signal-safe logging behaviour in the tor man page.
+      Also add some comments to the relevant functions. Closes
+      ticket 31839.
+    - Explain why we can't destroy the backtrace buffer mutex. Explain
+      why we don't need to destroy the log mutex. Closes ticket 31736.
+    - The Tor source code repository now includes a (somewhat dated)
+      description of Tor's modular architecture, in doc/HACKING/design.
+      This is based on the old "tor-guts.git" repository, which we are
+      adopting and superseding. Closes ticket 31849.
+
+
+Changes in version 0.4.1.6 - 2019-09-19
+  This release backports several bugfixes to improve stability and
+  correctness.  Anyone experiencing build problems or crashes with 0.4.1.5,
+  or experiencing reliability issues with single onion services, should
+  upgrade.
+
+  o Major bugfixes (crash, Linux, Android, backport from 0.4.2.1-alpha):
+    - Tolerate systems (including some Android installations) where
+      madvise and MADV_DONTDUMP are available at build-time, but not at
+      run time. Previously, these systems would notice a failed syscall
+      and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha.
+    - Tolerate systems (including some Linux installations) where
+      madvise and/or MADV_DONTFORK are available at build-time, but not
+      at run time. Previously, these systems would notice a failed
+      syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Add more stub functions to fix compilation on Android with link-
+      time optimization when --disable-module-dirauth is used.
+      Previously, these compilation settings would make the compiler
+      look for functions that didn't exist. Fixes bug 31552; bugfix
+      on 0.4.1.1-alpha.
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix the MAPADDRESS controller command to accept one or more
+      arguments. Previously, it required two or more arguments, and ignored
+      the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (v3 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v3 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v3 single onion services used a 3-hop
+      path when rend circuits were retried after a remote or delayed
+      failure, but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+
+Changes in version 0.4.2.1-alpha - 2019-09-17
+  This is the first alpha release in the 0.4.2.x series. It adds new
+  defenses for denial-of-service attacks against onion services. It also
+  includes numerous kinds of bugfixes and refactoring to help improve
+  Tor's stability and ease of development.
+
+  o Major features (onion service v3, denial of service):
+    - Add onion service introduction denial of service defenses. Intro
+      points can now rate-limit client introduction requests, using
+      parameters that can be sent by the service within the
+      ESTABLISH_INTRO cell. If the cell extension for this is not used,
+      the intro point will honor the consensus parameters. Closes
+      ticket 30924.
+
+  o Major bugfixes (circuit build, guard):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close.
+      Previously we could end up in the situation where a subsystem is
+      notified of a circuit opening, but the circuit is still marked for
+      close, leading to undesirable behavior. Fixes bug 30871; bugfix
+      on 0.3.0.1-alpha.
+
+  o Major bugfixes (crash, Linux, Android):
+    - Tolerate systems (including some Android installations) where
+      madvise and MADV_DONTDUMP are available at build-time, but not at
+      run time. Previously, these systems would notice a failed syscall
+      and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha.
+    - Tolerate systems (including some Linux installations) where
+      madvise and/or MADV_DONTFORK are available at build-time, but not
+      at run time. Previously, these systems would notice a failed
+      syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha.
+
+  o Minor features (best practices tracker):
+    - Our best-practices tracker now integrates with our include-checker
+      tool to keep track of how many layering violations we have not yet
+      fixed. We hope to reduce this number over time to improve Tor's
+      modularity. Closes ticket 31176.
+    - Add a TOR_PRACTRACKER_OPTIONS variable for passing arguments to
+      practracker from the environment. We may want this for continuous
+      integration. Closes ticket 31309.
+    - Give a warning rather than an error when a practracker exception
+      is violated by a small amount, add a --list-overbroad option to
+      practracker that lists exceptions that are stricter than they need
+      to be, and provide an environment variable for disabling
+      practracker. Closes ticket 30752.
+    - Our best-practices tracker now looks at headers as well as C
+      files. Closes ticket 31175.
+
+  o Minor features (build system):
+    - Add --disable-manpage and --disable-html-manual options to
+      configure script. This will enable shortening build times by not
+      building documentation. Resolves issue 19381.
+
+  o Minor features (compilation):
+    - Log a more useful error message when we are compiling and one of
+      the compile-time hardening options we have selected can be linked
+      but not executed. Closes ticket 27530.
+
+  o Minor features (configuration):
+    - The configuration code has been extended to allow splitting
+      configuration data across multiple objects. Previously, all
+      configuration data needed to be kept in a single object, which
+      tended to become bloated. Closes ticket 31240.
+
+  o Minor features (continuous integration):
+    - When running CI builds on Travis, put some random data in
+      ~/.torrc, to make sure no tests are reading the Tor configuration
+      file from its default location. Resolves issue 30102.
+
+  o Minor features (debugging):
+    - Log a nonfatal assertion failure if we encounter a configuration
+      line whose command is "CLEAR" but which has a nonempty value. This
+      should be impossible, according to the rules of our configuration
+      line parsing. Closes ticket 31529.
+
+  o Minor features (git hooks):
+    - Our pre-commit git hook now checks for a special file before
+      running practracker, so that practracker only runs on branches
+      that are based on master. Since the pre-push hook calls the pre-
+      commit hook, practracker will also only run before pushes of
+      branches based on master. Closes ticket 30979.
+
+  o Minor features (git scripts):
+    - Add a "--" command-line argument, to separate git-push-all.sh
+      script arguments from arguments that are passed through to git
+      push. Closes ticket 31314.
+    - Add a -r <remote-name> argument to git-push-all.sh, so the script
+      can push test branches to a personal remote. Closes ticket 31314.
+    - Add a -t <test-branch-prefix> argument to git-merge-forward.sh and
+      git-push-all.sh, which makes these scripts create, merge forward,
+      and push test branches. Closes ticket 31314.
+    - Add a -u argument to git-merge-forward.sh, so that the script can
+      re-use existing test branches after a merge failure and fix.
+      Closes ticket 31314.
+    - Add a TOR_GIT_PUSH env var, which sets the default git push
+      command and arguments for git-push-all.sh. Closes ticket 31314.
+    - Add a TOR_PUSH_DELAY variable to git-push-all.sh, which makes the
+      script push master and maint branches with a delay between each
+      branch. These delays trigger the CI jobs in a set order, which
+      should show the most likely failures first. Also make pushes
+      atomic by default, and make the script pass any command-line
+      arguments to git push. Closes ticket 29879.
+    - Call the shellcheck script from the pre-commit hook. Closes
+      ticket 30967.
+    - Skip pushing test branches that are the same as a remote
+      maint/release/master branch in git-push-all.sh by default. Add a
+      -s argument, so git-push-all.sh can push all test branches. Closes
+      ticket 31314.
+
+  o Minor features (IPv6, logging):
+    - Log IPv6 addresses as well as IPv4 addresses when describing
+      routerinfos, routerstatuses, and nodes. Closes ticket 21003.
+
+  o Minor features (onion service v3):
+    - Do not allow single hop clients to fetch or post an HS descriptor
+      from an HSDir. Closes ticket 24964.
+
+  o Minor features (onion service):
+    - Disallow single-hop clients at the introduction point. We've
+      removed Tor2web support a while back and single-hop rendezvous
+      attempts are blocked at the relays. This change should remove load
+      off the network from spammy clients. Close ticket 24963.
+
+  o Minor features (stem tests):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor features (testing):
+    - Add a script to invoke "tor --dump-config" and "tor
+      --verify-config" with various configuration options, and see
+      whether tor's resulting configuration or error messages are what
+      we expect. Use it for integration testing of our +Option and
+      /Option flags. Closes ticket 31637.
+    - Improve test coverage for our existing configuration parsing and
+      management API. Closes ticket 30893.
+    - Add integration tests to make sure that practracker gives the
+      outputs we expect. Closes ticket 31477.
+    - The practracker self-tests are now run as part of the Tor test
+      suite. Closes ticket 31304.
+
+  o Minor features (token bucket):
+    - Implement a generic token bucket that uses a single counter, for
+      use in anti-DoS onion service work. Closes ticket 30687.
+
+  o Minor bugfixes (best practices tracker):
+    - Fix a few issues in the best-practices script, including tests,
+      tab tolerance, error reporting, and directory-exclusion logic.
+      Fixes bug 29746; bugfix on 0.4.1.1-alpha.
+    - When running check-best-practices, only consider files in the src
+      subdirectory. Previously we had recursively considered all
+      subdirectories, which made us get confused by the temporary
+      directories made by "make distcheck". Fixes bug 31578; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (build system):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (chutney, makefiles, documentation):
+    - "make test-network-all" now shows the warnings from each test-
+      network.sh run on the console, so developers see new warnings
+      early. We've also improved the documentation for this feature, and
+      renamed a Makefile variable so the code is self-documenting. Fixes
+      bug 30455; bugfix on 0.3.0.4-rc.
+
+  o Minor bugfixes (compilation):
+    - Add more stub functions to fix compilation on Android with link-
+      time optimization when --disable-module-dirauth is used.
+      Previously, these compilation settings would make the compiler
+      look for functions that didn't exist. Fixes bug 31552; bugfix
+      on 0.4.1.1-alpha.
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (configuration):
+    - Invalid floating-point values in the configuration file are now
+      treated as errors in the configuration. Previously, they were
+      ignored and treated as zero. Fixes bug 31475; bugfix on 0.0.1.
+
+  o Minor bugfixes (coverity):
+    - Add an assertion when parsing a BEGIN cell so that coverity can be
+      sure that we are not about to dereference a NULL address. Fixes
+      bug 31026; bugfix on 0.2.4.7-alpha. This is CID 1447296.
+    - In our siphash implementation, when building for coverity, use
+      memcpy in place of a switch statement, so that coverity can tell
+      we are not accessing out-of-bounds memory. Fixes bug 31025; bugfix
+      on 0.2.8.1-alpha. This is tracked as CID 1447293 and 1447295.
+    - Fix several coverity warnings from our unit tests. Fixes bug
+      31030; bugfix on 0.2.4.1-alpha, 0.3.2.1-alpha, and 0.4.0.1-alpha.
+
+  o Minor bugfixes (developer tooling):
+    - Only log git script changes in the post-merge script when the
+      merge was to the master branch. Fixes bug 31040; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Return a distinct status when formatting annotations fails. Fixes
+      bug 30780; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (error handling):
+    - On abort, try harder to flush the output buffers of log messages.
+      On some platforms (macOS), log messages could be discarded when
+      the process terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - Report the tor version whenever an assertion fails. Previously, we
+      only reported the Tor version on some crashes, and some non-fatal
+      assertions. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
+    - When tor aborts due to an error, close log file descriptors before
+      aborting. Closing the logs makes some OSes flush log file buffers,
+      rather than deleting buffered log lines. Fixes bug 31594; bugfix
+      on 0.2.5.2-alpha.
+
+  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6):
+    - When extracting an IPv6 address from a PF-based proxy, verify that
+      we are actually configured to receive an IPv6 address, and log an
+      internal error if not. Fixes part of bug 31687; bugfix
+      on 0.2.3.4-alpha.
+
+  o Minor bugfixes (git hooks):
+    - Remove a duplicate call to practracker from the pre-push hook. The
+      pre-push hook already calls the pre-commit hook, which calls
+      practracker. Fixes bug 31462; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (git scripts):
+    - Stop hard-coding the bash path in the git scripts. Some OSes don't
+      have bash in /usr/bin, others have an ancient bash at this path.
+      Fixes bug 30840; bugfix on 0.4.0.1-alpha.
+    - Stop hard-coding the tor master branch name and worktree path in
+      the git scripts. Fixes bug 30841; bugfix on 0.4.0.1-alpha.
+    - Allow git-push-all.sh to be run from any directory. Previously,
+      the script only worked if run from an upstream worktree directory.
+      Closes ticket 31678.
+
+  o Minor bugfixes (guards):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (ipv6):
+    - Check for private IPv6 addresses alongside their IPv4 equivalents
+      when authorities check descriptors. Previously, we only checked
+      for private IPv4 addresses. Fixes bug 31088; bugfix on
+      0.2.3.21-rc. Patch by Neel Chauhan.
+    - When parsing microdescriptors, we should check the IPv6 exit
+      policy alongside IPv4. Previously, we checked both exit policies
+      for only router info structures, while microdescriptors were
+      IPv4-only. Fixes bug 27284; bugfix on 0.2.3.1-alpha. Patch by
+      Neel Chauhan.
+
+  o Minor bugfixes (logging):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+    - Fix a code issue that would have broken our parsing of log domains
+      as soon as we had 33 of them. Fortunately, we still only have 29.
+      Fixes bug 31451; bugfix on 0.4.1.4-rc.
+
+  o Minor bugfixes (memory management):
+    - Stop leaking a small amount of memory in nt_service_install(), in
+      unreachable code. Fixes bug 30799; bugfix on 0.2.0.7-alpha. Patch
+      by Xiaoyin Liu.
+
+  o Minor bugfixes (networking, IP addresses):
+    - When parsing addresses via Tor's internal DNS lookup API, reject
+      IPv4 addresses in square brackets, and accept IPv6 addresses in
+      square brackets. This change completes the work started in 23082,
+      making address parsing consistent between tor's internal DNS
+      lookup and address parsing APIs. Fixes bug 30721; bugfix
+      on 0.2.1.5-alpha.
+    - When parsing addresses via Tor's internal address:port parsing and
+      DNS lookup APIs, require IPv6 addresses with ports to have square
+      brackets. But allow IPv6 addresses without ports, whether or not
+      they have square brackets. Fixes bug 30721; bugfix
+      on 0.2.1.5-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - When purging the client descriptor cache, close any introduction
+      point circuits associated with purged cache entries. This avoids
+      picking those circuits later when connecting to the same
+      introduction points. Fixes bug 30921; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - In the hs_ident_circuit_t data structure, remove the unused field
+      circuit_type and the respective argument in hs_ident_circuit_new().
+      This field was set by clients (for introduction) and services (for
+      introduction and rendezvous) but was never used afterwards. Fixes
+      bug 31490; bugfix on 0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (operator tools):
+    - Make tor-print-ed-signing-cert(1) print certificate expiration
+      date in RFC 1123 and UNIX timestamp formats, to make output
+      machine readable. Fixes bug 31012; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (rust):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+    - Raise the minimum rustc version to 1.31.0, as checked by configure
+      and CI. Fixes bug 31442; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (sendme, code structure):
+    - Rename the trunnel SENDME file definition from sendme.trunnel to
+      sendme_cell.trunnel to avoid having twice sendme.{c|h} in the
+      repository. Fixes bug 30769; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (statistics):
+    - Stop removing the ed25519 signature if the extra info file is too
+      big. If the signature data was removed, but the keyword was kept,
+      this could result in an unparseable extra info file. Fixes bug
+      30958; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (subsystems):
+    - Make the subsystem init order match the subsystem module
+      dependencies. Call windows process security APIs as early as
+      possible. Initialize logging before network and time, so that
+      network and time can use logging. Fixes bug 31615; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Teach the util/socketpair_ersatz test to work correctly when we
+      have no network stack configured. Fixes bug 30804; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (v2 single onion services):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (v3 single onion services):
+    - Always retry v3 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v3 single onion services used a 3-hop
+      path when rend circuits were retried after a remote or delayed
+      failure, but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation:
+    - Improve documentation in circuit padding subsystem. Patch by
+      Tobias Pulls. Closes ticket 31113.
+    - Include an example usage for IPv6 ORPort in our sample torrc.
+      Closes ticket 31320; patch from Ali Raheem.
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+  o Removed features:
+    - No longer include recommended package digests in votes as detailed
+      in proposal 301. The RecommendedPackages torrc option is
+      deprecated and will no longer have any effect. "package" lines
+      will still be considered when computing consensuses for consensus
+      methods that include them. (This change has no effect on the list
+      of recommended Tor versions, which is still in use.) Closes
+      ticket 29738.
+    - Remove torctl.in from contrib/dist directory. Resolves
+      ticket 30550.
+
+  o Testing:
+    - Run shellcheck for all non-third-party shell scripts that are
+      shipped with Tor. Closes ticket 29533.
+    - When checking shell scripts, ignore any user-created directories.
+      Closes ticket 30967.
+
+  o Code simplification and refactoring (config handling):
+    - Extract our variable manipulation code from confparse.c to a new
+      lower-level typedvar.h module. Closes ticket 30864.
+    - Lower another layer of object management from confparse.c to a
+      more general tool. Now typed structure members are accessible via
+      an abstract type. Implements ticket 30914.
+    - Move our backend logic for working with configuration and state
+      files into a lower-level library, since it no longer depends on
+      any tor-specific functionality. Closes ticket 31626.
+    - Numerous simplifications in configuration-handling logic: remove
+      duplicated macro definitions, replace magical names with flags,
+      and refactor "TestingTorNetwork" to use the same default-option
+      logic as the rest of Tor. Closes ticket 30935.
+    - Replace our ad-hoc set of flags for configuration variables and
+      configuration variable types with fine-grained orthogonal flags
+      corresponding to the actual behavior we want. Closes ticket 31625.
+
+  o Code simplification and refactoring (misc):
+    - Eliminate some uses of lower-level control reply abstractions,
+      primarily in the onion_helper functions. Closes ticket 30889.
+    - Rework bootstrap tracking to use the new publish-subscribe
+      subsystem. Closes ticket 29976.
+    - Rewrite format_node_description() and router_get_verbose_nickname()
+      to use strlcpy() and strlcat(). The previous implementation used
+      memcpy() and pointer arithmetic, which was error-prone. Closes
+      ticket 31545. This is CID 1452819.
+    - Split extrainfo_dump_to_string() into smaller functions. Closes
+      ticket 30956.
+    - Use the ptrdiff_t type consistently for expressing variable
+      offsets and pointer differences. Previously we incorrectly (but
+      harmlessly) used int and sometimes off_t for these cases. Closes
+      ticket 31532.
+    - Use the subsystems mechanism to manage the main event loop code.
+      Closes ticket 30806.
+    - Various simplifications and minor improvements to the circuit
+      padding machines. Patch by Tobias Pulls. Closes tickets 31112
+      and 31098.
+
+  o Documentation (hard-coded directories):
+    - Improve the documentation for the DirAuthority and FallbackDir
+      torrc options. Closes ticket 30955.
+
+  o Documentation (tor.1 man page):
+    - Fix typo in tor.1 man page: the option is "--help", not "-help".
+      Fixes bug 31008; bugfix on 0.2.2.9-alpha.
+
+
+Changes in version 0.4.1.5 - 2019-08-20
+  This is the first stable release in the 0.4.1.x series. This series
+  adds experimental circuit-level padding, authenticated SENDME cells to
+  defend against certain attacks, and several performance improvements
+  to save on CPU consumption. It fixes bugs in bootstrapping and v3
+  onion services. It also includes numerous smaller features and
+  bugfixes on earlier versions.
+
+  Per our support policy, we will support the 0.4.1.x series for nine
+  months, or until three months after the release of a stable 0.4.2.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.1.4-rc. For a complete list of changes
+  since 0.4.0.5, see the ReleaseNotes file.
+
+  o Directory authority changes:
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Minor features (circuit padding logging):
+    - Demote noisy client-side warn logs about circuit padding to
+      protocol warnings. Add additional log messages and circuit ID
+      fields to help with bug 30992 and any other future issues.
+
+  o Minor bugfixes (circuit padding negotiation):
+    - Bump the circuit padding protocol version to explicitly signify
+      that the HS setup machine support is finalized in 0.4.1.x-stable.
+      This also means that 0.4.1.x-alpha clients will not negotiate
+      padding with 0.4.1.x-stable relays, and 0.4.1.x-stable clients
+      will not negotiate padding with 0.4.1.x-alpha relays (or 0.4.0.x
+      relays). Fixes bug 31356; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (circuit padding):
+    - Ignore non-padding cells on padding circuits. This addresses
+      various warning messages from subsystems that were not expecting
+      padding circuits. Fixes bug 30942; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (clock skew detection):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compatibility, standards compliance):
+    - Fix a bug that would invoke undefined behavior on certain
+      operating systems when trying to asprintf() a string exactly
+      INT_MAX bytes long. We don't believe this is exploitable, but it's
+      better to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha.
+      Found and fixed by Tobias Stoeckmann.
+
+  o Minor bugfixes (compilation warning):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix
+      on 0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds. Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (distribution):
+    - Do not ship any temporary files found in the
+      scripts/maint/practracker directory. Fixes bug 31311; bugfix
+      on 0.4.1.1-alpha.
+
+  o Testing (continuous integration):
+    - In Travis, make stem log a controller trace to the console, and
+      tail stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary. Closes
+      ticket 30694.
+
+
+Changes in version 0.4.1.4-rc - 2019-07-25
+  Tor 0.4.1.4-rc fixes a few bugs from previous versions of Tor, and
+  updates to a new list of fallback directories. If no new bugs are
+  found, the next release in the 0.4.1.x serious should be stable.
+
+  o Major bugfixes (circuit build, guard):
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Minor features (continuous integration):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+
+  o Minor features (fallback directory list):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor bugfixes (circuit padding):
+    - On relays, properly check that a padding machine is absent before
+      logging a warning about it being absent. Fixes bug 30649; bugfix
+      on 0.4.0.1-alpha.
+    - Add two NULL checks in unreachable places to silence Coverity (CID
+      144729 and 1447291) and better future-proof ourselves. Fixes bug
+      31024; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (crash on exit):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (logging):
+    - Fix a conflict between the flag used for messaging-domain log
+      messages, and the LD_NO_MOCK testing flag. Fixes bug 31080; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a trivial memory leak when parsing an invalid value from a
+      download schedule in the configuration. Fixes bug 30894; bugfix
+      on 0.3.4.1-alpha.
+
+  o Code simplification and refactoring:
+    - Remove some dead code from circpad_machine_remove_token() to fix
+      some Coverity warnings (CID 1447298). Fixes bug 31027; bugfix
+      on 0.4.1.1-alpha.
+
+
+Changes in version 0.4.1.3-alpha - 2019-06-25
+  Tor 0.4.1.3-alpha resolves numerous bugs left over from the previous
+  alpha, most of them from earlier release series.
+
+  o Major bugfixes (Onion service reachability):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30852.
+
+  o Minor features (logging):
+    - Give a more useful assertion failure message if we think we have
+      minherit() but we fail to make a region non-inheritable. Give a
+      compile-time warning if our support for minherit() is incomplete.
+      Closes ticket 30686.
+
+  o Minor bugfixes (circuit isolation):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (continuous integration):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - When running as a bridge with pluggable transports, always publish
+      pluggable transport information in our extrainfo descriptor, even
+      if ExtraInfoStatistics is 0. This information is needed by
+      BridgeDB. Fixes bug 30956; bugfix on 0.4.1.1-alpha.
+
+  o Documentation:
+    - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md.
+      Closes ticket 30630.
+
+
+Changes in version 0.4.1.2-alpha - 2019-06-06
+  Tor 0.4.1.2-alpha resolves numerous bugs--some of them from the
+  previous alpha, and some much older. It also contains minor testing
+  improvements, and an improvement to the security of our authenticated
+  SENDME implementation.
+
+  o Major bugfixes (bridges):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (flow control, SENDME):
+    - Decrement the stream-level package window after packaging a cell.
+      Previously, it was done inside a log_debug() call, meaning that if
+      debug logs were not enabled, the decrement would never happen, and
+      thus the window would be out of sync with the other end point.
+      Fixes bug 30628; bugfix on 0.4.1.1-alpha.
+
+  o Major bugfixes (onion service reachability):
+    - Properly clean up the introduction point map and associated state
+      when circuits change purpose from onion service circuits to
+      pathbias, measurement, or other circuit types. This may fix some
+      instances of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor features (authenticated SENDME):
+    - Ensure that there is enough randomness on every circuit to prevent
+      an attacker from successfully predicting the hashes they will need
+      to include in authenticated SENDME cells. At a random interval, if
+      we have not sent randomness already, we now leave some extra space
+      at the end of a cell that we can fill with random bytes. Closes
+      ticket 26846.
+
+  o Minor features (continuous integration):
+    - When running coverage builds on Travis, we now set
+      TOR_TEST_RNG_SEED, to avoid RNG-based coverage differences. Part
+      of ticket 28878.
+
+  o Minor features (maintenance):
+    - Add a new "make autostyle" target that developers can use to apply
+      all automatic Tor style and consistency conversions to the
+      codebase. Closes ticket 30539.
+
+  o Minor features (testing):
+    - The circuitpadding tests now use a reproducible RNG implementation,
+      so that if a test fails, we can learn why. Part of ticket 28878.
+    - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED,
+      to set the RNG seed for tests that use a reproducible RNG. Part of
+      ticket 28878.
+    - When running tests in coverage mode, take additional care to make
+      our coverage deterministic, so that we can accurately track
+      changes in code coverage. Closes ticket 30519.
+
+  o Minor bugfixes (configuration, proxies):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (controller):
+    - POSTDESCRIPTOR requests should work again. Previously, they were
+      broken if a "purpose=" flag was specified. Fixes bug 30580; bugfix
+      on 0.4.1.1-alpha.
+    - Repair the HSFETCH command so that it works again. Previously, it
+      expected a body when it shouldn't have. Fixes bug 30646; bugfix
+      on 0.4.1.1-alpha.
+
+  o Minor bugfixes (developer tooling):
+    - Fix pre-push hook to allow fixup and squash commits when pushing
+      to non-upstream git remote. Fixes bug 30286; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (directory authority):
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (NetBSD):
+    - Fix usage of minherit() on NetBSD and other platforms that define
+      MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug
+      30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell.
+
+  o Minor bugfixes (out-of-memory handler):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (shutdown, libevent, memory safety):
+    - Avoid use-after-free bugs when shutting down, by making sure that
+      we shut down libevent only after shutting down all of its users.
+      We believe these are harmless in practice, since they only occur
+      on the shutdown path, and do not involve any attacker-controlled
+      data. Fixes bug 30629; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (static analysis):
+    - Fix several spurious Coverity warnings about the unit tests, to
+      lower our chances of missing real warnings in the future. Fixes
+      bug 30150; bugfix on 0.3.5.1-alpha and various other Tor versions.
+
+  o Testing:
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+
+Changes in version 0.4.1.1-alpha - 2019-05-22
+  This is the first alpha in the 0.4.1.x series. It introduces
+  lightweight circuit padding to make some onion-service circuits harder
+  to distinguish, includes a new "authenticated SENDME" feature to make
+  certain denial-of-service attacks more difficult, and improves
+  performance in several areas.
+
+  o Major features (circuit padding):
+    - Onion service clients now add padding cells at the start of their
+      INTRODUCE and RENDEZVOUS circuits, to make those circuits' traffic
+      look more like general purpose Exit traffic. The overhead for this
+      is 2 extra cells in each direction for RENDEZVOUS circuits, and 1
+      extra upstream cell and 10 downstream cells for INTRODUCE
+      circuits. This feature is only enabled when also supported by the
+      circuit's middle node. (Clients may specify fixed middle nodes
+      with the MiddleNodes option, and may force-disable this feature
+      with the CircuitPadding option.) Closes ticket 28634.
+
+  o Major features (code organization):
+    - Tor now includes a generic publish-subscribe message-passing
+      subsystem that we can use to organize intermodule dependencies. We
+      hope to use this to reduce dependencies between modules that don't
+      need to be related, and to generally simplify our codebase. Closes
+      ticket 28226.
+
+  o Major features (controller protocol):
+    - Controller commands are now parsed using a generalized parsing
+      subsystem. Previously, each controller command was responsible for
+      parsing its own input, which led to strange inconsistencies.
+      Closes ticket 30091.
+
+  o Major features (flow control):
+    - Implement authenticated SENDMEs as detailed in proposal 289. A
+      SENDME cell now includes the digest of the traffic that it
+      acknowledges, so that once an end point receives the SENDME, it
+      can confirm the other side's knowledge of the previous cells that
+      were sent, and prevent certain types of denial-of-service attacks.
+      This behavior is controlled by two new consensus parameters: see
+      the proposal for more details. Fixes ticket 26288.
+
+  o Major features (performance):
+    - Our node selection algorithm now excludes nodes in linear time.
+      Previously, the algorithm was quadratic, which could slow down
+      heavily used onion services. Closes ticket 30307.
+
+  o Major features (performance, RNG):
+    - Tor now constructs a fast secure pseudorandom number generator for
+      each thread, to use when performance is critical. This PRNG is
+      based on AES-CTR, using a buffering construction similar to
+      libottery and the (newer) OpenBSD arc4random() code. It
+      outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for
+      small outputs. Although we believe it to be cryptographically
+      strong, we are only using it when necessary for performance.
+      Implements tickets 29023 and 29536.
+
+  o Major bugfixes (onion service v3):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (circuit padding):
+    - We now use a fast PRNG when scheduling circuit padding. Part of
+      ticket 28636.
+    - Allow the padding machine designer to pick the edges of their
+      histogram instead of trying to compute them automatically using an
+      exponential formula. Resolves some undefined behavior in the case
+      of small histograms and allows greater flexibility on machine
+      design. Closes ticket 29298; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to hold a circuit open until they
+      are done padding it. Closes ticket 28780.
+
+  o Minor features (compile-time modules):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration):
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Minor features (controller):
+    - Add onion service version 3 support to the HSFETCH command.
+      Previously, only version 2 onion services were supported. Closes
+      ticket 25417. Patch by Neel Chauhan.
+
+  o Minor features (debugging):
+    - Introduce tor_assertf() and tor_assertf_nonfatal() to enable
+      logging of additional information during assert failure. Now we
+      can use format strings to include information for trouble
+      shooting. Resolves ticket 29662.
+
+  o Minor features (defense in depth):
+    - In smartlist_remove_keeporder(), set unused pointers to NULL, in
+      case a bug causes them to be used later. Closes ticket 30176.
+      Patch from Tobias Stoeckmann.
+    - Tor now uses a cryptographically strong PRNG even for decisions
+      that we do not believe are security-sensitive. Previously, for
+      performance reasons, we had used a trivially predictable linear
+      congruential generator algorithm for certain load-balancing and
+      statistical sampling decisions. Now we use our fast RNG in those
+      cases. Closes ticket 29542.
+
+  o Minor features (developer tools):
+    - Tor's "practracker" test script now checks for files and functions
+      that seem too long and complicated. Existing overlong functions
+      and files are accepted for now, but should eventually be
+      refactored. Closes ticket 29221.
+    - Add some scripts used for git maintenance to scripts/git. Closes
+      ticket 29391.
+    - Call practracker from pre-push and pre-commit git hooks to let
+      developers know if they made any code style violations. Closes
+      ticket 30051.
+    - Add a script to check that each header has a well-formed and
+      unique guard macro. Closes ticket 29756.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30522.
+
+  o Minor features (HTTP tunnel):
+    - Return an informative web page when the HTTPTunnelPort is used as
+      an HTTP proxy. Closes ticket 27821, patch by "eighthave".
+
+  o Minor features (IPv6, v3 onion services):
+    - Make v3 onion services put IPv6 addresses in service descriptors.
+      Before this change, service descriptors only contained IPv4
+      addresses. Implements 26992.
+
+  o Minor features (modularity):
+    - The "--disable-module-dirauth" compile-time option now disables
+      even more dirauth-only code. Closes ticket 30345.
+
+  o Minor features (performance):
+    - Use OpenSSL's implementations of SHA3 when available (in OpenSSL
+      1.1.1 and later), since they tend to be faster than tiny-keccak.
+      Closes ticket 28837.
+
+  o Minor features (testing):
+    - Tor's unit test code now contains helper functions to replace the
+      PRNG with a deterministic or reproducible version for testing.
+      Previously, various tests implemented this in various ways.
+      Implements ticket 29732.
+    - We now have a script, cov-test-determinism.sh, to identify places
+      where our unit test coverage has become nondeterministic. Closes
+      ticket 29436.
+    - Check that representative subsets of values of `int` and `unsigned
+      int` can be represented by `void *`. Resolves issue 29537.
+
+  o Minor bugfixes (bridge authority):
+    - Bridge authorities now set bridges as running or non-running when
+      about to dump their status to a file. Previously, they set bridges
+      as running in response to a GETINFO command, but those shouldn't
+      modify data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (channel padding statistics):
+    - Channel padding write totals and padding-enabled totals are now
+      counted properly in relay extrainfo descriptors. Fixes bug 29231;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (circuit padding):
+    - Add a "CircuitPadding" torrc option to disable circuit padding.
+      Fixes bug 28693; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to specify that they do not
+      contribute much overhead, and provide consensus flags and torrc
+      options to force clients to only use these low overhead machines.
+      Fixes bug 29203; bugfix on 0.4.0.1-alpha.
+    - Provide a consensus parameter to fully disable circuit padding, to
+      be used in emergency network overload situations. Fixes bug 30173;
+      bugfix on 0.4.0.1-alpha.
+    - The circuit padding subsystem will no longer schedule padding if
+      dormant mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha.
+    - Inspect a circuit-level cell queue before sending padding, to
+      avoid sending padding while too much data is already queued. Fixes
+      bug 29204; bugfix on 0.4.0.1-alpha.
+    - Avoid calling monotime_absolute_usec() in circuit padding machines
+      that do not use token removal or circuit RTT estimation. Fixes bug
+      29085; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Teach the controller parser to distinguish an object preceded by
+      an argument list from one without. Previously, it couldn't
+      distinguish an argument list from the first line of a multiline
+      object. Fixes bug 29984; bugfix on 0.2.3.8-alpha.
+
+  o Minor bugfixes (directory authority, ipv6):
+    - Directory authorities with IPv6 support now always mark themselves
+      as reachable via IPv6. Fixes bug 24338; bugfix on 0.2.4.1-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (documentation):
+    - Improve the documentation for using MapAddress with ".exit". Fixes
+      bug 30109; bugfix on 0.1.0.1-rc.
+    - Improve the monotonic time module and function documentation to
+      explain what "monotonic" actually means, and document some results
+      that have surprised people. Fixes bug 29640; bugfix
+      on 0.2.9.1-alpha.
+    - Use proper formatting when providing an example on quoting options
+      that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc.
+
+  o Minor bugfixes (logging):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+    - Warn operators when the MyFamily option is set but ContactInfo is
+      missing, as the latter should be set too. Fixes bug 25110; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leak):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (onion services):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+    - When refusing to launch a controller's HSFETCH request because of
+      rate-limiting, respond to the controller with a new response,
+      "QUERY_RATE_LIMITED". Previously, we would log QUERY_NO_HSDIR for
+      this case. Fixes bug 28269; bugfix on 0.3.1.1-alpha. Patch by
+      Neel Chauhan.
+    - When relaunching a circuit to a rendezvous service, mark the
+      circuit as needing high-uptime routers as appropriate. Fixes bug
+      17357; bugfix on 0.1.0.1-rc. Patch by Neel Chauhan.
+    - Stop ignoring IPv6 link specifiers sent to v3 onion services.
+      (IPv6 support for v3 onion services is still incomplete: see
+      ticket 23493 for details.) Fixes bug 23588; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (onion services, performance):
+    - When building circuits to onion services, call tor_addr_parse()
+      less often. Previously, we called tor_addr_parse() in
+      circuit_is_acceptable() even if its output wasn't used. This
+      change should improve performance when building circuits. Fixes
+      bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan.
+
+  o Minor bugfixes (performance):
+    - When checking whether a node is a bridge, use a fast check to make
+      sure that its identity is set. Previously, we used a constant-time
+      check, which is not necessary in this case. Fixes bug 30308;
+      bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as
+      well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (probability distributions):
+    - Refactor and improve parts of the probability distribution code
+      that made Coverity complain. Fixes bug 29805; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (python):
+    - Stop assuming that /usr/bin/python3 exists. For scripts that work
+      with python2, use /usr/bin/python. Otherwise, use /usr/bin/env
+      python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha.
+
+  o Minor bugfixes (relay):
+    - When running as a relay, if IPv6Exit is set to 1 while ExitRelay
+      is auto, act as if ExitRelay is 1. Previously, we would ignore
+      IPv6Exit if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on
+      0.3.5.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including bandwidth usage
+      statistics, GeoIPFile hashes, ServerTransportPlugin lines, and
+      bridge statistics by country in extra-info documents. Fixes bug
+      29018; bugfix on 0.2.4.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Call setrlimit() to disable core dumps in test_bt_cl.c. Previously
+      we used `ulimit -c` in test_bt.sh, which violates POSIX shell
+      compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha.
+    - Fix some incorrect code in the v3 onion service unit tests. Fixes
+      bug 29243; bugfix on 0.3.2.1-alpha.
+    - In the "routerkeys/*" tests, check the return values of mkdir()
+      for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha.
+      Found by Coverity as CID 1444254.
+    - Split test_utils_general() into several smaller test functions.
+      This makes it easier to perform resource deallocation on assert
+      failure, and fixes Coverity warnings CID 1444117 and CID 1444118.
+      Fixes bug 29823; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - Fix a memory leak in tor-resolve that could happen if Tor gave it
+      a malformed SOCKS response. (Memory leaks in tor-resolve don't
+      actually matter, but it's good to fix them anyway.) Fixes bug
+      30151; bugfix on 0.4.0.1-alpha.
+
+  o Code simplification and refactoring:
+    - Abstract out the low-level formatting of replies on the control
+      port. Implements ticket 30007.
+    - Add several assertions in an attempt to fix some Coverity
+      warnings. Closes ticket 30149.
+    - Introduce a connection_dir_buf_add() helper function that checks
+      for compress_state of dir_connection_t and automatically writes a
+      string to directory connection with or without compression.
+      Resolves issue 28816.
+    - Make the base32_decode() API return the number of bytes written,
+      for consistency with base64_decode(). Closes ticket 28913.
+    - Move most relay-only periodic events out of mainloop.c into the
+      relay subsystem. Closes ticket 30414.
+    - Refactor and encapsulate parts of the codebase that manipulate
+      crypt_path_t objects. Resolves issue 30236.
+    - Refactor several places in our code that Coverity incorrectly
+      believed might have memory leaks. Closes ticket 30147.
+    - Remove redundant return values in crypto_format, and the
+      associated return value checks elsewhere in the code. Make the
+      implementations in crypto_format consistent, and remove redundant
+      code. Resolves ticket 29660.
+    - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that
+      it is not a constant-time function. Closes ticket 30309.
+    - Replace hs_desc_link_specifier_t with link_specifier_t, and remove
+      all hs_desc_link_specifier_t-specific code. Fixes bug 22781;
+      bugfix on 0.3.2.1-alpha.
+    - Simplify v3 onion service link specifier handling code. Fixes bug
+      23576; bugfix on 0.3.2.1-alpha.
+    - Split crypto_digest.c into NSS code, OpenSSL code, and shared
+      code. Resolves ticket 29108.
+    - Split control.c into several submodules, in preparation for
+      distributing its current responsibilities throughout the codebase.
+      Closes ticket 29894.
+    - Start to move responsibility for knowing about periodic events to
+      the appropriate subsystems, so that the mainloop doesn't need to
+      know all the periodic events in the rest of the codebase.
+      Implements tickets 30293 and 30294.
+
+  o Documentation:
+    - Document how to find git commits and tags for bug fixes in
+      CodingStandards.md. Update some file documentation. Closes
+      ticket 30261.
+
+  o Removed features:
+    - Remove the linux-tor-prio.sh script from contrib/operator-tools
+      directory. Resolves issue 29434.
+    - Remove the obsolete OpenSUSE initscript. Resolves issue 30076.
+    - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves
+      issue 30075.
+
+  o Code simplification and refactoring (shell scripts):
+    - Clean up many of our shell scripts to fix shellcheck warnings.
+      These include autogen.sh (ticket 26069), test_keygen.sh (ticket
+      29062), test_switch_id.sh (ticket 29065), test_rebind.sh (ticket
+      29063), src/test/fuzz/minimize.sh (ticket 30079), test_rust.sh
+      (ticket 29064), torify (ticket 29070), asciidoc-helper.sh (29926),
+      fuzz_multi.sh (30077), fuzz_static_testcases.sh (ticket 29059),
+      nagios-check-tor-authority-cert (ticket 29071),
+      src/test/fuzz/fixup_filenames.sh (ticket 30078), test-network.sh
+      (ticket 29060), test_key_expiration.sh (ticket 30002),
+      zero_length_keys.sh (ticket 29068), and test_workqueue_*.sh
+      (ticket 29067).
+
+  o Testing (chutney):
+    - In "make test-network-all", test IPv6-only v3 single onion
+      services, using the chutney network single-onion-v23-ipv6-md.
+      Closes ticket 27251.
+
+
+Changes in version 0.4.0.5 - 2019-05-02
+  This is the first stable release in the 0.4.0.x series. It contains
+  improvements for power management and bootstrap reporting, as well as
+  preliminary backend support for circuit padding to prevent some kinds
+  of traffic analysis. It also continues our work in refactoring Tor for
+  long-term maintainability.
+
+  Per our support policy, we will support the 0.4.0.x series for nine
+  months, or until three months after the release of a stable 0.4.1.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.0.4-rc. For a complete list of changes
+  since 0.3.5.7, see the ReleaseNotes file.
+
+  o Minor features (continuous integration):
+    - In Travis, tell timelimit to use stem's backtrace signals, and
+      launch python directly from timelimit, so python receives the
+      signals from timelimit, rather than make. Closes ticket 30117.
+
+  o Minor features (diagnostic):
+    - Add more diagnostic log messages in an attempt to solve the issue
+      of NUL bytes appearing in a microdescriptor cache. Related to
+      ticket 28223.
+
+  o Minor features (testing):
+    - Use the approx_time() function when setting the "Expires" header
+      in directory replies, to make them more testable. Needed for
+      ticket 30001.
+
+  o Minor bugfixes (rust):
+    - Abort on panic in all build profiles, instead of potentially
+      unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (shellcheck):
+    - Look for scripts in their correct locations during "make
+      shellcheck". Previously we had looked in the wrong place during
+      out-of-tree builds. Fixes bug 30263; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Check the time in the "Expires" header using approx_time(). Fixes
+      bug 30001; bugfix on 0.4.0.4-rc.
+
+  o Minor bugfixes (UI):
+    - Lower log level of unlink() errors during bootstrap. Fixes bug
+      29930; bugfix on 0.4.0.1-alpha.
+
+
+Changes in version 0.4.0.4-rc - 2019-04-11
+  Tor 0.4.0.4-rc is the first release candidate in its series; it fixes
+  several bugs from earlier versions, including some that had affected
+  stability, and one that prevented relays from working with NSS.
+
+  o Major bugfixes (NSS, relay):
+    - When running with NSS, disable TLS 1.2 ciphersuites that use
+      SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for
+      these ciphersuites don't work -- which caused relays to fail to
+      handshake with one another when these ciphersuites were enabled.
+      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
+
+  o Minor features (bandwidth authority):
+    - Make bandwidth authorities ignore relays that are reported in the
+      bandwidth file with the flag "vote=0". This change allows us to
+      report unmeasured relays for diagnostic reasons without including
+      their bandwidth in the bandwidth authorities' vote. Closes
+      ticket 29806.
+    - When a directory authority is using a bandwidth file to obtain the
+      bandwidth values that will be included in the next vote, serve
+      this bandwidth file at /tor/status-vote/next/bandwidth. Closes
+      ticket 21377.
+
+  o Minor features (circuit padding):
+    - Stop warning about undefined behavior in the probability
+      distribution tests. Float division by zero may technically be
+      undefined behavior in C, but it's well defined in IEEE 754.
+      Partial backport of 29298. Closes ticket 29527; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor features (continuous integration):
+    - On Travis Rust builds, cleanup Rust registry and refrain from
+      caching the "target/" directory to speed up builds. Resolves
+      issue 29962.
+
+  o Minor features (dormant mode):
+    - Add a DormantCanceledByStartup option to tell Tor that it should
+      treat a startup event as cancelling any previous dormant state.
+      Integrators should use this option with caution: it should only be
+      used if Tor is being started because of something that the user
+      did, and not if Tor is being automatically started in the
+      background. Closes ticket 29357.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29992.
+
+  o Minor features (NSS, diagnostic):
+    - Try to log an error from NSS (if there is any) and a more useful
+      description of our situation if we are using NSS and a call to
+      SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241.
+
+  o Minor bugfixes (security):
+    - Fix a potential double free bug when reading huge bandwidth files.
+      The issue is not exploitable in the current Tor network because
+      the vulnerable code is only reached when directory authorities
+      read bandwidth files, but bandwidth files come from a trusted
+      source (usually the authorities themselves). Furthermore, the
+      issue is only exploitable in rare (non-POSIX) 32-bit architectures,
+      which are not used by any of the current authorities. Fixes bug
+      30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by
+      Tobias Stoeckmann.
+    - Verify in more places that we are not about to create a buffer
+      with more than INT_MAX bytes, to avoid possible OOB access in the
+      event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and
+      fixed by Tobias Stoeckmann.
+
+  o Minor bugfix (continuous integration):
+    - Reset coverage state on disk after Travis CI has finished. This
+      should prevent future coverage merge errors from causing the test
+      suite for the "process" subsystem to fail. The process subsystem
+      was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix
+      on 0.2.9.15.
+    - Terminate test-stem if it takes more than 9.5 minutes to run.
+      (Travis terminates the job after 10 minutes of no output.)
+      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (bootstrap reporting):
+    - During bootstrap reporting, correctly distinguish pluggable
+      transports from plain proxies. Fixes bug 28925; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug
+      29824; bugfix on 0.3.1.1-alpha. This is Coverity warning
+      CID 1444119.
+
+  o Minor bugfixes (circuitpadding testing):
+    - Minor tweaks to avoid rare test failures related to timers and
+      monotonic time. Fixes bug 29500; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Actually include the bandwidth-file-digest line in directory
+      authority votes. Fixes bug 29959; bugfix on 0.4.0.2-alpha.
+
+  o Minor bugfixes (logging):
+    - On Windows, when errors cause us to reload a consensus from disk,
+      tell the user that we are retrying at log level "notice".
+      Previously we only logged this information at "info", which was
+      confusing because the errors themselves were logged at "warning".
+      Improves previous fix for 28614. Fixes bug 30004; bugfix
+      on 0.4.0.2-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Restore old behavior when it comes to discovering the path of a
+      given Pluggable Transport executable file. A change in
+      0.4.0.1-alpha had broken this behavior on paths containing a
+      space. Fixes bug 29874; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a
+      recent test-network.sh to use new chutney features in CI. Fixes
+      bug 29703; bugfix on 0.2.9.1-alpha.
+    - Fix a test failure on Windows caused by an unexpected "BUG"
+      warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix
+      on 0.2.9.3-alpha.
+
+  o Minor bugfixes (TLS protocol):
+    - When classifying a client's selection of TLS ciphers, if the
+      client ciphers are not yet available, do not cache the result.
+      Previously, we had cached the unavailability of the cipher list
+      and never looked again, which in turn led us to assume that the
+      client only supported the ancient V1 link protocol. This, in turn,
+      was causing Stem integration tests to stall in some cases. Fixes
+      bug 30021; bugfix on 0.2.4.8-alpha.
+
+  o Code simplification and refactoring:
+    - Introduce a connection_dir_buf_add() helper function that detects
+      whether compression is in use, and adds a string accordingly.
+      Resolves issue 28816.
+    - Refactor handle_get_next_bandwidth() to use
+      connection_dir_buf_add(). Implements ticket 29897.
+
+  o Documentation:
+    - Clarify that Tor performs stream isolation among *Port listeners
+      by default. Resolves issue 29121.
+
+
+Changes in version 0.4.0.3-alpha - 2019-03-22
+  Tor 0.4.0.3-alpha is the third in its series; it fixes several small
+  bugs from earlier versions.
+
+  o Minor features (address selection):
+    - Treat the subnet 100.64.0.0/10 as public for some purposes;
+      private for others. This subnet is the RFC 6598 (Carrier Grade
+      NAT) IP range, and is deployed by many ISPs as an alternative to
+      RFC 1918 that does not break existing internal networks. Tor now
+      blocks SOCKS and control ports on these addresses and warns users
+      if client ports or ExtORPorts are listening on a RFC 6598 address.
+      Closes ticket 28525. Patch by Neel Chauhan.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the March 4 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29666.
+
+  o Minor bugfixes (circuitpadding):
+    - Inspect the circuit-level cell queue before sending padding, to
+      avoid sending padding when too much data is queued. Fixes bug
+      29204; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Correct a misleading error message when IPv4Only or IPv6Only is
+      used but the resolved address can not be interpreted as an address
+      of the specified IP version. Fixes bug 13221; bugfix on
+      0.2.3.9-alpha. Patch from Kris Katterjohn.
+    - Log the correct port number for listening sockets when "auto" is
+      used to let Tor pick the port number. Previously, port 0 was
+      logged instead of the actual port number. Fixes bug 29144; bugfix
+      on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+    - Stop logging a BUG() warning when Tor is waiting for exit
+      descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (memory management):
+    - Refactor the shared random state's memory management so that it
+      actually takes ownership of the shared random value pointers.
+      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (memory management, testing):
+    - Stop leaking parts of the shared random state in the shared-random
+      unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Fix an assertion failure crash bug when a pluggable transport is
+      terminated during the bootstrap phase. Fixes bug 29562; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (Rust, protover):
+    - Add a missing "Padding" value to the Rust implementation of
+      protover. Fixes bug 29631; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (single onion services):
+    - Allow connections to single onion services to remain idle without
+      being disconnected. Previously, relays acting as rendezvous points
+      for single onion services were mistakenly closing idle rendezvous
+      circuits after 60 seconds, thinking that they were unused
+      directory-fetching circuits that had served their purpose. Fixes
+      bug 29665; bugfix on 0.2.1.26.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Downgrade some LOG_ERR messages in the address/* tests to
+      warnings. The LOG_ERR messages were occurring when we had no
+      configured network. We were failing the unit tests, because we
+      backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug
+      29530; bugfix on 0.3.5.8.
+    - Fix our gcov wrapper script to look for object files at the
+      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
+    - Decrease the false positive rate of stochastic probability
+      distribution tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (Windows, CI):
+    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit
+      Windows Server 2012 R2 job. The remaining 2 jobs still provide
+      coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set
+      fast_finish, so failed jobs terminate the build immediately. Fixes
+      bug 29601; bugfix on 0.3.5.4-alpha.
+
+
+Changes in version 0.3.5.8 - 2019-02-21
+  Tor 0.3.5.8 backports several fixes from later releases, including fixes
+  for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x
+  releases.
+
+  It also includes a fix for a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking, backport from 0.4.0.2-alpha):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth message and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Minor features (compilation, backport from 0.4.0.2-alpha):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor features (testing, backport from 0.4.0.2-alpha):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+
+  o Minor bugfixes (onion service v3, client, backport from 0.4.0.1-alpha):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (IPv6, backport from 0.4.0.1-alpha):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (client, clock skew, backport from 0.4.0.1-alpha):
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.0.1-alpha):
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (documentation, backport from 0.4.0.2-alpha):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (logging, backport from 0.4.0.1-alpha):
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (logging, backport from 0.4.0.2-alpha):
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (misc, backport from 0.4.0.2-alpha):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (tests, directory clients, backport from 0.4.0.1-alpha):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (tests, backport from 0.4.0.2-alpha):
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (usability, backport from 0.4.0.1-alpha):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.11 - 2019-02-21
+  Tor 0.3.4.11 is the third stable release in its series.  It includes
+  a fix for a medium-severity security bug affecting Tor 0.3.2.1-alpha and
+  later. All Tor instances running an affected release should upgrade to
+  0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.3.3.12 - 2019-02-21
+  Tor 0.3.3.12 fixes a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  This release marks the end of support for the Tor 0.3.3.x series. We
+  recommend that users switch to either the Tor 0.3.4 series (supported
+  until at least 10 June 2019), or the Tor 0.3.5 series, which will
+  receive long-term support until at least 1 Feb 2022.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.4.0.2-alpha - 2019-02-21
+  Tor 0.4.0.2-alpha is the second alpha in its series; it fixes several
+  bugs from earlier versions, including several that had broken
+  backward compatibility.
+
+  It also includes a fix for a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth messsage and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (windows, startup):
+    - When reading a consensus file from disk, detect whether it was
+      written in text mode, and re-read it in text mode if so. Always
+      write consensus files in binary mode so that we can map them into
+      memory later. Previously, we had written in text mode, which
+      confused us when we tried to map the file on windows. Fixes bug
+      28614; bugfix on 0.4.0.1-alpha.
+
+  o Minor features (compilation):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (developer tooling):
+    - Check that bugfix versions in changes files look like Tor versions
+      from the versions spec. Warn when bugfixes claim to be on a future
+      release. Closes ticket 27761.
+    - Provide a git pre-commit hook that disallows committing if we have
+      any failures in our code and changelog formatting checks. It is
+      now available in scripts/maint/pre-commit.git-hook. Implements
+      feature 28976.
+
+  o Minor features (directory authority):
+    - When a directory authority is using a bandwidth file to obtain
+      bandwidth values, include the digest of that file in the vote.
+      Closes ticket 26698.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor features (testing):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+
+  o Minor bugfixes (build, compatibility, rust):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix compilation warnings in test_circuitpadding.c. Fixes bug
+      29169; bugfix on 0.4.0.1-alpha.
+    - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug
+      29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (documentation):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (linux seccomp sandbox):
+    - Fix startup crash when experimental sandbox support is enabled.
+      Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber.
+
+  o Minor bugfixes (logging):
+    - Avoid logging that we are relaxing a circuit timeout when that
+      timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha.
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (misc):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (onion services):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (scheduler):
+    - When re-adding channels to the pending list, check the correct
+      channel's sched_heap_idx. This issue has had no effect in mainline
+      Tor, but could have led to bugs down the road in improved versions
+      of our circuit scheduling code. Fixes bug 29508; bugfix
+      on 0.3.2.10.
+
+  o Minor bugfixes (tests):
+    - Fix intermittent failures on an adaptive padding test. Fixes one
+      case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Disable an unstable circuit-padding test that was failing
+      intermittently because of an ill-defined small histogram. Such
+      histograms will be allowed again after 29298 is implemented. Fixes
+      a second case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+
+  o Documentation:
+    - In the manpage entry describing MapAddress torrc setting, use
+      example IP addresses from ranges specified for use in documentation
+      by RFC 5737. Resolves issue 28623.
+
+  o Removed features:
+    - Remove the old check-tor script. Resolves issue 29072.
+
+
+Changes in version 0.4.0.1-alpha - 2019-01-18
+  Tor 0.4.0.1-alpha is the first release in the new 0.4.0.x series. It
+  introduces improved features for power and bandwidth conservation,
+  more accurate reporting of bootstrap progress for user interfaces, and
+  an experimental backend for an exciting new adaptive padding feature.
+  There is also the usual assortment of bugfixes and minor features, all
+  described below.
+
+  o Major features (battery management, client, dormant mode):
+    - When Tor is running as a client, and it is unused for a long time,
+      it can now enter a "dormant" state. When Tor is dormant, it avoids
+      network and CPU activity until it is reawoken either by a user
+      request or by a controller command. For more information, see the
+      configuration options starting with "Dormant". Implements tickets
+      2149 and 28335.
+    - The client's memory of whether it is "dormant", and how long it
+      has spent idle, persists across invocations. Implements
+      ticket 28624.
+    - There is a DormantOnFirstStartup option that integrators can use
+      if they expect that in many cases, Tor will be installed but
+      not used.
+
+  o Major features (bootstrap reporting):
+    - When reporting bootstrap progress, report the first connection
+      uniformly, regardless of whether it's a connection for building
+      application circuits. This allows finer-grained reporting of early
+      progress than previously possible, with the improvements of ticket
+      27169. Closes tickets 27167 and 27103. Addresses ticket 27308.
+    - When reporting bootstrap progress, treat connecting to a proxy or
+      pluggable transport as separate from having successfully used that
+      proxy or pluggable transport to connect to a relay. Closes tickets
+      27100 and 28884.
+
+  o Major features (circuit padding):
+    - Implement preliminary support for the circuit padding portion of
+      Proposal 254. The implementation supports Adaptive Padding (aka
+      WTF-PAD) state machines for use between experimental clients and
+      relays. Support is also provided for APE-style state machines that
+      use probability distributions instead of histograms to specify
+      inter-packet delay. At the moment, Tor does not provide any
+      padding state machines that are used in normal operation: for now,
+      this feature exists solely for experimentation. Closes
+      ticket 28142.
+
+  o Major features (refactoring):
+    - Tor now uses an explicit list of its own subsystems when
+      initializing and shutting down. Previously, these systems were
+      managed implicitly in various places throughout the codebase.
+      (There may still be some subsystems using the old system.) Closes
+      ticket 28330.
+
+  o Minor features (bootstrap reporting):
+    - When reporting bootstrap progress, stop distinguishing between
+      situations where only internal paths are available and situations
+      where external paths are available. Previously, Tor would often
+      erroneously report that it had only internal paths. Closes
+      ticket 27402.
+
+  o Minor features (continuous integration):
+    - Log Python version during each Travis CI job. Resolves
+      issue 28551.
+
+  o Minor features (controller):
+    - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP.
+      Implements ticket 28843.
+
+  o Minor features (developer tooling):
+    - Provide a git hook script to prevent "fixup!" and "squash!"
+      commits from ending up in the master branch, as scripts/main/pre-
+      push.git-hook. Closes ticket 27993.
+
+  o Minor features (directory authority):
+    - Directory authorities support a new consensus algorithm, under
+      which the family lines in microdescriptors are encoded in a
+      canonical form. This change makes family lines more compressible
+      in transit, and on the client. Closes ticket 28266; implements
+      proposal 298.
+
+  o Minor features (directory authority, relay):
+    - Authorities now vote on a "StaleDesc" flag to indicate that a
+      relay's descriptor is so old that the relay should upload again
+      soon. Relays treat this flag as a signal to upload a new
+      descriptor. This flag will eventually let us remove the
+      'published' date from routerstatus entries, and make our consensus
+      diffs much smaller. Closes ticket 26770; implements proposal 293.
+
+  o Minor features (fallback directory mirrors):
+    - Update the fallback whitelist based on operator opt-ins and opt-
+      outs. Closes ticket 24805, patch by Phoul.
+
+  o Minor features (FreeBSD):
+    - On FreeBSD-based systems, warn relay operators if the
+      "net.inet.ip.random_id" sysctl (IP ID randomization) is disabled.
+      Closes ticket 28518.
+
+  o Minor features (HTTP standards compliance):
+    - Stop sending the header "Content-type: application/octet-stream"
+      along with transparently compressed documents: this confused
+      browsers. Closes ticket 28100.
+
+  o Minor features (IPv6):
+    - We add an option ClientAutoIPv6ORPort, to make clients randomly
+      prefer a node's IPv4 or IPv6 ORPort. The random preference is set
+      every time a node is loaded from a new consensus or bridge config.
+      We expect that this option will enable clients to bootstrap more
+      quickly without having to determine whether they support IPv4,
+      IPv6, or both. Closes ticket 27490. Patch by Neel Chauhan.
+    - When using addrs_in_same_network_family(), avoid choosing circuit
+      paths that pass through the same IPv6 subnet more than once.
+      Previously, we only checked IPv4 subnets. Closes ticket 24393.
+      Patch by Neel Chauhan.
+
+  o Minor features (log messages):
+    - Improve log message in v3 onion services that could print out
+      negative revision counters. Closes ticket 27707. Patch
+      by "ffmancera".
+
+  o Minor features (memory usage):
+    - Save memory by storing microdescriptor family lists with a more
+      compact representation. Closes ticket 27359.
+    - Tor clients now use mmap() to read consensus files from disk, so
+      that they no longer need keep the full text of a consensus in
+      memory when parsing it or applying a diff. Closes ticket 27244.
+
+  o Minor features (parsing):
+    - Directory authorities now validate that router descriptors and
+      ExtraInfo documents are in a valid subset of UTF-8, and reject
+      them if they are not. Closes ticket 27367.
+
+  o Minor features (performance):
+    - Cache the results of summarize_protocol_flags(), so that we don't
+      have to parse the same protocol-versions string over and over.
+      This should save us a huge number of malloc calls on startup, and
+      may reduce memory fragmentation with some allocators. Closes
+      ticket 27225.
+    - Remove a needless memset() call from get_token_arguments, thereby
+      speeding up the tokenization of directory objects by about 20%.
+      Closes ticket 28852.
+    - Replace parse_short_policy() with a faster implementation, to
+      improve microdescriptor parsing time. Closes ticket 28853.
+    - Speed up directory parsing a little by avoiding use of the non-
+      inlined strcmp_len() function. Closes ticket 28856.
+    - Speed up microdescriptor parsing by about 30%, to help improve
+      startup time. Closes ticket 28839.
+
+  o Minor features (pluggable transports):
+    - Add support for emitting STATUS updates to Tor's control port from
+      a pluggable transport process. Closes ticket 28846.
+    - Add support for logging to Tor's logging subsystem from a
+      pluggable transport process. Closes ticket 28180.
+
+  o Minor features (process management):
+    - Add a new process API for handling child processes. This new API
+      allows Tor to have bi-directional communication with child
+      processes on both Unix and Windows. Closes ticket 28179.
+    - Use the subsystem manager to initialize and shut down the process
+      module. Closes ticket 28847.
+
+  o Minor features (relay):
+    - When listing relay families, list them in canonical form including
+      the relay's own identity, and try to give a more useful set of
+      warnings. Part of ticket 28266 and proposal 298.
+
+  o Minor features (required protocols):
+    - Before exiting because of a missing required protocol, Tor will
+      now check the publication time of the consensus, and not exit
+      unless the consensus is newer than the Tor program's own release
+      date. Previously, Tor would not check the consensus publication
+      time, and so might exit because of a missing protocol that might
+      no longer be required in a current consensus. Implements proposal
+      297; closes ticket 27735.
+
+  o Minor features (testing):
+    - Allow a HeartbeatPeriod of less than 30 minutes in testing Tor
+      networks. Closes ticket 28840. Patch by Rob Jansen.
+
+  o Minor bugfixes (client, clock skew):
+    - Bootstrap successfully even when Tor's clock is behind the clocks
+      on the authorities. Fixes bug 28591; bugfix on 0.2.0.9-alpha.
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (directory clients):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Even when a directory mirror's clock is behind the clocks on the
+      authorities, we now allow the mirror to serve "future"
+      consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (DNS):
+    - Gracefully handle an empty or absent resolve.conf file by falling
+      back to using "localhost" as a DNS server (and hoping it works).
+      Previously, we would just stop running as an exit. Fixes bug
+      21900; bugfix on 0.2.1.10-alpha.
+
+  o Minor bugfixes (guards):
+    - In count_acceptable_nodes(), the minimum number is now one bridge
+      or guard node, and two non-guard nodes for a circuit. Previously,
+      we had added up the sum of all nodes with a descriptor, but that
+      could cause us to build failing circuits when we had either too
+      many bridges or not enough guard nodes. Fixes bug 25885; bugfix on
+      0.2.3.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (IPv6):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (logging):
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (networking):
+    - Introduce additional checks into tor_addr_parse() to reject
+      certain incorrect inputs that previously were not detected. Fixes
+      bug 23082; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (onion service v3, client):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (periodic events):
+    - Refrain from calling routerlist_remove_old_routers() from
+      check_descriptor_callback(). Instead, create a new hourly periodic
+      event. Fixes bug 27929; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Make sure that data is continously read from standard output and
+      standard error pipes of a pluggable transport child-process, to
+      avoid deadlocking when a pipe's buffer is full. Fixes bug 26360;
+      bugfix on 0.2.3.6-alpha.
+
+  o Minor bugfixes (unit tests):
+    - Instead of relying on hs_free_all() to clean up all onion service
+      objects in test_build_descriptors(), we now deallocate them one by
+      one. This lets Coverity know that we are not leaking memory there
+      and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (usability):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+  o Code simplification and refactoring:
+    - Reimplement NETINFO cell parsing and generation to rely on
+      trunnel-generated wire format handling code. Closes ticket 27325.
+    - Remove unnecessary unsafe code from the Rust macro "cstr!". Closes
+      ticket 28077.
+    - Rework SOCKS wire format handling to rely on trunnel-generated
+      parsing/generation code. Resolves ticket 27620.
+    - Split out bootstrap progress reporting from control.c into a
+      separate file. Part of ticket 27402.
+    - The .may_include files that we use to describe our directory-by-
+      directory dependency structure now describe a noncircular
+      dependency graph over the directories that they cover. Our
+      checkIncludes.py tool now enforces this noncircularity. Closes
+      ticket 28362.
+
+  o Documentation:
+    - Mention that you cannot add a new onion service if Tor is already
+      running with Sandbox enabled. Closes ticket 28560.
+    - Improve ControlPort documentation. Mention that it accepts
+      address:port pairs, and can be used multiple times. Closes
+      ticket 28805.
+    - Document the exact output of "tor --version". Closes ticket 28889.
+
+  o Removed features:
+    - Stop responding to the 'GETINFO status/version/num-concurring' and
+      'GETINFO status/version/num-versioning' control port commands, as
+      those were deprecated back in 0.2.0.30. Also stop listing them in
+      output of 'GETINFO info/names'. Resolves ticket 28757.
+    - The scripts used to generate and maintain the list of fallback
+      directories have been extracted into a new "fallback-scripts"
+      repository. Closes ticket 27914.
+
+  o Testing:
+    - Run shellcheck for scripts in the in scripts/ directory. Closes
+      ticket 28058.
+    - Add unit tests for tokenize_string() and get_next_token()
+      functions. Resolves ticket 27625.
+
+  o Code simplification and refactoring (onion service v3):
+    - Consolidate the authorized client descriptor cookie computation
+      code from client and service into one function. Closes
+      ticket 27549.
+
+  o Code simplification and refactoring (shell scripts):
+    - Cleanup scan-build.sh to silence shellcheck warnings. Closes
+      ticket 28007.
+    - Fix issues that shellcheck found in chutney-git-bisect.sh.
+      Resolves ticket 28006.
+    - Fix issues that shellcheck found in updateRustDependencies.sh.
+      Resolves ticket 28012.
+    - Fix shellcheck warnings in cov-diff script. Resolves issue 28009.
+    - Fix shellcheck warnings in run_calltool.sh. Resolves ticket 28011.
+    - Fix shellcheck warnings in run_trunnel.sh. Resolves issue 28010.
+    - Fix shellcheck warnings in scripts/test/coverage. Resolves
+      issue 28008.
+
+
+Changes in version 0.3.3.11 - 2019-01-07
+  Tor 0.3.3.11 backports numerous fixes from later versions of Tor.
+  numerous fixes, including an important fix for anyone using OpenSSL
+  1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade
+  to this version, or to a later series.
+
+  As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019.
+  We anticipate that this will be the last release of Tor 0.3.3, unless
+  some major bug is before then. Some time between now and then, users
+  should switch to either the Tor 0.3.4 series (supported until at least
+  10 June 2019), or the Tor 0.3.5 series, which will receive long-term
+  support until at least 1 Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.3.5.1-alpha):
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, backport from 0.3.5.3-alpha):
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha):
+    - Avoid undefined behavior in an end-of-string check when parsing
+      the BEGIN line in a directory object. Fixes bug 28202; bugfix
+      on 0.2.0.3-alpha.
+
+  o Minor bugfixes (code safety, backport from 0.3.5.3-alpha):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha):
+    - Log additional info when we get a relay that shares an ed25519 ID
+      with a different relay, instead making a BUG() warning. Fixes bug
+      27800; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha):
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to
+      an HTTP request we couldn't handle. Fixes bug 26470; bugfix on
+      0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (protover, backport from 0.3.5.3-alpha):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.1-alpha):
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.2-alpha):
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (rust, backport from 0.3.5.4-alpha):
+    - Fix a potential null dereference in protover_all_supported(). Add
+      a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
+    - Return a string that can be safely freed by C code, not one
+      created by the rust allocator, in protover_all_supported(). Fixes
+      bug 27740; bugfix on 0.3.3.1-alpha.
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
+
+  o Minor bugfixes (testing, backport from 0.3.5.1-alpha):
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.4-alpha):
+    - Treat backtrace test failures as expected on BSD-derived systems
+      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
+      (FreeBSD failures have been treated as expected since 18204 in
+      0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.10 - 2019-01-07
+  Tor 0.3.4.9 is the second stable release in its series; it backports
+  numerous fixes, including an important fix for relays, and for anyone
+  using OpenSSL 1.1.1. Anyone running an  earlier version of Tor 0.3.4
+  should upgrade.
+
+  As a reminder, the Tor 0.3.4 series will be supported until 10 June
+  2019. Some time between now and then, users should switch to the Tor
+  0.3.5 series, which will receive long-term support until at least 1
+  Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (relay, directory, backport from 0.3.5.7):
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+
+  o Minor features (controller, backport from 0.3.5.1-alpha):
+    - For purposes of CIRC_BW-based dropped cell detection, track half-
+      closed stream ids, and allow their ENDs, SENDMEs, DATA and path
+      bias check cells to arrive without counting it as dropped until
+      either the END arrives, or the windows are empty. Closes
+      ticket 25573.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28395.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.5.7 - 2019-01-07
+  Tor 0.3.5.7 is the first stable release in its series; it includes
+  compilation and portability fixes, and a fix for a severe problem
+  affecting directory caches.
+
+  The Tor 0.3.5 series includes several new features and performance
+  improvements, including client authorization for v3 onion services,
+  cleanups to bootstrap reporting, support for improved bandwidth-
+  measurement tools, experimental support for NSS in place of OpenSSL,
+  and much more. It also begins a full reorganization of Tor's code
+  layout, for improved modularity and maintainability in the future.
+  Finally, there is the usual set of performance improvements and
+  bugfixes that we try to do in every release series.
+
+  There are a couple of changes in the 0.3.5 that may affect
+  compatibility. First, the default version for newly created onion
+  services is now v3. Use the HiddenServiceVersion option if you want to
+  override this. Second, some log messages related to bootstrapping have
+  changed; if you use stem, you may need to update to the latest version
+  so it will recognize them.
+
+  We have designated 0.3.5 as a "long-term support" (LTS) series: we
+  will continue to patch major bugs in typical configurations of 0.3.5
+  until at least 1 Feb 2022. (We do not plan to provide long-term
+  support for embedding, Rust support, NSS support, running a directory
+  authority, or unsupported platforms. For these, you will need to stick
+  with the latest stable release.)
+
+  Below are the changes since 0.3.5.6-rc. For a complete list of changes
+  since 0.3.4.9, see the ReleaseNotes file.
+
+  o Major bugfixes (relay, directory):
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Minor features (compilation):
+    - When possible, place our warning flags in a separate file, to
+      avoid flooding verbose build logs. Closes ticket 28924.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (OpenSSL bug workaround):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor features (performance):
+    - Remove about 96% of the work from the function that we run at
+      startup to test our curve25519_basepoint implementation. Since
+      this function has yet to find an actual failure, we now only run
+      it for 8 iterations instead of 200. Based on our profile
+      information, this change should save around 8% of our startup time
+      on typical desktops, and may have a similar effect on other
+      platforms. Closes ticket 28838.
+    - Stop re-validating our hardcoded Diffie-Hellman parameters on
+      every startup. Doing this wasted time and cycles, especially on
+      low-powered devices. Closes ticket 28851.
+
+  o Minor bugfixes (compilation):
+    - Fix compilation for Android by adding a missing header to
+      freespace.c. Fixes bug 28974; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (correctness):
+    - Fix an unreached code path where we checked the value of
+      "hostname" inside send_resolved_hostname_cell(). Previously, we
+      used it before checking it; now we check it first. Fixes bug
+      28879; bugfix on 0.1.2.7-alpha.
+
+  o Minor bugfixes (testing):
+    - Make sure that test_rebind.py actually obeys its timeout, even
+      when it receives a large number of log messages. Fixes bug 28883;
+      bugfix on 0.3.5.4-alpha.
+    - Stop running stem's unit tests as part of "make test-stem", but
+      continue to run stem's unit and online tests during "make test-
+      stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha.
+
+  o Minor bugfixes (windows services):
+    - Make Tor start correctly as an NT service again: previously it was
+      broken by refactoring. Fixes bug 28612; bugfix on 0.3.5.3-alpha.
+
+  o Code simplification and refactoring:
+    - When parsing a port configuration, make it more obvious to static
+      analyzer tools that we always initialize the address. Closes
+      ticket 28881.
+
+
+Changes in version 0.3.5.6-rc - 2018-12-18
+  Tor 0.3.5.6-rc fixes numerous small bugs in earlier versions of Tor.
+  It is the first release candidate in the 0.3.5.x series; if no further
+  huge bugs are found, our next release may be the stable 0.3.5.x.
+
+  o Minor features (continuous integration, Windows):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+
+  o Minor features (fallback directory list):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28744.
+
+  o Minor bugfixes (compilation):
+    - Add missing dependency on libgdi32.dll for tor-print-ed-signing-
+      cert.exe on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows):
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+
+  o Minor bugfixes (onion service v3):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (restart-in-process, boostrap):
+    - Add missing resets of bootstrap tracking state when shutting down
+      (regression caused by ticket 27169). Fixes bug 28524; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Use a separate DataDirectory for the test_rebind script.
+      Previously, this script would run using the default DataDirectory,
+      and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha.
+      Patch from Taylor R Campbell.
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (Windows):
+    - Correctly identify Windows 8.1, Windows 10, and Windows Server
+      2008 and later from their NT versions. Fixes bug 28096; bugfix on
+      0.2.2.34; reported by Keifer Bly.
+    - On recent Windows versions, the GetVersionEx() function may report
+      an earlier Windows version than the running OS. To avoid user
+      confusion, add "[or later]" to Tor's version string on affected
+      versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported
+      by Keifer Bly.
+    - Remove Windows versions that were never supported by the
+      GetVersionEx() function. Stop duplicating the latest Windows
+      version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34;
+      reported by Keifer Bly.
+
+  o Testing:
+    - Increase logging and tag all log entries with timestamps in
+      test_rebind.py. Provides diagnostics for issue 28229.
+
+  o Code simplification and refactoring (shared random, dirauth):
+    - Change many tor_assert() to use BUG() instead. The idea is to not
+      crash a dirauth but rather scream loudly with a stacktrace and let
+      it continue run. The shared random subsystem is very resilient and
+      if anything wrong happens with it, at worst a non coherent value
+      will be put in the vote and discarded by the other authorities.
+      Closes ticket 19566.
+
+  o Documentation (onion services):
+    - Document in the man page that changing ClientOnionAuthDir value or
+      adding a new file in the directory will not work at runtime upon
+      sending a HUP if Sandbox 1. Closes ticket 28128.
+    - Note in the man page that the only real way to fully revoke an
+      onion service v3 client authorization is by restarting the tor
+      process. Closes ticket 28275.
+
+
+Changes in version 0.3.5.5-alpha - 2018-11-16
+  Tor 0.3.5.5-alpha includes numerous bugfixes on earlier releases,
+  including several that we hope to backport to older release series in
+  the future.
+
+  o Major bugfixes (OpenSSL, portability):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28395.
+
+  o Minor bugfixes (compilation):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection, relay):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (documentation):
+    - Make Doxygen work again after the code movement in the 0.3.5
+      source tree. Fixes bug 28435; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Permit the "shutdown()" system call, which is apparently used by
+      OpenSSL under some circumstances. Fixes bug 28183; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Stop talking about the Named flag in log messages. Clients have
+      ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (onion services):
+    - On an intro point for a version 3 onion service, stop closing
+      introduction circuits on an NACK. This lets the client decide
+      whether to reuse the circuit or discard it. Previously, we closed
+      intro circuits when sending NACKs. Fixes bug 27841; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chaunan.
+    - When replacing a descriptor in the client cache, make sure to
+      close all client introduction circuits for the old descriptor, so
+      we don't end up with unusable leftover circuits. Fixes bug 27471;
+      bugfix on 0.3.2.1-alpha.
+
+
 Changes in version 0.3.5.4-alpha - 2018-11-08
   Tor 0.3.5.4-alpha includes numerous bugfixes on earlier versions and
   improves our continuous integration support. It continues our attempts
@@ -2580,7 +5300,7 @@ Changes in version 0.3.4.1-alpha - 2018-05-17
     - Remove duplicate code in parse_{c,s}method_line and bootstrap
       their functionalities into a single function. Fixes bug 6236;
       bugfix on 0.2.3.6-alpha.
-    - We remove the PortForwsrding and PortForwardingHelper options,
+    - We remove the PortForwarding and PortForwardingHelper options,
       related functions, and the port_forwarding tests. These options
       were used by the now-deprecated Vidalia to help ordinary users
       become Tor relays or bridges. Closes ticket 25409. Patch by
@@ -9182,7 +11902,7 @@ Changes in version 0.2.9.3-alpha - 2016-09-23
       HiddenServiceNonAnonymousMode options. When both are set to 1,
       every hidden service on a Tor instance becomes a non-anonymous
       Single Onion Service. Single Onions make one-hop (direct)
-      connections to their introduction and renzedvous points. One-hop
+      connections to their introduction and rendezvous points. One-hop
       circuits make Single Onion servers easily locatable, but clients
       remain location-anonymous. This is compatible with the existing
       hidden service implementation, and works on the current tor
@@ -9685,13 +12405,12 @@ Changes in version 0.2.9.1-alpha - 2016-08-08
       needless crash bugs. Closes ticket 18613.
 
   o Minor features (performance):
-    - Changer the "optimistic data" extension from "off by default" to
+    - Change the "optimistic data" extension from "off by default" to
       "on by default". The default was ordinarily overridden by a
       consensus option, but when clients were bootstrapping for the
       first time, they would not have a consensus to get the option
-      from. Changing this default When fetching a consensus for the
-      first time, use optimistic data. This saves a round-trip during
-      startup. Closes ticket 18815.
+      from. Changing this default saves a round-trip during startup.
+      Closes ticket 18815.
 
   o Minor features (relay, usability):
     - When the directory authorities refuse a bad relay's descriptor,
diff --git a/LICENSE b/LICENSE
index cc27668427..438208426c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -9,7 +9,8 @@
         there may be other license terms that you should be aware of.
 
 ===============================================================================
-Tor is distributed under this license:
+Tor is distributed under the "3-clause BSD" license, a commonly used
+software license that means Tor is both free software and open source:
 
 Copyright (c) 2001-2004, Roger Dingledine
 Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
diff --git a/Makefile.am b/Makefile.am
index a868be7362..f6346fe0fc 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -31,9 +31,7 @@ TESTING_TOR_BINARY=$(top_builddir)/src/app/tor$(EXEEXT)
 endif
 
 if USE_RUST
-## this MUST be $(), otherwise am__DEPENDENCIES will not track it
-rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH) \
-	$(TOR_RUST_EXTRA_LIBS)
+rust_ldadd=$(top_builddir)/$(TOR_RUST_LIB_PATH)
 else
 rust_ldadd=
 endif
@@ -42,6 +40,10 @@ endif
 TOR_UTIL_LIBS = \
         src/lib/libtor-geoip.a \
 	src/lib/libtor-process.a \
+        src/lib/libtor-buf.a \
+	src/lib/libtor-confmgt.a \
+	src/lib/libtor-pubsub.a \
+	src/lib/libtor-dispatch.a \
 	src/lib/libtor-time.a \
 	src/lib/libtor-fs.a \
 	src/lib/libtor-encoding.a \
@@ -62,6 +64,7 @@ TOR_UTIL_LIBS = \
 	src/lib/libtor-malloc.a \
 	src/lib/libtor-wallclock.a \
 	src/lib/libtor-err.a \
+	src/lib/libtor-version.a \
 	src/lib/libtor-intmath.a \
 	src/lib/libtor-ctime.a
 
@@ -71,6 +74,10 @@ if UNITTESTS_ENABLED
 TOR_UTIL_TESTING_LIBS = \
         src/lib/libtor-geoip-testing.a \
 	src/lib/libtor-process-testing.a \
+        src/lib/libtor-buf-testing.a \
+	src/lib/libtor-confmgt-testing.a \
+	src/lib/libtor-pubsub-testing.a \
+	src/lib/libtor-dispatch-testing.a \
 	src/lib/libtor-time-testing.a \
 	src/lib/libtor-fs-testing.a \
 	src/lib/libtor-encoding-testing.a \
@@ -91,6 +98,7 @@ TOR_UTIL_TESTING_LIBS = \
 	src/lib/libtor-malloc-testing.a \
 	src/lib/libtor-wallclock-testing.a \
 	src/lib/libtor-err-testing.a \
+	src/lib/libtor-version-testing.a \
 	src/lib/libtor-intmath.a \
 	src/lib/libtor-ctime-testing.a
 endif
@@ -161,7 +169,28 @@ EXTRA_DIST+= \
 	README						\
 	ReleaseNotes					\
 	scripts/maint/checkIncludes.py                  \
-	scripts/maint/checkSpace.pl
+	scripts/maint/checkSpace.pl 			\
+	scripts/maint/checkShellScripts.sh			\
+	scripts/maint/practracker/README                \
+	scripts/maint/practracker/exceptions.txt	\
+	scripts/maint/practracker/includes.py		\
+	scripts/maint/practracker/metrics.py		\
+	scripts/maint/practracker/practracker.py	\
+	scripts/maint/practracker/practracker_tests.py	\
+	scripts/maint/practracker/problem.py		\
+	scripts/maint/practracker/testdata/.may_include	\
+	scripts/maint/practracker/testdata/a.c		\
+	scripts/maint/practracker/testdata/b.c 		\
+	scripts/maint/practracker/testdata/ex0-expected.txt \
+	scripts/maint/practracker/testdata/ex0.txt 	\
+	scripts/maint/practracker/testdata/ex1-expected.txt \
+	scripts/maint/practracker/testdata/ex1.txt 	\
+	scripts/maint/practracker/testdata/ex1-overbroad-expected.txt 	\
+	scripts/maint/practracker/testdata/ex.txt 	\
+	scripts/maint/practracker/testdata/header.h 	\
+	scripts/maint/practracker/testdata/not_c_file	\
+	scripts/maint/practracker/test_practracker.sh   \
+	scripts/maint/practracker/util.py
 
 ## This tells etags how to find mockable function definitions.
 AM_ETAGSFLAGS=--regex='{c}/MOCK_IMPL([^,]+,\W*\([a-zA-Z0-9_]+\)\W*,/\1/s'
@@ -179,7 +208,7 @@ TEST_CFLAGS=
 TEST_CPPFLAGS=-DTOR_UNIT_TESTS @TOR_MODULES_ALL_ENABLED@
 TEST_NETWORK_FLAGS=--hs-multi-client 1
 endif
-TEST_NETWORK_WARNING_FLAGS=--quiet --only-warnings
+TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS=--quiet --only-warnings
 
 if LIBFUZZER_ENABLED
 TEST_CFLAGS += -fsanitize-coverage=trace-pc-guard,trace-cmp,trace-div
@@ -215,7 +244,10 @@ doxygen:
 test: all
 	$(top_builddir)/src/test/test
 
-check-local: check-spaces check-changes check-includes
+shellcheck:
+	$(top_srcdir)/scripts/maint/checkShellScripts.sh
+
+check-local: check-spaces check-changes check-includes shellcheck
 
 need-chutney-path:
 	@if test ! -d "$$CHUTNEY_PATH"; then \
@@ -235,12 +267,15 @@ test-network: need-chutney-path $(TESTING_TOR_BINARY) src/tools/tor-gencert
 	$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_FLAGS)
 
 # Run all available tests using automake's test-driver
-# only run IPv6 tests if we can ping6 ::1 (localhost)
-# only run IPv6 tests if we can ping ::1 (localhost)
-# some IPv6 tests will fail without an IPv6 DNS server (see #16971 and #17011)
-# only run mixed tests if we have a tor-stable binary
-# Try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
-# because they're incompatible
+#   - only run IPv6 tests if we can ping6 or ping -6 ::1 (localhost)
+#     we try the syntax for BSD ping6, Linux ping6, and Linux ping -6,
+#     because they're incompatible
+#   - some IPv6 tests may fail without an IPv6 DNS server
+#     (see #16971 and #17011)
+#   - only run mixed tests if we have a tor-stable binary
+#   - show tor warnings on the console after each network run
+#     (otherwise, warnings go to the logs, and people don't see them unless
+#     there is a network failure)
 test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/tor-gencert
 	mkdir -p $(TEST_NETWORK_ALL_LOG_DIR)
 	rm -f $(TEST_NETWORK_ALL_LOG_DIR)/*.log $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
@@ -264,7 +299,7 @@ test-network-all: need-chutney-path test-driver $(TESTING_TOR_BINARY) src/tools/
 	done; \
 	for f in $$flavors; do \
 		$(SHELL) $(top_srcdir)/test-driver --test-name $$f --log-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.log --trs-file $(TEST_NETWORK_ALL_LOG_DIR)/$$f.trs $(TEST_NETWORK_ALL_DRIVER_FLAGS) $(top_srcdir)/src/test/test-network.sh --flavor $$f $(TEST_NETWORK_FLAGS); \
-		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_WARNING_FLAGS); \
+		$(top_srcdir)/src/test/test-network.sh $(TEST_NETWORK_SHOW_WARNINGS_FOR_LAST_RUN_FLAGS); \
 	done; \
 	echo "Log and result files are available in $(TEST_NETWORK_ALL_LOG_DIR)."; \
 	! grep -q FAIL $(TEST_NETWORK_ALL_LOG_DIR)/*.trs
@@ -319,11 +354,8 @@ coverage-html-full: all
 	lcov --remove "$(HTML_COVER_DIR)/lcov.tmp" --rc lcov_branch_coverage=1 'test/*' 'ext/tinytest*' '/usr/*' --output-file "$(HTML_COVER_DIR)/lcov.info"
 	genhtml --branch-coverage -o "$(HTML_COVER_DIR)" "$(HTML_COVER_DIR)/lcov.info"
 
-# Avoid strlcpy.c, strlcat.c, aes.c, OpenBSD_malloc_Linux.c, sha256.c,
-# tinytest*.[ch]
-check-spaces:
-if USE_PERL
-	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+# For scripts: avoid src/ext and src/trunnel.
+OWNED_TOR_C_FILES=\
 		$(top_srcdir)/src/lib/*/*.[ch] \
 		$(top_srcdir)/src/core/*/*.[ch] \
 		$(top_srcdir)/src/feature/*/*.[ch] \
@@ -331,13 +363,26 @@ if USE_PERL
 		$(top_srcdir)/src/test/*.[ch] \
 		$(top_srcdir)/src/test/*/*.[ch] \
 		$(top_srcdir)/src/tools/*.[ch]
+
+check-spaces:
+if USE_PERL
+	$(PERL) $(top_srcdir)/scripts/maint/checkSpace.pl -C \
+		$(OWNED_TOR_C_FILES)
 endif
 
 check-includes:
 if USEPYTHON
-	$(PYTHON) $(top_srcdir)/scripts/maint/checkIncludes.py
+	$(PYTHON) $(top_srcdir)/scripts/maint/practracker/includes.py $(top_srcdir)
 endif
 
+check-best-practices:
+if USEPYTHON
+	@$(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py $(top_srcdir) $(TOR_PRACTRACKER_OPTIONS)
+endif
+
+practracker-regen:
+	$(PYTHON) $(top_srcdir)/scripts/maint/practracker/practracker.py --regen $(top_srcdir)
+
 check-docs: all
 	$(PERL) $(top_builddir)/scripts/maint/checkOptionDocs.pl
 
@@ -414,13 +459,13 @@ endif
 check-changes:
 if USEPYTHON
 	@if test -d "$(top_srcdir)/changes"; then \
-		$(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \
+		PACKAGE_VERSION=$(PACKAGE_VERSION) $(PYTHON) $(top_srcdir)/scripts/maint/lintChanges.py $(top_srcdir)/changes; \
 		fi
 endif
 
 .PHONY: update-versions
 update-versions:
-	$(PERL) $(top_builddir)/scripts/maint/updateVersions.pl
+	abs_top_srcdir="$(abs_top_srcdir)" $(PYTHON) $(top_srcdir)/scripts/maint/update_versions.py
 
 .PHONY: callgraph
 callgraph:
@@ -433,6 +478,25 @@ version:
 	   (cd "$(top_srcdir)" && git rev-parse --short=16 HEAD); \
 	fi
 
+.PHONY: autostyle-ifdefs
+autostyle-ifdefs:
+	$(PYTHON) scripts/maint/annotate_ifdef_directives.py $(OWNED_TOR_C_FILES)
+
+.PHONY: autostyle-ifdefs
+autostyle-operators:
+	$(PERL) scripts/coccinelle/test-operator-cleanup $(OWNED_TOR_C_FILES)
+
+.PHONY: rectify-includes
+rectify-includes:
+	$(PYTHON) scripts/maint/rectify_include_paths.py
+
+.PHONY: update-copyright
+update-copyright:
+	$(PERL) scripts/maint/updateCopyright.pl $(OWNED_TOR_C_FILES)
+
+.PHONY: autostyle
+autostyle: update-versions rustfmt autostyle-ifdefs rectify-includes
+
 mostlyclean-local:
 	rm -f $(top_builddir)/src/*/*.gc{da,no} $(top_builddir)/src/*/*/*.gc{da,no}
 	rm -rf $(HTML_COVER_DIR)
diff --git a/ReleaseNotes b/ReleaseNotes
index e0a25a74b7..ad24efd606 100644
--- a/ReleaseNotes
+++ b/ReleaseNotes
@@ -2,6 +2,2441 @@ This document summarizes new features and bugfixes in each stable
 release of Tor. If you want to see more detailed descriptions of the
 changes in each development snapshot, see the ChangeLog file.
 
+Changes in version 0.4.1.6 - 2019-09-19
+  This release backports several bugfixes to improve stability and
+  correctness.  Anyone experiencing build problems or crashes with 0.4.1.5,
+  or experiencing reliability issues with single onion services, should
+  upgrade.
+
+  o Major bugfixes (crash, Linux, Android, backport from 0.4.2.1-alpha):
+    - Tolerate systems (including some Android installations) where
+      madvise and MADV_DONTDUMP are available at build-time, but not at
+      run time. Previously, these systems would notice a failed syscall
+      and abort. Fixes bug 31570; bugfix on 0.4.1.1-alpha.
+    - Tolerate systems (including some Linux installations) where
+      madvise and/or MADV_DONTFORK are available at build-time, but not
+      at run time. Previously, these systems would notice a failed
+      syscall and abort. Fixes bug 31696; bugfix on 0.4.1.1-alpha.
+
+  o Minor features (stem tests, backport from 0.4.2.1-alpha):
+    - Change "make test-stem" so it only runs the stem tests that use
+      tor. This change makes test-stem faster and more reliable. Closes
+      ticket 31554.
+
+  o Minor bugfixes (build system, backport form 0.4.2.1-alpha):
+    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows
+      systems. Fixes bug 31673; bugfix on 0.2.5.4-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.2.1-alpha):
+    - Add more stub functions to fix compilation on Android with link-
+      time optimization when --disable-module-dirauth is used.
+      Previously, these compilation settings would make the compiler
+      look for functions that didn't exist. Fixes bug 31552; bugfix
+      on 0.4.1.1-alpha.
+    - Suppress spurious float-conversion warnings from GCC when calling
+      floating-point classifier functions on FreeBSD. Fixes part of bug
+      31687; bugfix on 0.3.1.5-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Fix the MAPADDRESS controller command to accept one or more
+      arguments. Previously, it required two or more arguments, and ignored
+      the first. Fixes bug 31772; bugfix on 0.4.1.1-alpha.
+
+  o Minor bugfixes (guards, backport from 0.4.2.1-alpha):
+    - When tor is missing descriptors for some primary entry guards,
+      make the log message less alarming. It's normal for descriptors to
+      expire, as long as tor fetches new ones soon after. Fixes bug
+      31657; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (logging, backport from 0.4.2.1-alpha):
+    - Change log level of message "Hash of session info was not as
+      expected" to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix
+      on 0.1.1.10-alpha.
+
+  o Minor bugfixes (rust, backport from 0.4.2.1-alpha):
+    - Correctly exclude a redundant rust build job in Travis. Fixes bug
+      31463; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (v2 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v2 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v2 single onion services used a 3-hop
+      path when rendezvous circuits were retried after a remote or
+      delayed failure, but a 1-hop path for immediate retries. Fixes bug
+      23818; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (v3 single onion services, backport from 0.4.2.1-alpha):
+    - Always retry v3 single onion service intro and rend circuits with
+      a 3-hop path. Previously, v3 single onion services used a 3-hop
+      path when rend circuits were retried after a remote or delayed
+      failure, but a 1-hop path for immediate retries. Fixes bug 23818;
+      bugfix on 0.3.2.1-alpha.
+    - Make v3 single onion services fall back to a 3-hop intro, when all
+      intro points are unreachable via a 1-hop path. Previously, v3
+      single onion services failed when all intro nodes were unreachable
+      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
+
+  o Documentation (backport from 0.4.2.1-alpha):
+    - Use RFC 2397 data URL scheme to embed an image into tor-exit-
+      notice.html so that operators no longer have to host it
+      themselves. Closes ticket 31089.
+
+
+Changes in version 0.4.1.5 - 2019-08-20
+  This is the first stable release in the 0.4.1.x series. This series
+  adds experimental circuit-level padding, authenticated SENDME cells to
+  defend against certain attacks, and several performance improvements
+  to save on CPU consumption. It fixes bugs in bootstrapping and v3
+  onion services. It also includes numerous smaller features and
+  bugfixes on earlier versions.
+
+  Per our support policy, we will support the 0.4.1.x series for nine
+  months, or until three months after the release of a stable 0.4.2.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.4.0.5. For a list of only the changes
+  since 0.4.1.4-rc, see the ChangeLog file.
+
+  o Directory authority changes:
+    - The directory authority "dizum" has a new IP address. Closes
+      ticket 31406.
+
+  o Major features (circuit padding):
+    - Onion service clients now add padding cells at the start of their
+      INTRODUCE and RENDEZVOUS circuits, to make those circuits' traffic
+      look more like general purpose Exit traffic. The overhead for this
+      is 2 extra cells in each direction for RENDEZVOUS circuits, and 1
+      extra upstream cell and 10 downstream cells for INTRODUCE
+      circuits. This feature is only enabled when also supported by the
+      circuit's middle node. (Clients may specify fixed middle nodes
+      with the MiddleNodes option, and may force-disable this feature
+      with the CircuitPadding option.) Closes ticket 28634.
+
+  o Major features (code organization):
+    - Tor now includes a generic publish-subscribe message-passing
+      subsystem that we can use to organize intermodule dependencies. We
+      hope to use this to reduce dependencies between modules that don't
+      need to be related, and to generally simplify our codebase. Closes
+      ticket 28226.
+
+  o Major features (controller protocol):
+    - Controller commands are now parsed using a generalized parsing
+      subsystem. Previously, each controller command was responsible for
+      parsing its own input, which led to strange inconsistencies.
+      Closes ticket 30091.
+
+  o Major features (flow control):
+    - Implement authenticated SENDMEs as detailed in proposal 289. A
+      SENDME cell now includes the digest of the traffic that it
+      acknowledges, so that once an end point receives the SENDME, it
+      can confirm the other side's knowledge of the previous cells that
+      were sent, and prevent certain types of denial-of-service attacks.
+      This behavior is controlled by two new consensus parameters: see
+      the proposal for more details. Fixes ticket 26288.
+
+  o Major features (performance):
+    - Our node selection algorithm now excludes nodes in linear time.
+      Previously, the algorithm was quadratic, which could slow down
+      heavily used onion services. Closes ticket 30307.
+
+  o Major features (performance, RNG):
+    - Tor now constructs a fast secure pseudorandom number generator for
+      each thread, to use when performance is critical. This PRNG is
+      based on AES-CTR, using a buffering construction similar to
+      libottery and the (newer) OpenBSD arc4random() code. It
+      outperforms OpenSSL 1.1.1a's CSPRNG by roughly a factor of 100 for
+      small outputs. Although we believe it to be cryptographically
+      strong, we are only using it when necessary for performance.
+      Implements tickets 29023 and 29536.
+
+  o Major bugfixes (bridges):
+    - Consider our directory information to have changed when our list
+      of bridges changes. Previously, Tor would not re-compute the
+      status of its directory information when bridges changed, and
+      therefore would not realize that it was no longer able to build
+      circuits. Fixes part of bug 29875.
+    - Do not count previously configured working bridges towards our
+      total of working bridges. Previously, when Tor's list of bridges
+      changed, it would think that the old bridges were still usable,
+      and delay fetching router descriptors for the new ones. Fixes part
+      of bug 29875; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (circuit build, guard):
+    - On relays, properly check that a padding machine is absent before
+      logging a warning about it being absent. Fixes bug 30649; bugfix
+      on 0.4.0.1-alpha.
+    - When considering upgrading circuits from "waiting for guard" to
+      "open", always ignore circuits that are marked for close. Otherwise,
+      we can end up in the situation where a subsystem is notified that
+      a closing circuit has just opened, leading to undesirable
+      behavior. Fixes bug 30871; bugfix on 0.3.0.1-alpha.
+
+  o Major bugfixes (onion service reachability):
+    - Properly clean up the introduction point map when circuits change
+      purpose from onion service circuits to pathbias, measurement, or
+      other circuit types. This should fix some service-side instances
+      of introduction point failure. Fixes bug 29034; bugfix
+      on 0.3.2.1-alpha.
+
+  o Major bugfixes (onion service v3):
+    - Fix an unreachable bug in which an introduction point could try to
+      send an INTRODUCE_ACK with a status code that Trunnel would refuse
+      to encode, leading the relay to assert(). We've consolidated the
+      ABI values into Trunnel now. Fixes bug 30454; bugfix
+      on 0.3.0.1-alpha.
+    - Clients can now handle unknown status codes from INTRODUCE_ACK
+      cells. (The NACK behavior will stay the same.) This will allow us
+      to extend status codes in the future without breaking the normal
+      client behavior. Fixes another part of bug 30454; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor features (authenticated SENDME):
+    - Ensure that there is enough randomness on every circuit to prevent
+      an attacker from successfully predicting the hashes they will need
+      to include in authenticated SENDME cells. At a random interval, if
+      we have not sent randomness already, we now leave some extra space
+      at the end of a cell that we can fill with random bytes. Closes
+      ticket 26846.
+
+  o Minor features (circuit padding logging):
+    - Demote noisy client-side warn logs about circuit padding to protocol
+      warnings. Add additional log messages and circuit ID fields to help
+      with bug 30992 and any other future issues.
+
+  o Minor features (circuit padding):
+    - We now use a fast PRNG when scheduling circuit padding. Part of
+      ticket 28636.
+    - Allow the padding machine designer to pick the edges of their
+      histogram instead of trying to compute them automatically using an
+      exponential formula. Resolves some undefined behavior in the case
+      of small histograms and allows greater flexibility on machine
+      design. Closes ticket 29298; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to hold a circuit open until they
+      are done padding it. Closes ticket 28780.
+
+  o Minor features (compile-time modules):
+    - Add a "--list-modules" command to print a list of which compile-
+      time modules are enabled. Closes ticket 30452.
+
+  o Minor features (continuous integration):
+    - Our Travis configuration now uses Chutney to run some network
+      integration tests automatically. Closes ticket 29280.
+    - When running coverage builds on Travis, we now set
+      TOR_TEST_RNG_SEED, to avoid RNG-based coverage differences. Part
+      of ticket 28878.
+    - Remove sudo configuration lines from .travis.yml as they are no
+      longer needed with current Travis build environment. Resolves
+      issue 30213.
+    - In Travis, show stem's tor log after failure. Closes ticket 30234.
+
+  o Minor features (controller):
+    - Add onion service version 3 support to the HSFETCH command.
+      Previously, only version 2 onion services were supported. Closes
+      ticket 25417. Patch by Neel Chauhan.
+
+  o Minor features (debugging):
+    - Introduce tor_assertf() and tor_assertf_nonfatal() to enable
+      logging of additional information during assert failure. Now we
+      can use format strings to include information for trouble
+      shooting. Resolves ticket 29662.
+
+  o Minor features (defense in depth):
+    - In smartlist_remove_keeporder(), set unused pointers to NULL, in
+      case a bug causes them to be used later. Closes ticket 30176.
+      Patch from Tobias Stoeckmann.
+    - Tor now uses a cryptographically strong PRNG even for decisions
+      that we do not believe are security-sensitive. Previously, for
+      performance reasons, we had used a trivially predictable linear
+      congruential generator algorithm for certain load-balancing and
+      statistical sampling decisions. Now we use our fast RNG in those
+      cases. Closes ticket 29542.
+
+  o Minor features (developer tools):
+    - Tor's "practracker" test script now checks for files and functions
+      that seem too long and complicated. Existing overlong functions
+      and files are accepted for now, but should eventually be
+      refactored. Closes ticket 29221.
+    - Add some scripts used for git maintenance to scripts/git. Closes
+      ticket 29391.
+    - Call practracker from pre-push and pre-commit git hooks to let
+      developers know if they made any code style violations. Closes
+      ticket 30051.
+    - Add a script to check that each header has a well-formed and
+      unique guard macro. Closes ticket 29756.
+
+  o Minor features (fallback directory list):
+    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
+      in December 2018 (of which ~122 were still functional), with a
+      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
+      in June 2019. Closes ticket 28795.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30852.
+    - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2
+      Country database. Closes ticket 30522.
+
+  o Minor features (HTTP tunnel):
+    - Return an informative web page when the HTTPTunnelPort is used as
+      an HTTP proxy. Closes ticket 27821, patch by "eighthave".
+
+  o Minor features (IPv6, v3 onion services):
+    - Make v3 onion services put IPv6 addresses in service descriptors.
+      Before this change, service descriptors only contained IPv4
+      addresses. Implements 26992.
+
+  o Minor features (logging):
+    - Give a more useful assertion failure message if we think we have
+      minherit() but we fail to make a region non-inheritable. Give a
+      compile-time warning if our support for minherit() is incomplete.
+      Closes ticket 30686.
+
+  o Minor features (maintenance):
+    - Add a new "make autostyle" target that developers can use to apply
+      all automatic Tor style and consistency conversions to the
+      codebase. Closes ticket 30539.
+
+  o Minor features (modularity):
+    - The "--disable-module-dirauth" compile-time option now disables
+      even more dirauth-only code. Closes ticket 30345.
+
+  o Minor features (performance):
+    - Use OpenSSL's implementations of SHA3 when available (in OpenSSL
+      1.1.1 and later), since they tend to be faster than tiny-keccak.
+      Closes ticket 28837.
+
+  o Minor features (testing):
+    - The circuitpadding tests now use a reproducible RNG implementation,
+      so that if a test fails, we can learn why. Part of ticket 28878.
+    - Tor's tests now support an environment variable, TOR_TEST_RNG_SEED,
+      to set the RNG seed for tests that use a reproducible RNG. Part of
+      ticket 28878.
+    - When running tests in coverage mode, take additional care to make
+      our coverage deterministic, so that we can accurately track
+      changes in code coverage. Closes ticket 30519.
+    - Tor's unit test code now contains helper functions to replace the
+      PRNG with a deterministic or reproducible version for testing.
+      Previously, various tests implemented this in various ways.
+      Implements ticket 29732.
+    - We now have a script, cov-test-determinism.sh, to identify places
+      where our unit test coverage has become nondeterministic. Closes
+      ticket 29436.
+    - Check that representative subsets of values of `int` and `unsigned
+      int` can be represented by `void *`. Resolves issue 29537.
+
+  o Minor bugfixes (bridge authority):
+    - Bridge authorities now set bridges as running or non-running when
+      about to dump their status to a file. Previously, they set bridges
+      as running in response to a GETINFO command, but those shouldn't
+      modify data structures. Fixes bug 24490; bugfix on 0.2.0.13-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (channel padding statistics):
+    - Channel padding write totals and padding-enabled totals are now
+      counted properly in relay extrainfo descriptors. Fixes bug 29231;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (circuit isolation):
+    - Fix a logic error that prevented the SessionGroup sub-option from
+      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
+
+  o Minor bugfixes (circuit padding):
+    - Add a "CircuitPadding" torrc option to disable circuit padding.
+      Fixes bug 28693; bugfix on 0.4.0.1-alpha.
+    - Allow circuit padding machines to specify that they do not
+      contribute much overhead, and provide consensus flags and torrc
+      options to force clients to only use these low overhead machines.
+      Fixes bug 29203; bugfix on 0.4.0.1-alpha.
+    - Provide a consensus parameter to fully disable circuit padding, to
+      be used in emergency network overload situations. Fixes bug 30173;
+      bugfix on 0.4.0.1-alpha.
+    - The circuit padding subsystem will no longer schedule padding if
+      dormant mode is enabled. Fixes bug 28636; bugfix on 0.4.0.1-alpha.
+    - Inspect a circuit-level cell queue before sending padding, to
+      avoid sending padding while too much data is already queued. Fixes
+      bug 29204; bugfix on 0.4.0.1-alpha.
+    - Avoid calling monotime_absolute_usec() in circuit padding machines
+      that do not use token removal or circuit RTT estimation. Fixes bug
+      29085; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (clock skew detection):
+    - Don't believe clock skew results from NETINFO cells that appear to
+      arrive before we sent the VERSIONS cells they are responding to.
+      Previously, we would accept them up to 3 minutes "in the past".
+      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compatibility, standards compliance):
+    - Fix a bug that would invoke undefined behavior on certain
+      operating systems when trying to asprintf() a string exactly
+      INT_MAX bytes long. We don't believe this is exploitable, but it's
+      better to fix it anyway. Fixes bug 31001; bugfix on 0.2.2.11-alpha.
+      Found and fixed by Tobias Stoeckmann.
+
+  o Minor bugfixes (compilation warning):
+    - Fix a compilation warning on Windows about casting a function
+      pointer for GetTickCount64(). Fixes bug 31374; bugfix on
+      0.2.9.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Avoid using labs() on time_t, which can cause compilation warnings
+      on 64-bit Windows builds.  Fixes bug 31343; bugfix on 0.2.4.4-alpha.
+
+  o Minor bugfixes (compilation, unusual configurations):
+    - Avoid failures when building with the ALL_BUGS_ARE_FATAL option
+      due to missing declarations of abort(), and prevent other such
+      failures in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (configuration, proxies):
+    - Fix a bug that prevented us from supporting SOCKS5 proxies that
+      want authentication along with configured (but unused!)
+      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
+
+  o Minor bugfixes (continuous integration):
+    - Allow the test-stem job to fail in Travis, because it sometimes
+      hangs. Fixes bug 30744; bugfix on 0.3.5.4-alpha.
+    - Skip test_rebind on macOS in Travis, because it is unreliable on
+      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environment
+      variable is set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (controller protocol):
+    - Teach the controller parser to distinguish an object preceded by
+      an argument list from one without. Previously, it couldn't
+      distinguish an argument list from the first line of a multiline
+      object. Fixes bug 29984; bugfix on 0.2.3.8-alpha.
+
+  o Minor bugfixes (crash on exit):
+    - Avoid a set of possible code paths that could try to use freed
+      memory in routerlist_free() while Tor was exiting. Fixes bug
+      31003; bugfix on 0.1.2.2-alpha.
+
+  o Minor bugfixes (developer tooling):
+    - Fix pre-push hook to allow fixup and squash commits when pushing
+      to non-upstream git remote. Fixes bug 30286; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (directory authorities):
+    - Stop crashing after parsing an unknown descriptor purpose
+      annotation. We think this bug can only be triggered by modifying a
+      local file. Fixes bug 30781; bugfix on 0.2.0.8-alpha.
+    - Move the "bandwidth-file-headers" line in directory authority
+      votes so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix
+      on 0.3.5.1-alpha.
+    - Directory authorities with IPv6 support now always mark themselves
+      as reachable via IPv6. Fixes bug 24338; bugfix on 0.2.4.1-alpha.
+      Patch by Neel Chauhan.
+
+  o Minor bugfixes (documentation):
+    - Improve the documentation for using MapAddress with ".exit". Fixes
+      bug 30109; bugfix on 0.1.0.1-rc.
+    - Improve the monotonic time module and function documentation to
+      explain what "monotonic" actually means, and document some results
+      that have surprised people. Fixes bug 29640; bugfix
+      on 0.2.9.1-alpha.
+    - Use proper formatting when providing an example on quoting options
+      that contain whitespace. Fixes bug 29635; bugfix on 0.2.3.18-rc.
+
+  o Minor bugfixes (logging):
+    - Do not log a warning when running with an OpenSSL version other
+      than the one Tor was compiled with, if the two versions should be
+      compatible. Previously, we would warn whenever the version was
+      different. Fixes bug 30190; bugfix on 0.2.4.2-alpha.
+    - Warn operators when the MyFamily option is set but ContactInfo is
+      missing, as the latter should be set too. Fixes bug 25110; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Avoid a minor memory leak that could occur on relays when failing
+      to create a "keys" directory. Fixes bug 30148; bugfix
+      on 0.3.3.1-alpha.
+    - Fix a trivial memory leak when parsing an invalid value from a
+      download schedule in the configuration. Fixes bug 30894; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (NetBSD):
+    - Fix usage of minherit() on NetBSD and other platforms that define
+      MAP_INHERIT_{ZERO,NONE} instead of INHERIT_{ZERO,NONE}. Fixes bug
+      30614; bugfix on 0.4.0.2-alpha. Patch from Taylor Campbell.
+
+  o Minor bugfixes (onion services):
+    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
+      implemenation) when failing to load an onion service client
+      authorization file. Fixes bug 30475; bugfix on 0.3.5.1-alpha.
+    - When refusing to launch a controller's HSFETCH request because of
+      rate-limiting, respond to the controller with a new response,
+      "QUERY_RATE_LIMITED". Previously, we would log QUERY_NO_HSDIR for
+      this case. Fixes bug 28269; bugfix on 0.3.1.1-alpha. Patch by
+      Neel Chauhan.
+    - When relaunching a circuit to a rendezvous service, mark the
+      circuit as needing high-uptime routers as appropriate. Fixes bug
+      17357; bugfix on 0.1.0.1-rc. Patch by Neel Chauhan.
+    - Stop ignoring IPv6 link specifiers sent to v3 onion services.
+      (IPv6 support for v3 onion services is still incomplete: see
+      ticket 23493 for details.) Fixes bug 23588; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (onion services, performance):
+    - When building circuits to onion services, call tor_addr_parse()
+      less often. Previously, we called tor_addr_parse() in
+      circuit_is_acceptable() even if its output wasn't used. This
+      change should improve performance when building circuits. Fixes
+      bug 22210; bugfix on 0.2.8.12. Patch by Neel Chauhan.
+
+  o Minor bugfixes (out-of-memory handler):
+    - When purging the DNS cache because of an out-of-memory condition,
+      try purging just the older entries at first. Previously, we would
+      always purge the whole thing. Fixes bug 29617; bugfix
+      on 0.3.5.1-alpha.
+
+  o Minor bugfixes (performance):
+    - When checking whether a node is a bridge, use a fast check to make
+      sure that its identity is set. Previously, we used a constant-time
+      check, which is not necessary in this case. Fixes bug 30308;
+      bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Tor now sets TOR_PT_EXIT_ON_STDIN_CLOSE=1 for client transports as
+      well as servers. Fixes bug 25614; bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (portability):
+    - Avoid crashing in our tor_vasprintf() implementation on systems
+      that define neither vasprintf() nor _vscprintf(). (This bug has
+      been here long enough that we question whether people are running
+      Tor on such systems, but we're applying the fix out of caution.)
+      Fixes bug 30561; bugfix on 0.2.8.2-alpha. Found and fixed by
+      Tobias Stoeckmann.
+
+  o Minor bugfixes (probability distributions):
+    - Refactor and improve parts of the probability distribution code
+      that made Coverity complain. Fixes bug 29805; bugfix
+      on 0.4.0.1-alpha.
+
+  o Minor bugfixes (python):
+    - Stop assuming that /usr/bin/python3 exists. For scripts that work
+      with python2, use /usr/bin/python. Otherwise, use /usr/bin/env
+      python3. Fixes bug 29913; bugfix on 0.2.5.3-alpha.
+
+  o Minor bugfixes (relay):
+    - When running as a relay, if IPv6Exit is set to 1 while ExitRelay
+      is auto, act as if ExitRelay is 1. Previously, we would ignore
+      IPv6Exit if ExitRelay was 0 or auto. Fixes bug 29613; bugfix on
+      0.3.5.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (static analysis):
+    - Fix several spurious Coverity warnings about the unit tests, to
+      lower our chances of missing real warnings in the future. Fixes
+      bug 30150; bugfix on 0.3.5.1-alpha and various other Tor versions.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including bandwidth usage
+      statistics, GeoIPFile hashes, ServerTransportPlugin lines, and
+      bridge statistics by country in extra-info documents. Fixes bug
+      29018; bugfix on 0.2.4.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Call setrlimit() to disable core dumps in test_bt_cl.c. Previously
+      we used `ulimit -c` in test_bt.sh, which violates POSIX shell
+      compatibility. Fixes bug 29061; bugfix on 0.3.5.1-alpha.
+    - Fix some incorrect code in the v3 onion service unit tests. Fixes
+      bug 29243; bugfix on 0.3.2.1-alpha.
+    - In the "routerkeys/*" tests, check the return values of mkdir()
+      for possible failures. Fixes bug 29939; bugfix on 0.2.7.2-alpha.
+      Found by Coverity as CID 1444254.
+    - Split test_utils_general() into several smaller test functions.
+      This makes it easier to perform resource deallocation on assert
+      failure, and fixes Coverity warnings CID 1444117 and CID 1444118.
+      Fixes bug 29823; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (tor-resolve):
+    - Fix a memory leak in tor-resolve that could happen if Tor gave it
+      a malformed SOCKS response. (Memory leaks in tor-resolve don't
+      actually matter, but it's good to fix them anyway.) Fixes bug
+      30151; bugfix on 0.4.0.1-alpha.
+
+  o Code simplification and refactoring:
+    - Abstract out the low-level formatting of replies on the control
+      port. Implements ticket 30007.
+    - Add several assertions in an attempt to fix some Coverity
+      warnings. Closes ticket 30149.
+    - Introduce a connection_dir_buf_add() helper function that checks
+      for compress_state of dir_connection_t and automatically writes a
+      string to directory connection with or without compression.
+      Resolves issue 28816.
+    - Make the base32_decode() API return the number of bytes written,
+      for consistency with base64_decode(). Closes ticket 28913.
+    - Move most relay-only periodic events out of mainloop.c into the
+      relay subsystem. Closes ticket 30414.
+    - Refactor and encapsulate parts of the codebase that manipulate
+      crypt_path_t objects. Resolves issue 30236.
+    - Refactor several places in our code that Coverity incorrectly
+      believed might have memory leaks. Closes ticket 30147.
+    - Remove redundant return values in crypto_format, and the
+      associated return value checks elsewhere in the code. Make the
+      implementations in crypto_format consistent, and remove redundant
+      code. Resolves ticket 29660.
+    - Rename tor_mem_is_zero() to fast_mem_is_zero(), to emphasize that
+      it is not a constant-time function. Closes ticket 30309.
+    - Replace hs_desc_link_specifier_t with link_specifier_t, and remove
+      all hs_desc_link_specifier_t-specific code. Fixes bug 22781;
+      bugfix on 0.3.2.1-alpha.
+    - Simplify v3 onion service link specifier handling code. Fixes bug
+      23576; bugfix on 0.3.2.1-alpha.
+    - Split crypto_digest.c into NSS code, OpenSSL code, and shared
+      code. Resolves ticket 29108.
+    - Split control.c into several submodules, in preparation for
+      distributing its current responsibilities throughout the codebase.
+      Closes ticket 29894.
+    - Start to move responsibility for knowing about periodic events to
+      the appropriate subsystems, so that the mainloop doesn't need to
+      know all the periodic events in the rest of the codebase.
+      Implements tickets 30293 and 30294.
+
+  o Documentation:
+    - Mention URLs for Travis/Appveyor/Jenkins in ReleasingTor.md.
+      Closes ticket 30630.
+    - Document how to find git commits and tags for bug fixes in
+      CodingStandards.md. Update some file documentation. Closes
+      ticket 30261.
+
+  o Removed features:
+    - Remove the linux-tor-prio.sh script from contrib/operator-tools
+      directory. Resolves issue 29434.
+    - Remove the obsolete OpenSUSE initscript. Resolves issue 30076.
+    - Remove the obsolete script at contrib/dist/tor.sh.in. Resolves
+      issue 30075.
+
+  o Testing:
+    - Specify torrc paths (with empty files) when launching tor in
+      integration tests; refrain from reading user and system torrcs.
+      Resolves issue 29702.
+
+  o Code simplification and refactoring (shell scripts):
+    - Clean up many of our shell scripts to fix shellcheck warnings.
+      These include autogen.sh (ticket 26069), test_keygen.sh (ticket
+      29062), test_switch_id.sh (ticket 29065), test_rebind.sh (ticket
+      29063), src/test/fuzz/minimize.sh (ticket 30079), test_rust.sh
+      (ticket 29064), torify (ticket 29070), asciidoc-helper.sh (29926),
+      fuzz_multi.sh (30077), fuzz_static_testcases.sh (ticket 29059),
+      nagios-check-tor-authority-cert (ticket 29071),
+      src/test/fuzz/fixup_filenames.sh (ticket 30078), test-network.sh
+      (ticket 29060), test_key_expiration.sh (ticket 30002),
+      zero_length_keys.sh (ticket 29068), and test_workqueue_*.sh
+      (ticket 29067).
+
+  o Testing (chutney):
+    - In "make test-network-all", test IPv6-only v3 single onion
+      services, using the chutney network single-onion-v23-ipv6-md.
+      Closes ticket 27251.
+
+  o Testing (continuous integration):
+    - In Travis, make stem log a controller trace to the console, and tail
+      stem's tor log after failure. Closes ticket 30591.
+    - In Travis, only run the stem tests that use a tor binary.
+      Closes ticket 30694.
+
+
+Changes in version 0.4.0.5 - 2019-05-02
+  This is the first stable release in the 0.4.0.x series. It contains
+  improvements for power management and bootstrap reporting, as well as
+  preliminary backend support for circuit padding to prevent some kinds
+  of traffic analysis. It also continues our work in refactoring Tor for
+  long-term maintainability.
+
+  Per our support policy, we will support the 0.4.0.x series for nine
+  months, or until three months after the release of a stable 0.4.1.x:
+  whichever is longer. If you need longer-term support, please stick
+  with 0.3.5.x, which will we plan to support until Feb 2022.
+
+  Below are the changes since 0.3.5.7. For a complete list of changes
+  since 0.4.0.4-rc, see the ChangeLog file.
+
+  o Major features (battery management, client, dormant mode):
+    - When Tor is running as a client, and it is unused for a long time,
+      it can now enter a "dormant" state. When Tor is dormant, it avoids
+      network and CPU activity until it is reawoken either by a user
+      request or by a controller command. For more information, see the
+      configuration options starting with "Dormant". Implements tickets
+      2149 and 28335.
+    - The client's memory of whether it is "dormant", and how long it
+      has spent idle, persists across invocations. Implements
+      ticket 28624.
+    - There is a DormantOnFirstStartup option that integrators can use
+      if they expect that in many cases, Tor will be installed but
+      not used.
+
+  o Major features (bootstrap reporting):
+    - When reporting bootstrap progress, report the first connection
+      uniformly, regardless of whether it's a connection for building
+      application circuits. This allows finer-grained reporting of early
+      progress than previously possible, with the improvements of ticket
+      27169. Closes tickets 27167 and 27103. Addresses ticket 27308.
+    - When reporting bootstrap progress, treat connecting to a proxy or
+      pluggable transport as separate from having successfully used that
+      proxy or pluggable transport to connect to a relay. Closes tickets
+      27100 and 28884.
+
+  o Major features (circuit padding):
+    - Implement preliminary support for the circuit padding portion of
+      Proposal 254. The implementation supports Adaptive Padding (aka
+      WTF-PAD) state machines for use between experimental clients and
+      relays. Support is also provided for APE-style state machines that
+      use probability distributions instead of histograms to specify
+      inter-packet delay. At the moment, Tor does not provide any
+      padding state machines that are used in normal operation: for now,
+      this feature exists solely for experimentation. Closes
+      ticket 28142.
+
+  o Major features (refactoring):
+    - Tor now uses an explicit list of its own subsystems when
+      initializing and shutting down. Previously, these systems were
+      managed implicitly in various places throughout the codebase.
+      (There may still be some subsystems using the old system.) Closes
+      ticket 28330.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth message and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (NSS, relay):
+    - When running with NSS, disable TLS 1.2 ciphersuites that use
+      SHA384 for their PRF. Due to an NSS bug, the TLS key exporters for
+      these ciphersuites don't work -- which caused relays to fail to
+      handshake with one another when these ciphersuites were enabled.
+      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
+
+  o Major bugfixes (windows, startup):
+    - When reading a consensus file from disk, detect whether it was
+      written in text mode, and re-read it in text mode if so. Always
+      write consensus files in binary mode so that we can map them into
+      memory later. Previously, we had written in text mode, which
+      confused us when we tried to map the file on windows. Fixes bug
+      28614; bugfix on 0.4.0.1-alpha.
+
+  o Minor features (address selection):
+    - Treat the subnet 100.64.0.0/10 as public for some purposes;
+      private for others. This subnet is the RFC 6598 (Carrier Grade
+      NAT) IP range, and is deployed by many ISPs as an alternative to
+      RFC 1918 that does not break existing internal networks. Tor now
+      blocks SOCKS and control ports on these addresses and warns users
+      if client ports or ExtORPorts are listening on a RFC 6598 address.
+      Closes ticket 28525. Patch by Neel Chauhan.
+
+  o Minor features (bandwidth authority):
+    - Make bandwidth authorities ignore relays that are reported in the
+      bandwidth file with the flag "vote=0". This change allows us to
+      report unmeasured relays for diagnostic reasons without including
+      their bandwidth in the bandwidth authorities' vote. Closes
+      ticket 29806.
+    - When a directory authority is using a bandwidth file to obtain the
+      bandwidth values that will be included in the next vote, serve
+      this bandwidth file at /tor/status-vote/next/bandwidth. Closes
+      ticket 21377.
+
+  o Minor features (bootstrap reporting):
+    - When reporting bootstrap progress, stop distinguishing between
+      situations where only internal paths are available and situations
+      where external paths are available. Previously, Tor would often
+      erroneously report that it had only internal paths. Closes
+      ticket 27402.
+
+  o Minor features (compilation):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (continuous integration):
+    - On Travis Rust builds, cleanup Rust registry and refrain from
+      caching the "target/" directory to speed up builds. Resolves
+      issue 29962.
+    - Log Python version during each Travis CI job. Resolves
+      issue 28551.
+    - In Travis, tell timelimit to use stem's backtrace signals, and
+      launch python directly from timelimit, so python receives the
+      signals from timelimit, rather than make. Closes ticket 30117.
+
+  o Minor features (controller):
+    - Add a DROPOWNERSHIP command to undo the effects of TAKEOWNERSHIP.
+      Implements ticket 28843.
+
+  o Minor features (developer tooling):
+    - Check that bugfix versions in changes files look like Tor versions
+      from the versions spec. Warn when bugfixes claim to be on a future
+      release. Closes ticket 27761.
+    - Provide a git pre-commit hook that disallows committing if we have
+      any failures in our code and changelog formatting checks. It is
+      now available in scripts/maint/pre-commit.git-hook. Implements
+      feature 28976.
+    - Provide a git hook script to prevent "fixup!" and "squash!"
+      commits from ending up in the master branch, as scripts/main/pre-
+      push.git-hook. Closes ticket 27993.
+
+  o Minor features (diagnostic):
+    - Add more diagnostic log messages in an attempt to solve the issue
+      of NUL bytes appearing in a microdescriptor cache. Related to
+      ticket 28223.
+
+  o Minor features (directory authority):
+    - When a directory authority is using a bandwidth file to obtain
+      bandwidth values, include the digest of that file in the vote.
+      Closes ticket 26698.
+    - Directory authorities support a new consensus algorithm, under
+      which the family lines in microdescriptors are encoded in a
+      canonical form. This change makes family lines more compressible
+      in transit, and on the client. Closes ticket 28266; implements
+      proposal 298.
+
+  o Minor features (directory authority, relay):
+    - Authorities now vote on a "StaleDesc" flag to indicate that a
+      relay's descriptor is so old that the relay should upload again
+      soon. Relays treat this flag as a signal to upload a new
+      descriptor. This flag will eventually let us remove the
+      'published' date from routerstatus entries, and make our consensus
+      diffs much smaller. Closes ticket 26770; implements proposal 293.
+
+  o Minor features (dormant mode):
+    - Add a DormantCanceledByStartup option to tell Tor that it should
+      treat a startup event as cancelling any previous dormant state.
+      Integrators should use this option with caution: it should only be
+      used if Tor is being started because of something that the user
+      did, and not if Tor is being automatically started in the
+      background. Closes ticket 29357.
+
+  o Minor features (fallback directory mirrors):
+    - Update the fallback whitelist based on operator opt-ins and opt-
+      outs. Closes ticket 24805, patch by Phoul.
+
+  o Minor features (FreeBSD):
+    - On FreeBSD-based systems, warn relay operators if the
+      "net.inet.ip.random_id" sysctl (IP ID randomization) is disabled.
+      Closes ticket 28518.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29992.
+
+  o Minor features (HTTP standards compliance):
+    - Stop sending the header "Content-type: application/octet-stream"
+      along with transparently compressed documents: this confused
+      browsers. Closes ticket 28100.
+
+  o Minor features (IPv6):
+    - We add an option ClientAutoIPv6ORPort, to make clients randomly
+      prefer a node's IPv4 or IPv6 ORPort. The random preference is set
+      every time a node is loaded from a new consensus or bridge config.
+      We expect that this option will enable clients to bootstrap more
+      quickly without having to determine whether they support IPv4,
+      IPv6, or both. Closes ticket 27490. Patch by Neel Chauhan.
+    - When using addrs_in_same_network_family(), avoid choosing circuit
+      paths that pass through the same IPv6 subnet more than once.
+      Previously, we only checked IPv4 subnets. Closes ticket 24393.
+      Patch by Neel Chauhan.
+
+  o Minor features (log messages):
+    - Improve log message in v3 onion services that could print out
+      negative revision counters. Closes ticket 27707. Patch
+      by "ffmancera".
+
+  o Minor features (memory usage):
+    - Save memory by storing microdescriptor family lists with a more
+      compact representation. Closes ticket 27359.
+    - Tor clients now use mmap() to read consensus files from disk, so
+      that they no longer need keep the full text of a consensus in
+      memory when parsing it or applying a diff. Closes ticket 27244.
+
+  o Minor features (NSS, diagnostic):
+    - Try to log an error from NSS (if there is any) and a more useful
+      description of our situation if we are using NSS and a call to
+      SSL_ExportKeyingMaterial() fails. Diagnostic for ticket 29241.
+
+  o Minor features (parsing):
+    - Directory authorities now validate that router descriptors and
+      ExtraInfo documents are in a valid subset of UTF-8, and reject
+      them if they are not. Closes ticket 27367.
+
+  o Minor features (performance):
+    - Cache the results of summarize_protocol_flags(), so that we don't
+      have to parse the same protocol-versions string over and over.
+      This should save us a huge number of malloc calls on startup, and
+      may reduce memory fragmentation with some allocators. Closes
+      ticket 27225.
+    - Remove a needless memset() call from get_token_arguments, thereby
+      speeding up the tokenization of directory objects by about 20%.
+      Closes ticket 28852.
+    - Replace parse_short_policy() with a faster implementation, to
+      improve microdescriptor parsing time. Closes ticket 28853.
+    - Speed up directory parsing a little by avoiding use of the non-
+      inlined strcmp_len() function. Closes ticket 28856.
+    - Speed up microdescriptor parsing by about 30%, to help improve
+      startup time. Closes ticket 28839.
+
+  o Minor features (pluggable transports):
+    - Add support for emitting STATUS updates to Tor's control port from
+      a pluggable transport process. Closes ticket 28846.
+    - Add support for logging to Tor's logging subsystem from a
+      pluggable transport process. Closes ticket 28180.
+
+  o Minor features (process management):
+    - Add a new process API for handling child processes. This new API
+      allows Tor to have bi-directional communication with child
+      processes on both Unix and Windows. Closes ticket 28179.
+    - Use the subsystem manager to initialize and shut down the process
+      module. Closes ticket 28847.
+
+  o Minor features (relay):
+    - When listing relay families, list them in canonical form including
+      the relay's own identity, and try to give a more useful set of
+      warnings. Part of ticket 28266 and proposal 298.
+
+  o Minor features (required protocols):
+    - Before exiting because of a missing required protocol, Tor will
+      now check the publication time of the consensus, and not exit
+      unless the consensus is newer than the Tor program's own release
+      date. Previously, Tor would not check the consensus publication
+      time, and so might exit because of a missing protocol that might
+      no longer be required in a current consensus. Implements proposal
+      297; closes ticket 27735.
+
+  o Minor features (testing):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+    - Allow a HeartbeatPeriod of less than 30 minutes in testing Tor
+      networks. Closes ticket 28840. Patch by Rob Jansen.
+    - Use the approx_time() function when setting the "Expires" header
+      in directory replies, to make them more testable. Needed for
+      ticket 30001.
+
+  o Minor bugfixes (security):
+    - Fix a potential double free bug when reading huge bandwidth files.
+      The issue is not exploitable in the current Tor network because
+      the vulnerable code is only reached when directory authorities
+      read bandwidth files, but bandwidth files come from a trusted
+      source (usually the authorities themselves). Furthermore, the
+      issue is only exploitable in rare (non-POSIX) 32-bit architectures,
+      which are not used by any of the current authorities. Fixes bug
+      30040; bugfix on 0.3.5.1-alpha. Bug found and fixed by
+      Tobias Stoeckmann.
+    - Verify in more places that we are not about to create a buffer
+      with more than INT_MAX bytes, to avoid possible OOB access in the
+      event of bugs. Fixes bug 30041; bugfix on 0.2.0.16. Found and
+      fixed by Tobias Stoeckmann.
+
+  o Minor bugfix (continuous integration):
+    - Reset coverage state on disk after Travis CI has finished. This
+      should prevent future coverage merge errors from causing the test
+      suite for the "process" subsystem to fail. The process subsystem
+      was introduced in 0.4.0.1-alpha. Fixes bug 29036; bugfix
+      on 0.2.9.15.
+    - Terminate test-stem if it takes more than 9.5 minutes to run.
+      (Travis terminates the job after 10 minutes of no output.)
+      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
+
+  o Minor bugfixes (build, compatibility, rust):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug
+      29824; bugfix on 0.3.1.1-alpha. This is Coverity warning
+      CID 1444119.
+
+  o Minor bugfixes (client, clock skew):
+    - Bootstrap successfully even when Tor's clock is behind the clocks
+      on the authorities. Fixes bug 28591; bugfix on 0.2.0.9-alpha.
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation):
+    - Fix compilation warnings in test_circuitpadding.c. Fixes bug
+      29169; bugfix on 0.4.0.1-alpha.
+    - Silence a compiler warning in test-memwipe.c on OpenBSD. Fixes bug
+      29145; bugfix on 0.2.9.3-alpha. Patch from Kris Katterjohn.
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (directory clients):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (directory mirrors):
+    - Even when a directory mirror's clock is behind the clocks on the
+      authorities, we now allow the mirror to serve "future"
+      consensuses. Fixes bug 28654; bugfix on 0.3.0.1-alpha.
+
+  o Minor bugfixes (DNS):
+    - Gracefully handle an empty or absent resolve.conf file by falling
+      back to using "localhost" as a DNS server (and hoping it works).
+      Previously, we would just stop running as an exit. Fixes bug
+      21900; bugfix on 0.2.1.10-alpha.
+
+  o Minor bugfixes (documentation):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (guards):
+    - In count_acceptable_nodes(), the minimum number is now one bridge
+      or guard node, and two non-guard nodes for a circuit. Previously,
+      we had added up the sum of all nodes with a descriptor, but that
+      could cause us to build failing circuits when we had either too
+      many bridges or not enough guard nodes. Fixes bug 25885; bugfix on
+      0.2.3.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (IPv6):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (linux seccomp sandbox):
+    - Fix startup crash when experimental sandbox support is enabled.
+      Fixes bug 29150; bugfix on 0.4.0.1-alpha. Patch by Peter Gerber.
+
+  o Minor bugfixes (logging):
+    - Correct a misleading error message when IPv4Only or IPv6Only is
+      used but the resolved address can not be interpreted as an address
+      of the specified IP version. Fixes bug 13221; bugfix on
+      0.2.3.9-alpha. Patch from Kris Katterjohn.
+    - Log the correct port number for listening sockets when "auto" is
+      used to let Tor pick the port number. Previously, port 0 was
+      logged instead of the actual port number. Fixes bug 29144; bugfix
+      on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+    - Stop logging a BUG() warning when Tor is waiting for exit
+      descriptors. Fixes bug 28656; bugfix on 0.3.5.1-alpha.
+    - Avoid logging that we are relaxing a circuit timeout when that
+      timeout is fixed. Fixes bug 28698; bugfix on 0.2.4.7-alpha.
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (memory management):
+    - Refactor the shared random state's memory management so that it
+      actually takes ownership of the shared random value pointers.
+      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
+    - Stop leaking parts of the shared random state in the shared-random
+      unit tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
+
+  o Minor bugfixes (misc):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (networking):
+    - Introduce additional checks into tor_addr_parse() to reject
+      certain incorrect inputs that previously were not detected. Fixes
+      bug 23082; bugfix on 0.2.0.10-alpha.
+
+  o Minor bugfixes (onion service v3, client):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (onion services):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (periodic events):
+    - Refrain from calling routerlist_remove_old_routers() from
+      check_descriptor_callback(). Instead, create a new hourly periodic
+      event. Fixes bug 27929; bugfix on 0.2.8.1-alpha.
+
+  o Minor bugfixes (pluggable transports):
+    - Make sure that data is continously read from standard output and
+      standard error pipes of a pluggable transport child-process, to
+      avoid deadlocking when a pipe's buffer is full. Fixes bug 26360;
+      bugfix on 0.2.3.6-alpha.
+
+  o Minor bugfixes (rust):
+    - Abort on panic in all build profiles, instead of potentially
+      unwinding into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (scheduler):
+    - When re-adding channels to the pending list, check the correct
+      channel's sched_heap_idx. This issue has had no effect in mainline
+      Tor, but could have led to bugs down the road in improved versions
+      of our circuit scheduling code. Fixes bug 29508; bugfix
+      on 0.3.2.10.
+
+  o Minor bugfixes (shellcheck):
+    - Look for scripts in their correct locations during "make
+      shellcheck". Previously we had looked in the wrong place during
+      out-of-tree builds. Fixes bug 30263; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (single onion services):
+    - Allow connections to single onion services to remain idle without
+      being disconnected. Previously, relays acting as rendezvous points
+      for single onion services were mistakenly closing idle rendezvous
+      circuits after 60 seconds, thinking that they were unused
+      directory-fetching circuits that had served their purpose. Fixes
+      bug 29665; bugfix on 0.2.1.26.
+
+  o Minor bugfixes (stats):
+    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
+      relay and bridge extra-info documents. Fixes bug 29017; bugfix
+      on 0.3.1.1-alpha.
+
+  o Minor bugfixes (testing):
+    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9. We need a
+      recent test-network.sh to use new chutney features in CI. Fixes
+      bug 29703; bugfix on 0.2.9.1-alpha.
+    - Fix a test failure on Windows caused by an unexpected "BUG"
+      warning in our tests for tor_gmtime_r(-1). Fixes bug 29922; bugfix
+      on 0.2.9.3-alpha.
+    - Downgrade some LOG_ERR messages in the address/* tests to
+      warnings. The LOG_ERR messages were occurring when we had no
+      configured network. We were failing the unit tests, because we
+      backported 28668 to 0.3.5.8, but did not backport 29530. Fixes bug
+      29530; bugfix on 0.3.5.8.
+    - Fix our gcov wrapper script to look for object files at the
+      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
+    - Decrease the false positive rate of stochastic probability
+      distribution tests. Fixes bug 29693; bugfix on 0.4.0.1-alpha.
+    - Fix intermittent failures on an adaptive padding test. Fixes one
+      case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Disable an unstable circuit-padding test that was failing
+      intermittently because of an ill-defined small histogram. Such
+      histograms will be allowed again after 29298 is implemented. Fixes
+      a second case of bug 29122; bugfix on 0.4.0.1-alpha.
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+    - Instead of relying on hs_free_all() to clean up all onion service
+      objects in test_build_descriptors(), we now deallocate them one by
+      one. This lets Coverity know that we are not leaking memory there
+      and fixes CID 1442277. Fixes bug 28989; bugfix on 0.3.5.1-alpha.
+    - Check the time in the "Expires" header using approx_time(). Fixes
+      bug 30001; bugfix on 0.4.0.4-rc.
+
+  o Minor bugfixes (TLS protocol):
+    - When classifying a client's selection of TLS ciphers, if the
+      client ciphers are not yet available, do not cache the result.
+      Previously, we had cached the unavailability of the cipher list
+      and never looked again, which in turn led us to assume that the
+      client only supported the ancient V1 link protocol. This, in turn,
+      was causing Stem integration tests to stall in some cases. Fixes
+      bug 30021; bugfix on 0.2.4.8-alpha.
+
+  o Minor bugfixes (UI):
+    - Lower log level of unlink() errors during bootstrap. Fixes bug
+      29930; bugfix on 0.4.0.1-alpha.
+
+  o Minor bugfixes (usability):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+  o Minor bugfixes (Windows, CI):
+    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit
+      Windows Server 2012 R2 job. The remaining 2 jobs still provide
+      coverage of 64/32-bit, and Windows Server 2016/2012 R2. Also set
+      fast_finish, so failed jobs terminate the build immediately. Fixes
+      bug 29601; bugfix on 0.3.5.4-alpha.
+
+  o Code simplification and refactoring:
+    - Introduce a connection_dir_buf_add() helper function that detects
+      whether compression is in use, and adds a string accordingly.
+      Resolves issue 28816.
+    - Refactor handle_get_next_bandwidth() to use
+      connection_dir_buf_add(). Implements ticket 29897.
+    - Reimplement NETINFO cell parsing and generation to rely on
+      trunnel-generated wire format handling code. Closes ticket 27325.
+    - Remove unnecessary unsafe code from the Rust macro "cstr!". Closes
+      ticket 28077.
+    - Rework SOCKS wire format handling to rely on trunnel-generated
+      parsing/generation code. Resolves ticket 27620.
+    - Split out bootstrap progress reporting from control.c into a
+      separate file. Part of ticket 27402.
+    - The .may_include files that we use to describe our directory-by-
+      directory dependency structure now describe a noncircular
+      dependency graph over the directories that they cover. Our
+      checkIncludes.py tool now enforces this noncircularity. Closes
+      ticket 28362.
+
+  o Documentation:
+    - Clarify that Tor performs stream isolation among *Port listeners
+      by default. Resolves issue 29121.
+    - In the manpage entry describing MapAddress torrc setting, use
+      example IP addresses from ranges specified for use in documentation
+      by RFC 5737. Resolves issue 28623.
+    - Mention that you cannot add a new onion service if Tor is already
+      running with Sandbox enabled. Closes ticket 28560.
+    - Improve ControlPort documentation. Mention that it accepts
+      address:port pairs, and can be used multiple times. Closes
+      ticket 28805.
+    - Document the exact output of "tor --version". Closes ticket 28889.
+
+  o Removed features:
+    - Remove the old check-tor script. Resolves issue 29072.
+    - Stop responding to the 'GETINFO status/version/num-concurring' and
+      'GETINFO status/version/num-versioning' control port commands, as
+      those were deprecated back in 0.2.0.30. Also stop listing them in
+      output of 'GETINFO info/names'. Resolves ticket 28757.
+    - The scripts used to generate and maintain the list of fallback
+      directories have been extracted into a new "fallback-scripts"
+      repository. Closes ticket 27914.
+
+  o Testing:
+    - Run shellcheck for scripts in the in scripts/ directory. Closes
+      ticket 28058.
+    - Add unit tests for tokenize_string() and get_next_token()
+      functions. Resolves ticket 27625.
+
+  o Code simplification and refactoring (onion service v3):
+    - Consolidate the authorized client descriptor cookie computation
+      code from client and service into one function. Closes
+      ticket 27549.
+
+  o Code simplification and refactoring (shell scripts):
+    - Cleanup scan-build.sh to silence shellcheck warnings. Closes
+      ticket 28007.
+    - Fix issues that shellcheck found in chutney-git-bisect.sh.
+      Resolves ticket 28006.
+    - Fix issues that shellcheck found in updateRustDependencies.sh.
+      Resolves ticket 28012.
+    - Fix shellcheck warnings in cov-diff script. Resolves issue 28009.
+    - Fix shellcheck warnings in run_calltool.sh. Resolves ticket 28011.
+    - Fix shellcheck warnings in run_trunnel.sh. Resolves issue 28010.
+    - Fix shellcheck warnings in scripts/test/coverage. Resolves
+      issue 28008.
+
+
+Changes in version 0.3.5.8 - 2019-02-21
+  Tor 0.3.5.8 backports several fixes from later releases, including fixes
+  for an annoying SOCKS-parsing bug that affected users in earlier 0.3.5.x
+  releases.
+
+  It also includes a fix for a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Major bugfixes (networking, backport from 0.4.0.2-alpha):
+    - Gracefully handle empty username/password fields in SOCKS5
+      username/password auth messsage and allow SOCKS5 handshake to
+      continue. Previously, we had rejected these handshakes, breaking
+      certain applications. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
+
+  o Minor features (compilation, backport from 0.4.0.2-alpha):
+    - Compile correctly when OpenSSL is built with engine support
+      disabled, or with deprecated APIs disabled. Closes ticket 29026.
+      Patches from "Mangix".
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor features (testing, backport from 0.4.0.2-alpha):
+    - Treat all unexpected ERR and BUG messages as test failures. Closes
+      ticket 28668.
+
+  o Minor bugfixes (onion service v3, client, backport from 0.4.0.1-alpha):
+    - Stop logging a "BUG()" warning and stacktrace when we find a SOCKS
+      connection waiting for a descriptor that we actually have in the
+      cache. It turns out that this can actually happen, though it is
+      rare. Now, tor will recover and retry the descriptor. Fixes bug
+      28669; bugfix on 0.3.2.4-alpha.
+
+  o Minor bugfixes (IPv6, backport from 0.4.0.1-alpha):
+    - Fix tor_ersatz_socketpair on IPv6-only systems. Previously, the
+      IPv6 socket was bound using an address family of AF_INET instead
+      of AF_INET6. Fixes bug 28995; bugfix on 0.3.5.1-alpha. Patch from
+      Kris Katterjohn.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (client, clock skew, backport from 0.4.0.1-alpha):
+    - Select guards even if the consensus has expired, as long as the
+      consensus is still reasonably live. Fixes bug 24661; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (compilation, backport from 0.4.0.1-alpha):
+    - Compile correctly on OpenBSD; previously, we were missing some
+      headers required in order to detect it properly. Fixes bug 28938;
+      bugfix on 0.3.5.1-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (documentation, backport from 0.4.0.2-alpha):
+    - Describe the contents of the v3 onion service client authorization
+      files correctly: They hold public keys, not private keys. Fixes
+      bug 28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
+
+  o Minor bugfixes (logging, backport from 0.4.0.1-alpha):
+    - Rework rep_hist_log_link_protocol_counts() to iterate through all
+      link protocol versions when logging incoming/outgoing connection
+      counts. Tor no longer skips version 5, and we won't have to
+      remember to update this function when new link protocol version is
+      developed. Fixes bug 28920; bugfix on 0.2.6.10.
+
+  o Minor bugfixes (logging, backport from 0.4.0.2-alpha):
+    - Log more information at "warning" level when unable to read a
+      private key; log more information at "info" level when unable to
+      read a public key. We had warnings here before, but they were lost
+      during our NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (misc, backport from 0.4.0.2-alpha):
+    - The amount of total available physical memory is now determined
+      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
+      when it is defined and a 64-bit variant is not available. Fixes
+      bug 28981; bugfix on 0.2.5.4-alpha. Patch from Kris Katterjohn.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains more
+      than one private key for a hidden service. Fixes bug 29040; bugfix
+      on 0.3.5.1-alpha.
+    - In hs_cache_store_as_client() log an HSDesc we failed to parse at
+      "debug" level. Tor used to log it as a warning, which caused very
+      long log lines to appear for some users. Fixes bug 29135; bugfix
+      on 0.3.2.1-alpha.
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+  o Minor bugfixes (tests, directory clients, backport from 0.4.0.1-alpha):
+    - Mark outdated dirservers when Tor only has a reasonably live
+      consensus. Fixes bug 28569; bugfix on 0.3.2.5-alpha.
+
+  o Minor bugfixes (tests, backport from 0.4.0.2-alpha):
+    - Detect and suppress "bug" warnings from the util/time test on
+      Windows. Fixes bug 29161; bugfix on 0.2.9.3-alpha.
+    - Do not log an error-level message if we fail to find an IPv6
+      network interface from the unit tests. Fixes bug 29160; bugfix
+      on 0.2.7.3-rc.
+
+  o Minor bugfixes (usability, backport from 0.4.0.1-alpha):
+    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate().
+      Some users took this phrasing to mean that the mentioned guard was
+      under their control or responsibility, which it is not. Fixes bug
+      28895; bugfix on Tor 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.11 - 2019-02-21
+  Tor 0.3.4.11 is the third stable release in its series.  It includes
+  a fix for a medium-severity security bug affecting Tor 0.3.2.1-alpha and
+  later. All Tor instances running an affected release should upgrade to
+  0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.3.3.12 - 2019-02-21
+  Tor 0.3.3.12 fixes a medium-severity security bug affecting Tor
+  0.3.2.1-alpha and later. All Tor instances running an affected release
+  should upgrade to 0.3.3.12, 0.3.4.11, 0.3.5.8, or 0.4.0.2-alpha.
+
+  This release marks the end of support for the Tor 0.3.3.x series. We
+  recommend that users switch to either the Tor 0.3.4 series (supported
+  until at least 10 June 2019), or the Tor 0.3.5 series, which will
+  receive long-term support until at least 1 Feb 2022.
+
+  o Major bugfixes (cell scheduler, KIST, security):
+    - Make KIST consider the outbuf length when computing what it can
+      put in the outbuf. Previously, KIST acted as though the outbuf
+      were empty, which could lead to the outbuf becoming too full. It
+      is possible that an attacker could exploit this bug to cause a Tor
+      client or relay to run out of memory and crash. Fixes bug 29168;
+      bugfix on 0.3.2.1-alpha. This issue is also being tracked as
+      TROVE-2019-001 and CVE-2019-8955.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29478.
+
+  o Minor bugfixes (build, compatibility, rust, backport from 0.4.0.2-alpha):
+    - Update Cargo.lock file to match the version made by the latest
+      version of Rust, so that "make distcheck" will pass again. Fixes
+      bug 29244; bugfix on 0.3.3.4-alpha.
+
+  o Minor bugfixes (onion services, backport from 0.4.0.2-alpha):
+    - Stop logging "Tried to establish rendezvous on non-OR circuit..."
+      as a warning. Instead, log it as a protocol warning, because there
+      is nothing that relay operators can do to fix it. Fixes bug 29029;
+      bugfix on 0.2.5.7-rc.
+
+
+Changes in version 0.3.3.11 - 2019-01-07
+  Tor 0.3.3.11 backports numerous fixes from later versions of Tor.
+  numerous fixes, including an important fix for anyone using OpenSSL
+  1.1.1. Anyone running an earlier version of Tor 0.3.3 should upgrade
+  to this version, or to a later series.
+
+  As a reminder, support the Tor 0.3.3 series will end on 22 Feb 2019.
+  We anticipate that this will be the last release of Tor 0.3.3, unless
+  some major bug is before then. Some time between now and then, users
+  should switch to either the Tor 0.3.4 series (supported until at least
+  10 June 2019), or the Tor 0.3.5 series, which will receive long-term
+  support until at least 1 Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (restart-in-process, backport from 0.3.5.1-alpha):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (continuous integration, backport from 0.3.5.1-alpha):
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, backport from 0.3.5.3-alpha):
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (C correctness, backport from 0.3.5.4-alpha):
+    - Avoid undefined behavior in an end-of-string check when parsing
+      the BEGIN line in a directory object. Fixes bug 28202; bugfix
+      on 0.2.0.3-alpha.
+
+  o Minor bugfixes (code safety, backport from 0.3.5.3-alpha):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (directory authority, backport from 0.3.5.4-alpha):
+    - Log additional info when we get a relay that shares an ed25519 ID
+      with a different relay, instead making a BUG() warning. Fixes bug
+      27800; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.1-alpha):
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.2-alpha):
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to
+      an HTTP request we couldn't handle. Fixes bug 26470; bugfix on
+      0.3.2.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (netflow padding, backport from 0.3.5.1-alpha):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (protover, backport from 0.3.5.3-alpha):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.1-alpha):
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (rust, backport from 0.3.5.2-alpha):
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (rust, backport from 0.3.5.4-alpha):
+    - Fix a potential null dereference in protover_all_supported(). Add
+      a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
+    - Return a string that can be safely freed by C code, not one
+      created by the rust allocator, in protover_all_supported(). Fixes
+      bug 27740; bugfix on 0.3.3.1-alpha.
+    - Fix an API mismatch in the rust implementation of
+      protover_compute_vote(). This bug could have caused crashes on any
+      directory authorities running Tor with Rust (which we do not yet
+      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
+
+  o Minor bugfixes (testing, backport from 0.3.5.1-alpha):
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (testing, backport from 0.3.5.4-alpha):
+    - Treat backtrace test failures as expected on BSD-derived systems
+      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
+      (FreeBSD failures have been treated as expected since 18204 in
+      0.2.8.) Fixes bug 27948; bugfix on 0.2.5.2-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.4.10 - 2019-01-07
+  Tor 0.3.4.9 is the second stable release in its series; it backports
+  numerous fixes, including an important fix for relays, and for anyone
+  using OpenSSL 1.1.1. Anyone running an  earlier version of Tor 0.3.4
+  should upgrade.
+
+  As a reminder, the Tor 0.3.4 series will be supported until 10 June
+  2019. Some time between now and then, users should switch to the Tor
+  0.3.5 series, which will receive long-term support until at least 1
+  Feb 2022.
+
+  o Major bugfixes (OpenSSL, portability, backport from 0.3.5.5-alpha):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (relay, directory, backport from 0.3.5.7):
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Minor features (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+
+  o Minor features (controller, backport from 0.3.5.1-alpha):
+    - For purposes of CIRC_BW-based dropped cell detection, track half-
+      closed stream ids, and allow their ENDs, SENDMEs, DATA and path
+      bias check cells to arrive without counting it as dropped until
+      either the END arrives, or the windows are empty. Closes
+      ticket 25573.
+
+  o Minor features (fallback directory list, backport from 0.3.5.6-rc):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
+      Country database. Closes ticket 28395.
+
+  o Minor features (OpenSSL bug workaround, backport from 0.3.5.7):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor bugfixes (compilation, backport from 0.3.5.5-alpha):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (connection, relay, backport from 0.3.5.5-alpha):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.5-alpha):
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows, backport from 0.3.5.6-rc):
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+
+  o Minor bugfixes (directory permissions, backport form 0.3.5.3-alpha):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (memory leaks, backport from 0.3.5.5-alpha):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.3-alpha):
+    - Don't warn so loudly when Tor is unable to decode an onion
+      descriptor. This can now happen as a normal use case if a client
+      gets a descriptor with client authorization but the client is not
+      authorized. Fixes bug 27550; bugfix on 0.3.5.1-alpha.
+
+  o Minor bugfixes (onion service v3, backport from 0.3.5.6-rc):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (relay statistics, backport from 0.3.5.7):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (unit tests, guard selection, backport from 0.3.5.6-rc):
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+
+
+Changes in version 0.3.5.7 - 2019-01-07
+  Tor 0.3.5.7 is the first stable release in its series; it includes
+  compilation and portability fixes, and a fix for a severe problem
+  affecting directory caches.
+
+  The Tor 0.3.5 series includes several new features and performance
+  improvements, including client authorization for v3 onion services,
+  cleanups to bootstrap reporting, support for improved bandwidth-
+  measurement tools, experimental support for NSS in place of OpenSSL,
+  and much more. It also begins a full reorganization of Tor's code
+  layout, for improved modularity and maintainability in the future.
+  Finally, there is the usual set of performance improvements and
+  bugfixes that we try to do in every release series.
+
+  There are a couple of changes in the 0.3.5 that may affect
+  compatibility. First, the default version for newly created onion
+  services is now v3. Use the HiddenServiceVersion option if you want to
+  override this. Second, some log messages related to bootstrapping have
+  changed; if you use stem, you may need to update to the latest version
+  so it will recognize them.
+
+  We have designated 0.3.5 as a "long-term support" (LTS) series: we
+  will continue to patch major bugs in typical configurations of 0.3.5
+  until at least 1 Feb 2022. (We do not plan to provide long-term
+  support for embedding, Rust support, NSS support, running a directory
+  authority, or unsupported platforms. For these, you will need to stick
+  with the latest stable release.)
+
+  Below are the changes since 0.3.4.9. For a complete list of changes
+  since 0.3.5.6-rc, see the ChangeLog file.
+
+  o Major features (bootstrap):
+    - Don't report directory progress until after a connection to a
+      relay or bridge has succeeded. Previously, we'd report 80%
+      progress based on cached directory information when we couldn't
+      even connect to the network. Closes ticket 27169.
+
+  o Major features (new code layout):
+    - Nearly all of Tor's source code has been moved around into more
+      logical places. The "common" directory is now divided into a set
+      of libraries in "lib", and files in the "or" directory have been
+      split into "core" (logic absolutely needed for onion routing),
+      "feature" (independent modules in Tor), and "app" (to configure
+      and invoke the rest of Tor). See doc/HACKING/CodeStructure.md for
+      more information. Closes ticket 26481.
+
+      This refactoring is not complete: although the libraries have been
+      refactored to be acyclic, the main body of Tor is still too
+      interconnected. We will attempt to improve this in the future.
+
+  o Major features (onion services v3):
+    - Implement onion service client authorization at the descriptor
+      level: only authorized clients can decrypt a service's descriptor
+      to find out how to contact it. A new torrc option was added to
+      control this client side: ClientOnionAuthDir <path>. On the
+      service side, if the "authorized_clients/" directory exists in the
+      onion service directory path, client configurations are read from
+      the files within. See the manpage for more details. Closes ticket
+      27547. Patch done by Suphanat Chunhapanya (haxxpop).
+    - Improve revision counter generation in next-gen onion services.
+      Onion services can now scale by hosting multiple instances on
+      different hosts without synchronization between them, which was
+      previously impossible because descriptors would get rejected by
+      HSDirs. Addresses ticket 25552.
+    - Version 3 onion services can now use the per-service
+      HiddenServiceExportCircuitID option to differentiate client
+      circuits. It communicates with the service by using the HAProxy
+      protocol to assign virtual IP addresses to inbound client
+      circuits. Closes ticket 4700. Patch by Mahrud Sayrafi.
+
+  o Major features (onion services, UI change):
+    - For a newly created onion service, the default version is now 3.
+      Tor still supports existing version 2 services, but the operator
+      now needs to set "HiddenServiceVersion 2" in order to create a new
+      version 2 service. For existing services, Tor now learns the
+      version by reading the key file. Closes ticket 27215.
+
+  o Major features (portability, cryptography, experimental, TLS):
+    - Tor now has the option to compile with the NSS library instead of
+      OpenSSL. This feature is experimental, and we expect that bugs may
+      remain. It is mainly intended for environments where Tor's
+      performance is not CPU-bound, and where NSS is already known to be
+      installed. To try it out, configure Tor with the --enable-nss
+      flag. Closes tickets 26631, 26815, and 26816.
+
+      If you are experimenting with this option and using an old cached
+      consensus, Tor may fail to start. To solve this, delete your
+      "cached-consensus" and "cached-microdesc-consensus" files,
+      (if present), and restart Tor.
+
+  o Major features (relay, UI change):
+    - Relays no longer run as exits by default. If the "ExitRelay"
+      option is auto (or unset), and no exit policy is specified with
+      ExitPolicy or ReducedExitPolicy, we now treat ExitRelay as 0.
+      Previously in this case, we allowed exit traffic and logged a
+      warning message. Closes ticket 21530. Patch by Neel Chauhan.
+    - Tor now validates that the ContactInfo config option is valid UTF-
+      8 when parsing torrc. Closes ticket 27428.
+
+  o Major bugfixes (compilation):
+    - Fix compilation on ARM (and other less-used CPUs) when compiling
+      with OpenSSL before 1.1. Fixes bug 27781; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (compilation, rust):
+    - Rust tests can now build and run successfully with the
+      --enable-fragile-hardening option enabled. Doing this currently
+      requires the rust beta channel; it will be possible with stable
+      rust once Rust version 1.31 is released. Patch from Alex Crichton.
+      Fixes bugs 27272, 27273, and 27274. Bugfix on 0.3.1.1-alpha.
+
+  o Major bugfixes (directory authority):
+    - Actually check that the address we get from DirAuthority
+      configuration line is valid IPv4. Explicitly disallow DirAuthority
+      address to be a DNS hostname. Fixes bug 26488; bugfix
+      on 0.1.2.10-rc.
+
+  o Major bugfixes (embedding, main loop):
+    - When DisableNetwork becomes set, actually disable periodic events
+      that are already enabled. (Previously, we would refrain from
+      enabling new ones, but we would leave the old ones turned on.)
+      Fixes bug 28348; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (main loop, bootstrap):
+    - Make sure Tor bootstraps and works properly if only the
+      ControlPort is set. Prior to this fix, Tor would only bootstrap
+      when a client port was set (Socks, Trans, NATD, DNS or HTTPTunnel
+      port). Fixes bug 27849; bugfix on 0.3.4.1-alpha.
+
+  o Major bugfixes (onion service v3):
+    - On an intro point for a version 3 onion service, stop closing
+      introduction circuits on a NACK. This lets the client decide
+      whether to reuse the circuit or discard it. Previously, we closed
+      intro circuits when sending NACKs. Fixes bug 27841; bugfix on
+      0.3.2.1-alpha. Patch by Neel Chaunan.
+
+  o Major bugfixes (OpenSSL, portability):
+    - Fix our usage of named groups when running as a TLS 1.3 client in
+      OpenSSL 1.1.1. Previously, we only initialized EC groups when
+      running as a relay, which caused clients to fail to negotiate TLS
+      1.3 with relays. Fixes bug 28245; bugfix on 0.2.9.15 (when TLS 1.3
+      support was added).
+
+  o Major bugfixes (relay bandwidth statistics):
+    - When we close relayed circuits, report the data in the circuit
+      queues as being written in our relay bandwidth stats. This
+      mitigates guard discovery and other attacks that close circuits
+      for the explicit purpose of noticing this discrepancy in
+      statistics. Fixes bug 23512; bugfix on 0.0.8pre3.
+
+  o Major bugfixes (relay):
+    - When our write bandwidth limit is exhausted, stop writing on the
+      connection. Previously, we had a typo in the code that would make
+      us stop reading instead, leading to relay connections being stuck
+      indefinitely and consuming kernel RAM. Fixes bug 28089; bugfix
+      on 0.3.4.1-alpha.
+    - Always reactivate linked connections in the main loop so long as
+      any linked connection has been active. Previously, connections
+      serving directory information wouldn't get reactivated after the
+      first chunk of data was sent (usually 32KB), which would prevent
+      clients from bootstrapping. Fixes bug 28912; bugfix on
+      0.3.4.1-alpha. Patch by "cypherpunks3".
+
+  o Major bugfixes (restart-in-process):
+    - Fix a use-after-free error that could be caused by passing Tor an
+      impossible set of options that would fail during options_act().
+      Fixes bug 27708; bugfix on 0.3.3.1-alpha.
+
+  o Minor features (admin tools):
+    - Add a new --key-expiration option to print the expiration date of
+      the signing cert in an ed25519_signing_cert file. Resolves
+      issue 19506.
+
+  o Minor features (build):
+    - If you pass the "--enable-pic" option to configure, Tor will try
+      to tell the compiler to build position-independent code suitable
+      to link into a dynamic library. (The default remains -fPIE, for
+      code suitable for a relocatable executable.) Closes ticket 23846.
+
+  o Minor features (code correctness, testing):
+    - Tor's build process now includes a "check-includes" make target to
+      verify that no module of Tor relies on any headers from a higher-
+      level module. We hope to use this feature over time to help
+      refactor our codebase. Closes ticket 26447.
+
+  o Minor features (code layout):
+    - We have a new "lowest-level" error-handling API for use by code
+      invoked from within the logging module. With this interface, the
+      logging code is no longer at risk of calling into itself if a
+      failure occurs while it is trying to log something. Closes
+      ticket 26427.
+
+  o Minor features (compilation):
+    - When possible, place our warning flags in a separate file, to
+      avoid flooding verbose build logs. Closes ticket 28924.
+    - Tor's configure script now supports a --with-malloc= option to
+      select your malloc implementation. Supported options are
+      "tcmalloc", "jemalloc", "openbsd" (deprecated), and "system" (the
+      default). Addresses part of ticket 20424. Based on a patch from
+      Alex Xu.
+
+  o Minor features (config):
+    - The "auto" keyword in torrc is now case-insensitive. Closes
+      ticket 26663.
+
+  o Minor features (continuous integration):
+    - Add a Travis CI build for --enable-nss on Linux gcc. Closes
+      ticket 27751.
+    - Add new CI job to Travis configuration to run stem-based
+      integration tests. Closes ticket 27913.
+    - Use the Travis Homebrew addon to install packages on macOS during
+      Travis CI. The package list is the same, but the Homebrew addon
+      does not do a `brew update` by default. Implements ticket 27738.
+    - Report what program produced the mysterious core file that we
+      occasionally see on Travis CI during make distcheck. Closes
+      ticket 28024.
+    - Don't do a distcheck with --disable-module-dirauth in Travis.
+      Implements ticket 27252.
+    - Install libcap-dev and libseccomp2-dev so these optional
+      dependencies get tested on Travis CI. Closes ticket 26560.
+    - Only run one online rust build in Travis, to reduce network
+      errors. Skip offline rust builds on Travis for Linux gcc, because
+      they're redundant. Implements ticket 27252.
+    - Skip gcc on OSX in Travis CI, because it's rarely used. Skip a
+      duplicate hardening-off build in Travis on Tor 0.2.9. Skip gcc on
+      Linux with default settings, because all the non-default builds
+      use gcc on Linux. Implements ticket 27252.
+
+  o Minor features (continuous integration, Windows):
+    - Always show the configure and test logs, and upload them as build
+      artifacts, when building for Windows using Appveyor CI.
+      Implements 28459.
+    - Build tor on Windows Server 2012 R2 and Windows Server 2016 using
+      Appveyor's CI. Closes ticket 28318.
+
+  o Minor features (controller):
+    - Emit CIRC_BW events as soon as we detect that we processed an
+      invalid or otherwise dropped cell on a circuit. This allows
+      vanguards and other controllers to react more quickly to dropped
+      cells. Closes ticket 27678.
+    - For purposes of CIRC_BW-based dropped cell detection, track half-
+      closed stream ids, and allow their ENDs, SENDMEs, DATA and path
+      bias check cells to arrive without counting it as dropped until
+      either the END arrives, or the windows are empty. Closes
+      ticket 25573.
+    - Implement a 'GETINFO md/all' controller command to enable getting
+      all known microdescriptors. Closes ticket 8323.
+    - The GETINFO command now support an "uptime" argument, to return
+      Tor's uptime in seconds. Closes ticket 25132.
+
+  o Minor features (denial-of-service avoidance):
+    - Make our OOM handler aware of the DNS cache so that it doesn't
+      fill up the memory. This check is important for our DoS mitigation
+      subsystem. Closes ticket 18642. Patch by Neel Chauhan.
+
+  o Minor features (development):
+    - Tor's makefile now supports running the "clippy" Rust style tool
+      on our Rust code. Closes ticket 22156.
+
+  o Minor features (directory authority):
+    - There is no longer an artificial upper limit on the length of
+      bandwidth lines. Closes ticket 26223.
+    - When a bandwidth file is used to obtain the bandwidth measurements,
+      include this bandwidth file headers in the votes. Closes
+      ticket 3723.
+    - Improved support for networks with only a single authority or a
+      single fallback directory. Patch from Gabriel Somlo. Closes
+      ticket 25928.
+
+  o Minor features (embedding API):
+    - The Tor controller API now supports a function to launch Tor with
+      a preconstructed owning controller FD, so that embedding
+      applications don't need to manage controller ports and
+      authentication. Closes ticket 24204.
+    - The Tor controller API now has a function that returns the name
+      and version of the backend implementing the API. Closes
+      ticket 26947.
+
+  o Minor features (fallback directory list):
+    - Replace the 150 fallbacks originally introduced in Tor
+      0.3.3.1-alpha in January 2018 (of which ~115 were still
+      functional), with a list of 157 fallbacks (92 new, 65 existing, 85
+      removed) generated in December 2018. Closes ticket 24803.
+
+  o Minor features (geoip):
+    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
+      Country database. Closes ticket 29012.
+
+  o Minor features (memory management):
+    - Get Libevent to use the same memory allocator as Tor, by calling
+      event_set_mem_functions() during initialization. Resolves
+      ticket 8415.
+
+  o Minor features (memory usage):
+    - When not using them, store legacy TAP public onion keys in DER-
+      encoded format, rather than as expanded public keys. This should
+      save several megabytes on typical clients. Closes ticket 27246.
+
+  o Minor features (OpenSSL bug workaround):
+    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
+      key export function from handling long labels. When this bug is
+      detected, Tor will disable TLS 1.3. We recommend upgrading to a
+      version of OpenSSL without this bug when it becomes available.
+      Closes ticket 28973.
+
+  o Minor features (OpenSSL):
+    - When possible, use RFC5869 HKDF implementation from OpenSSL rather
+      than our own. Resolves ticket 19979.
+
+  o Minor features (performance):
+    - Remove about 96% of the work from the function that we run at
+      startup to test our curve25519_basepoint implementation. Since
+      this function has yet to find an actual failure, we now only run
+      it for 8 iterations instead of 200. Based on our profile
+      information, this change should save around 8% of our startup time
+      on typical desktops, and may have a similar effect on other
+      platforms. Closes ticket 28838.
+    - Stop re-validating our hardcoded Diffie-Hellman parameters on
+      every startup. Doing this wasted time and cycles, especially on
+      low-powered devices. Closes ticket 28851.
+
+  o Minor features (Rust, code quality):
+    - Improve rust code quality in the rust protover implementation by
+      making it more idiomatic. Includes changing an internal API to
+      take &str instead of &String. Closes ticket 26492.
+
+  o Minor features (testing):
+    - Add scripts/test/chutney-git-bisect.sh, for bisecting using
+      chutney. Implements ticket 27211.
+
+  o Minor features (tor-resolve):
+    - The tor-resolve utility can now be used with IPv6 SOCKS proxies.
+      Side-effect of the refactoring for ticket 26526.
+
+  o Minor features (UI):
+    - Log each included configuration file or directory as we read it,
+      to provide more visibility about where Tor is reading from. Patch
+      from Unto Sten; closes ticket 27186.
+    - Lower log level of "Scheduler type KIST has been enabled" to INFO.
+      Closes ticket 26703.
+
+  o Minor bugfixes (32-bit OSX and iOS, timing):
+    - Fix an integer overflow bug in our optimized 32-bit millisecond-
+      difference algorithm for 32-bit Apple platforms. Previously, it
+      would overflow when calculating the difference between two times
+      more than 47 days apart. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+    - Improve the precision of our 32-bit millisecond difference
+      algorithm for 32-bit Apple platforms. Fixes part of bug 27139;
+      bugfix on 0.3.4.1-alpha.
+    - Relax the tolerance on the mainloop/update_time_jumps test when
+      running on 32-bit Apple platforms. Fixes part of bug 27139; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (bootstrap):
+    - Try harder to get descriptors in non-exit test networks, by using
+      the mid weight for the third hop when there are no exits. Fixes
+      bug 27237; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (C correctness):
+    - Avoid casting smartlist index to int implicitly, as it may trigger
+      a warning (-Wshorten-64-to-32). Fixes bug 26282; bugfix on
+      0.2.3.13-alpha, 0.2.7.1-alpha and 0.2.1.1-alpha.
+    - Use time_t for all values in
+      predicted_ports_prediction_time_remaining(). Rework the code that
+      computes difference between durations/timestamps. Fixes bug 27165;
+      bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (client, memory usage):
+    - When not running as a directory cache, there is no need to store
+      the text of the current consensus networkstatus in RAM.
+      Previously, however, clients would store it anyway, at a cost of
+      over 5 MB. Now, they do not. Fixes bug 27247; bugfix
+      on 0.3.0.1-alpha.
+
+  o Minor bugfixes (client, ReachableAddresses):
+    - Instead of adding a "reject *:*" line to ReachableAddresses when
+      loading the configuration, add one to the policy after parsing it
+      in parse_reachable_addresses(). This prevents extra "reject *.*"
+      lines from accumulating on reloads. Fixes bug 20874; bugfix on
+      0.1.1.5-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (code quality):
+    - Rename sandbox_getaddrinfo() and other functions to no longer
+      misleadingly suggest that they are sandbox-only. Fixes bug 26525;
+      bugfix on 0.2.7.1-alpha.
+
+  o Minor bugfixes (code safety):
+    - Rewrite our assertion macros so that they no longer suppress the
+      compiler's -Wparentheses warnings. Fixes bug 27709; bugfix
+      on 0.0.6.
+
+  o Minor bugfixes (compilation):
+    - Initialize a variable unconditionally in aes_new_cipher(), since
+      some compilers cannot tell that we always initialize it before
+      use. Fixes bug 28413; bugfix on 0.2.9.3-alpha.
+
+  o Minor bugfixes (configuration):
+    - Refuse to start with relative file paths and RunAsDaemon set
+      (regression from the fix for bug 22731). Fixes bug 28298; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (configuration, Onion Services):
+    - In rend_service_parse_port_config(), disallow any input to remain
+      after address-port pair was parsed. This will catch address and
+      port being whitespace-separated by mistake of the user. Fixes bug
+      27044; bugfix on 0.2.9.10.
+
+  o Minor bugfixes (connection, relay):
+    - Avoid a logging a BUG() stacktrace when closing connection held
+      open because the write side is rate limited but not the read side.
+      Now, the connection read side is simply shut down until Tor is
+      able to flush the connection and close it. Fixes bug 27750; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (continuous integration, Windows):
+    - Stop reinstalling identical packages in our Windows CI. Fixes bug
+      27464; bugfix on 0.3.4.1-alpha.
+    - Install only the necessary mingw packages during our appveyor
+      builds. This change makes the build a little faster, and prevents
+      a conflict with a preinstalled mingw openssl that appveyor now
+      ships. Fixes bugs 27765 and 27943; bugfix on 0.3.4.2-alpha.
+    - Explicitly specify the path to the OpenSSL library and do not
+      download OpenSSL from Pacman, but instead use the library that is
+      already provided by AppVeyor. Fixes bug 28574; bugfix on master.
+    - Manually configure the zstd compiler options, when building using
+      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does
+      not come with a pkg-config file. Fixes bug 28454; bugfix
+      on 0.3.4.1-alpha.
+    - Stop using an external OpenSSL install, and stop installing MSYS2
+      packages, when building using mingw on Appveyor Windows CI. Fixes
+      bug 28399; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (controller):
+    - Consider all routerinfo errors other than "not a server" to be
+      transient for the purpose of "GETINFO exit-policy/*" controller
+      request. Print stacktrace in the unlikely case of failing to
+      recompute routerinfo digest. Fixes bug 27034; bugfix
+      on 0.3.4.1-alpha.
+
+  o Minor bugfixes (correctness):
+    - Fix an unreached code path where we checked the value of
+      "hostname" inside send_resolved_hostname_cell(). Previously, we
+      used it before checking it; now we check it first. Fixes bug
+      28879; bugfix on 0.1.2.7-alpha.
+
+  o Minor bugfixes (directory connection shutdown):
+    - Avoid a double-close when shutting down a stalled directory
+      connection. Fixes bug 26896; bugfix on 0.3.4.1-alpha.
+
+  o Minor bugfixes (directory permissions):
+    - When a user requests a group-readable DataDirectory, give it to
+      them. Previously, when the DataDirectory and the CacheDirectory
+      were the same, the default setting (0) for
+      CacheDirectoryGroupReadable would override the setting for
+      DataDirectoryGroupReadable. Fixes bug 26913; bugfix
+      on 0.3.3.1-alpha.
+
+  o Minor bugfixes (HTTP tunnel):
+    - Fix a bug warning when closing an HTTP tunnel connection due to an
+      HTTP request we couldn't handle. Fixes bug 26470; bugfix
+      on 0.3.2.1-alpha.
+
+  o Minor bugfixes (ipv6):
+    - In addrs_in_same_network_family(), we choose the subnet size based
+      on the IP version (IPv4 or IPv6). Previously, we chose a fixed
+      subnet size of /16 for both IPv4 and IPv6 addresses. Fixes bug
+      15518; bugfix on 0.2.3.1-alpha. Patch by Neel Chauhan.
+
+  o Minor bugfixes (Linux seccomp2 sandbox):
+    - Permit the "shutdown()" system call, which is apparently used by
+      OpenSSL under some circumstances. Fixes bug 28183; bugfix
+      on 0.2.5.1-alpha.
+
+  o Minor bugfixes (logging):
+    - Stop talking about the Named flag in log messages. Clients have
+      ignored the Named flag since 0.3.2. Fixes bug 28441; bugfix
+      on 0.3.2.1-alpha.
+    - As a precaution, do an early return from log_addr_has_changed() if
+      Tor is running as client. Also, log a stack trace for debugging as
+      this function should only be called when Tor runs as server. Fixes
+      bug 26892; bugfix on 0.1.1.9-alpha.
+    - Refrain from mentioning bug 21018 in the logs, as it is already
+      fixed. Fixes bug 25477; bugfix on 0.2.9.8.
+
+  o Minor bugfixes (logging, documentation):
+    - When SafeLogging is enabled, scrub IP address in
+      channel_tls_process_netinfo_cell(). Also, add a note to manpage
+      that scrubbing is not guaranteed on loglevels below Notice. Fixes
+      bug 26882; bugfix on 0.2.4.10-alpha.
+
+  o Minor bugfixes (memory leaks):
+    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
+      bugfix on 0.3.3.1-alpha. Patch from Martin Kepplinger.
+    - Fix a small memory leak when calling Tor with --dump-config. Fixes
+      bug 27893; bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (netflow padding):
+    - Ensure circuitmux queues are empty before scheduling or sending
+      padding. Fixes bug 25505; bugfix on 0.3.1.1-alpha.
+
+  o Minor bugfixes (onion service v2):
+    - Log at level "info", not "warning", in the case that we do not
+      have a consensus when a .onion request comes in. This can happen
+      normally while bootstrapping. Fixes bug 27040; bugfix
+      on 0.2.8.2-alpha.
+
+  o Minor bugfixes (onion service v3):
+    - When deleting an ephemeral onion service (DEL_ONION), do not close
+      any rendezvous circuits in order to let the existing client
+      connections finish by themselves or closed by the application. The
+      HS v2 is doing that already so now we have the same behavior for
+      all versions. Fixes bug 28619; bugfix on 0.3.3.1-alpha.
+    - Build the service descriptor's signing key certificate before
+      uploading, so we always have a fresh one: leaving no chances for
+      it to expire service side. Fixes bug 27838; bugfix
+      on 0.3.2.1-alpha.
+    - Stop dumping a stack trace when trying to connect to an intro
+      point without having a descriptor for it. Fixes bug 27774; bugfix
+      on 0.3.2.1-alpha.
+    - When selecting a v3 rendezvous point, don't only look at the
+      protover, but also check whether the curve25519 onion key is
+      present. This way we avoid picking a relay that supports the v3
+      rendezvous but for which we don't have the microdescriptor. Fixes
+      bug 27797; bugfix on 0.3.2.1-alpha.
+    - Close all SOCKS request (for the same .onion) if the newly fetched
+      descriptor is unusable. Before that, we would close only the first
+      one leaving the other hanging and let to time out by themselves.
+      Fixes bug 27410; bugfix on 0.3.2.1-alpha.
+    - When the onion service directory can't be created or has the wrong
+      permissions, do not log a stack trace. Fixes bug 27335; bugfix
+      on 0.3.2.1-alpha.
+    - When replacing a descriptor in the client cache, make sure to
+      close all client introduction circuits for the old descriptor, so
+      we don't end up with unusable leftover circuits. Fixes bug 27471;
+      bugfix on 0.3.2.1-alpha.
+
+  o Minor bugfixes (OS compatibility):
+    - Properly handle configuration changes that move a listener to/from
+      wildcard IP address. If the first attempt to bind a socket fails,
+      close the old listener and try binding the socket again. Fixes bug
+      17873; bugfix on 0.0.8pre-1.
+
+  o Minor bugfixes (performance)::
+    - Rework node_is_a_configured_bridge() to no longer call
+      node_get_all_orports(), which was performing too many memory
+      allocations. Fixes bug 27224; bugfix on 0.2.3.9.
+
+  o Minor bugfixes (protover):
+    - Reject protocol names containing bytes other than alphanumeric
+      characters and hyphens ([A-Za-z0-9-]). Fixes bug 27316; bugfix
+      on 0.2.9.4-alpha.
+
+  o Minor bugfixes (protover, rust):
+    - Reject extra commas in version strings. Fixes bug 27197; bugfix
+      on 0.3.3.3-alpha.
+    - protover_all_supported() would attempt to allocate up to 16GB on
+      some inputs, leading to a potential memory DoS. Fixes bug 27206;
+      bugfix on 0.3.3.5-rc.
+    - Compute protover votes correctly in the rust version of the
+      protover code. Previously, the protover rewrite in 24031 allowed
+      repeated votes from the same voter for the same protocol version
+      to be counted multiple times in protover_compute_vote(). Fixes bug
+      27649; bugfix on 0.3.3.5-rc.
+    - Reject protover names that contain invalid characters. Fixes bug
+      27687; bugfix on 0.3.3.1-alpha.
+
+  o Minor bugfixes (relay shutdown, systemd):
+    - Notify systemd of ShutdownWaitLength so it can be set to longer
+      than systemd's TimeoutStopSec. In Tor's systemd service file, set
+      TimeoutSec to 60 seconds to allow Tor some time to shut down.
+      Fixes bug 28113; bugfix on 0.2.6.2-alpha.
+
+  o Minor bugfixes (relay statistics):
+    - Update relay descriptor on bandwidth changes only when the uptime
+      is smaller than 24h, in order to reduce the efficiency of guard
+      discovery attacks. Fixes bug 24104; bugfix on 0.1.1.6-alpha.
+
+  o Minor bugfixes (relay):
+    - Consider the fact that we'll be making direct connections to our
+      entry and guard nodes when computing the fraction of nodes that
+      have their descriptors. Also, if we are using bridges and there is
+      at least one bridge with a full descriptor, treat the fraction of
+      guards available as 100%. Fixes bug 25886; bugfix on 0.2.4.10-alpha.
+      Patch by Neel Chauhan.
+    - Update the message logged on relays when DirCache is disabled.
+      Since 0.3.3.5-rc, authorities require DirCache (V2Dir) for the
+      Guard flag. Fixes bug 24312; bugfix on 0.3.3.5-rc.
+
+  o Minor bugfixes (testing):
+    - Stop running stem's unit tests as part of "make test-stem", but
+      continue to run stem's unit and online tests during "make test-
+      stem-full". Fixes bug 28568; bugfix on 0.2.6.3-alpha.
+    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
+      bugfix on 0.3.0.1-alpha.
+    - Make the hs_service tests use the same time source when creating
+      the introduction point and when testing it. Now tests work better
+      on very slow systems like ARM or Travis. Fixes bug 27810; bugfix
+      on 0.3.2.1-alpha.
+    - Revise the "conditionvar_timeout" test so that it succeeds even on
+      heavily loaded systems where the test threads are not scheduled
+      within 200 msec. Fixes bug 27073; bugfix on 0.2.6.3-alpha.
+    - Fix two unit tests to work when HOME environment variable is not
+      set. Fixes bug 27096; bugfix on 0.2.8.1-alpha.
+    - If a unit test running in a subprocess exits abnormally or with a
+      nonzero status code, treat the test as having failed, even if the
+      test reported success. Without this fix, memory leaks don't cause
+      the tests to fail, even with LeakSanitizer. Fixes bug 27658;
+      bugfix on 0.2.2.4-alpha.
+    - When logging a version mismatch in our openssl_version tests,
+      report the actual offending version strings. Fixes bug 26152;
+      bugfix on 0.2.9.1-alpha.
+    - Fix forking tests on Windows when there is a space somewhere in
+      the path. Fixes bug 26437; bugfix on 0.2.2.4-alpha.
+
+  o Minor bugfixes (Windows):
+    - Correctly identify Windows 8.1, Windows 10, and Windows Server
+      2008 and later from their NT versions. Fixes bug 28096; bugfix on
+      0.2.2.34; reported by Keifer Bly.
+    - On recent Windows versions, the GetVersionEx() function may report
+      an earlier Windows version than the running OS. To avoid user
+      confusion, add "[or later]" to Tor's version string on affected
+      versions of Windows. Fixes bug 28096; bugfix on 0.2.2.34; reported
+      by Keifer Bly.
+    - Remove Windows versions that were never supported by the
+      GetVersionEx() function. Stop duplicating the latest Windows
+      version in get_uname(). Fixes bug 28096; bugfix on 0.2.2.34;
+      reported by Keifer Bly.
+
+  o Code simplification and refactoring:
+    - When parsing a port configuration, make it more obvious to static
+      analyzer tools that we always initialize the address. Closes
+      ticket 28881.
+    - Divide more large Tor source files -- especially ones that span
+      multiple areas of functionality -- into smaller parts, including
+      onion.c and main.c. Closes ticket 26747.
+    - Divide the "routerparse.c" module into separate modules for each
+      group of parsed objects. Closes ticket 27924.
+    - Move protover_rust.c to the same place protover.c was moved to.
+      Closes ticket 27814.
+    - Split directory.c into separate pieces for client, server, and
+      common functionality. Closes ticket 26744.
+    - Split the non-statistics-related parts from the rephist.c and
+      geoip.c modules. Closes ticket 27892.
+    - Split the router.c file into relay-only and shared components, to
+      help with future modularization. Closes ticket 27864.
+    - Divide the routerlist.c and dirserv.c modules into smaller parts.
+      Closes ticket 27799.
+    - 'updateFallbackDirs.py' now ignores the blacklist file, as it's not
+      longer needed. Closes ticket 26502.
+    - Include paths to header files within Tor are now qualified by
+      directory within the top-level src directory.
+    - Many structures have been removed from the centralized "or.h"
+      header, and moved into their own headers. This will allow us to
+      reduce the number of places in the code that rely on each
+      structure's contents and layout. Closes ticket 26383.
+    - Remove ATTR_NONNULL macro from codebase. Resolves ticket 26527.
+    - Remove GetAdaptersAddresses_fn_t. The code that used it was
+      removed as part of the 26481 refactor. Closes ticket 27467.
+    - Rework Tor SOCKS server code to use Trunnel and benefit from
+      autogenerated functions for parsing and generating SOCKS wire
+      format. New implementation is cleaner, more maintainable and
+      should be less prone to heartbleed-style vulnerabilities.
+      Implements a significant fraction of ticket 3569.
+    - Split sampled_guards_update_from_consensus() and
+      select_entry_guard_for_circuit() into subfunctions. In
+      entry_guards_update_primary() unite three smartlist enumerations
+      into one and move smartlist comparison code out of the function.
+      Closes ticket 21349.
+    - Tor now assumes that you have standards-conformant stdint.h and
+      inttypes.h headers when compiling. Closes ticket 26626.
+    - Unify our bloom filter logic. Previously we had two copies of this
+      code: one for routerlist filtering, and one for address set
+      calculations. Closes ticket 26510.
+    - Use the simpler strcmpstart() helper in
+      rend_parse_v2_service_descriptor instead of strncmp(). Closes
+      ticket 27630.
+    - Utility functions that can perform a DNS lookup are now wholly
+      separated from those that can't, in separate headers and C
+      modules. Closes ticket 26526.
+
+  o Documentation:
+    - In the tor-resolve(1) manpage, fix the reference to socks-
+      extensions.txt by adding a web URL. Resolves ticket 27853.
+    - Mention that we require Python to be 2.7 or newer for some
+      integration tests that we ship with Tor. Resolves ticket 27677.
+    - Copy paragraph and URL to Tor's code of conduct document from
+      CONTRIBUTING to new CODE_OF_CONDUCT file. Resolves ticket 26638.
+    - Remove old instructions from INSTALL document. Closes ticket 26588.
+    - Warn users that they should not include MyFamily line(s) in their
+      torrc when running Tor bridge. Closes ticket 26908.
+
+  o Removed features:
+    - Tor no longer supports building with the dmalloc library. For
+      debugging memory issues, we suggest using gperftools or msan
+      instead. Closes ticket 26426.
+    - Tor no longer attempts to run on Windows environments without the
+      GetAdaptersAddresses() function. This function has existed since
+      Windows XP, which is itself already older than we support.
+    - Remove Tor2web functionality for version 2 onion services. The
+      Tor2webMode and Tor2webRendezvousPoints options are now obsolete.
+      (This feature was never shipped in vanilla Tor and it was only
+      possible to use this feature by building the support at compile
+      time. Tor2webMode is not implemented for version 3 onion services.)
+      Closes ticket 26367.
+
+  o Testing:
+    - Increase logging and tag all log entries with timestamps in
+      test_rebind.py. Provides diagnostics for issue 28229.
+
+  o Code simplification and refactoring (shared random, dirauth):
+    - Change many tor_assert() to use BUG() instead. The idea is to not
+      crash a dirauth but rather scream loudly with a stacktrace and let
+      it continue run. The shared random subsystem is very resilient and
+      if anything wrong happens with it, at worst a non coherent value
+      will be put in the vote and discarded by the other authorities.
+      Closes ticket 19566.
+
+  o Documentation (onion services):
+    - Improve HSv3 client authorization by making some options more
+      explicit and detailed. Closes ticket 28026. Patch by Mike Tigas.
+    - Document in the man page that changing ClientOnionAuthDir value or
+      adding a new file in the directory will not work at runtime upon
+      sending a HUP if Sandbox 1. Closes ticket 28128.
+    - Note in the man page that the only real way to fully revoke an
+      onion service v3 client authorization is by restarting the tor
+      process. Closes ticket 28275.
+
+
 Changes in version 0.3.4.9 - 2018-11-02
   Tor 0.3.4.9 is the second stable release in its series; it backports
   numerous fixes, including a fix for a bandwidth management bug that
@@ -1034,7 +3469,7 @@ Changes in version 0.3.4.8 - 2018-09-10
     - Remove duplicate code in parse_{c,s}method_line and bootstrap
       their functionalities into a single function. Fixes bug 6236;
       bugfix on 0.2.3.6-alpha.
-    - We remove the PortForwsrding and PortForwardingHelper options,
+    - We remove the PortForwarding and PortForwardingHelper options,
       related functions, and the port_forwarding tests. These options
       were used by the now-deprecated Vidalia to help ordinary users
       become Tor relays or bridges. Closes ticket 25409. Patch by
diff --git a/autogen.sh b/autogen.sh
index 276dd4047c..63ef6d49ef 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,9 +1,9 @@
 #!/bin/sh
 
-if [ -x "`which autoreconf 2>/dev/null`" ] ; then
+if command -v autoreconf; then
   opt="-i -f -W all,error"
 
-  for i in $@; do
+  for i in "$@"; do
     case "$i" in
       -v)
         opt="${opt} -v"
@@ -11,6 +11,7 @@ if [ -x "`which autoreconf 2>/dev/null`" ] ; then
     esac
   done
 
+  # shellcheck disable=SC2086
   exec autoreconf $opt
 fi
 
diff --git a/changes/29241_diagnostic b/changes/29241_diagnostic
deleted file mode 100644
index 1e38654957..0000000000
--- a/changes/29241_diagnostic
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (NSS, diagnostic):
-    - Try to log an error from NSS (if there is any) and a more useful
-      description of our situation if we are using NSS and a call to
-      SSL_ExportKeyingMaterial() fails.  Diagnostic for ticket 29241.
diff --git a/changes/bug12399 b/changes/bug12399
deleted file mode 100644
index 922c08c5e3..0000000000
--- a/changes/bug12399
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (logging):
-    - Change log level of message "Hash of session info was not as expected"
-      to LOG_PROTOCOL_WARN. Fixes bug 12399; bugfix on 0.1.1.10-alpha.
diff --git a/changes/bug13221 b/changes/bug13221
deleted file mode 100644
index 13935a1921..0000000000
--- a/changes/bug13221
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Correct a misleading error message when IPv4Only or IPv6Only
-      is used but the resolved address can not be interpreted as an
-      address of the specified IP version.  Fixes bug 13221; bugfix
-      on 0.2.3.9-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug22619 b/changes/bug22619
deleted file mode 100644
index 9c71996f5b..0000000000
--- a/changes/bug22619
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (circuit isolation):
-    - Fix a logic error that prevented the SessionGroup sub-option from
-      being accepted. Fixes bug 22619; bugfix on 0.2.7.2-alpha.
diff --git a/changes/bug23507 b/changes/bug23507
deleted file mode 100644
index de18273fdb..0000000000
--- a/changes/bug23507
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (v3 single onion services):
-    - Make v3 single onion services fall back to a 3-hop intro, when there
-      all intro points are unreachable via a 1-hop path. Previously, v3
-      single onion services failed when all intro nodes were unreachable
-      via a 1-hop path. Fixes bug 23507; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug23818_v2 b/changes/bug23818_v2
deleted file mode 100644
index 0219a20f49..0000000000
--- a/changes/bug23818_v2
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (v2 single onion services):
-    - Always retry v2 single onion service intro and rend circuits with a
-      3-hop path. Previously, v2 single onion services used a 3-hop path
-      when rend circuits were retried after a remote or delayed failure,
-      but a 1-hop path for immediate retries. Fixes bug 23818;
-      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug23818_v3 b/changes/bug23818_v3
deleted file mode 100644
index c430144d81..0000000000
--- a/changes/bug23818_v3
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (v3 single onion services):
-    - Always retry v3 single onion service intro and rend circuits with a
-      3-hop path. Previously, v3 single onion services used a 3-hop path
-      when rend circuits were retried after a remote or delayed failure,
-      but a 1-hop path for immediate retries. Fixes bug 23818;
-      bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug24661 b/changes/bug24661
deleted file mode 100644
index a915a93e0e..0000000000
--- a/changes/bug24661
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (client, guard selection):
-    - When Tor's consensus has expired, but is still reasonably live, use it
-      to select guards. Fixes bug 24661; bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug27197 b/changes/bug27197
deleted file mode 100644
index e389f85065..0000000000
--- a/changes/bug27197
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (protover, rust):
-    - Reject extra commas in version string. Fixes bug 27197; bugfix on
-      0.3.3.3-alpha.
diff --git a/changes/bug27199 b/changes/bug27199
deleted file mode 100644
index f9d2a422f9..0000000000
--- a/changes/bug27199
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (rust):
-    - Abort on panic in all build profiles, instead of potentially unwinding
-      into C code. Fixes bug 27199; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27740 b/changes/bug27740
deleted file mode 100644
index 76a17b7dda..0000000000
--- a/changes/bug27740
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (rust):
-    - Return a string that can be safely freed by C code, not one created by
-      the rust allocator, in protover_all_supported(). Fixes bug 27740; bugfix
-      on 0.3.3.1-alpha.
diff --git a/changes/bug27741 b/changes/bug27741
deleted file mode 100644
index 531e264b63..0000000000
--- a/changes/bug27741
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (rust, directory authority):
-    - Fix an API mismatch in the rust implementation of
-      protover_compute_vote(). This bug could have caused crashes on any
-      directory authorities running Tor with Rust (which we do not yet
-      recommend). Fixes bug 27741; bugfix on 0.3.3.6.
diff --git a/changes/bug27750 b/changes/bug27750
deleted file mode 100644
index c234788b1c..0000000000
--- a/changes/bug27750
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (connection, relay):
-    - Avoid a wrong BUG() stacktrace in case a closing connection is being held
-      open because the write side is rate limited but not the read side. Now,
-      the connection read side is simply shutdown instead of kept open until tor
-      is able to flush the connection and then fully close it. Fixes bug 27750;
-      bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug27800 b/changes/bug27800
deleted file mode 100644
index 63d5dbc681..0000000000
--- a/changes/bug27800
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (directory authority):
-    - Log additional info when we get a relay that shares an ed25519
-      ID with a different relay, instead making a BUG() warning.
-      Fixes bug 27800; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug27804 b/changes/bug27804
deleted file mode 100644
index fa7fec0bc5..0000000000
--- a/changes/bug27804
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (rust):
-    - Fix a potential null dereference in protover_all_supported().
-      Add a test for it. Fixes bug 27804; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug27841 b/changes/bug27841
deleted file mode 100644
index 9cd1da7275..0000000000
--- a/changes/bug27841
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (onion services):
-    - On an intro point for a version 3 onion service, we do not close
-      an introduction circuit on an NACK. This lets the client decide
-      whether to reuse the circuit or discard it. Previously, we closed
-      intro circuits on NACKs. Fixes bug 27841; bugfix on 0.3.2.1-alpha.
-      Patch by Neel Chaunan
-
diff --git a/changes/bug27948 b/changes/bug27948
deleted file mode 100644
index fea16f3d0f..0000000000
--- a/changes/bug27948
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (tests):
-    - Treat backtrace test failures as expected on BSD-derived systems
-      (NetBSD, OpenBSD, and macOS/Darwin) until we solve bug 17808.
-      (FreeBSD failures have been treated as expected since 18204 in 0.2.8.)
-      Fixes bug 27948; bugfix on 0.2.5.2-alpha.
-
diff --git a/changes/bug27963_timeradd b/changes/bug27963_timeradd
deleted file mode 100644
index 34b361cf8d..0000000000
--- a/changes/bug27963_timeradd
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation, opensolaris):
-    - Add a missing include to compat_pthreads.c, to fix compilation
-      on OpenSolaris and its descendants. Fixes bug 27963; bugfix
-      on 0.3.5.1-alpha.
diff --git a/changes/bug27968 b/changes/bug27968
deleted file mode 100644
index 78c8eee33a..0000000000
--- a/changes/bug27968
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (testing):
-    - Avoid hangs and race conditions in test_rebind.py.
-      Fixes bug 27968; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28096 b/changes/bug28096
deleted file mode 100644
index 6847df9798..0000000000
--- a/changes/bug28096
+++ /dev/null
@@ -1,13 +0,0 @@
-  o Minor bugfixes (Windows):
-    - Correctly identify Windows 8.1, Windows 10, and Windows Server 2008
-      and later from their NT versions.
-      Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly.
-    - On recent Windows versions, the GetVersionEx() function may report
-      an earlier Windows version than the running OS. To avoid user
-      confusion, add "[or later]" to Tor's version string on affected
-      versions of Windows.
-      Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly.
-    - Remove Windows versions that were never supported by the
-      GetVersionEx() function. Stop duplicating the latest Windows
-      version in get_uname().
-      Fixes bug 28096; bugfix on 0.2.2.34; reported by Keifer Bly.
diff --git a/changes/bug28115 b/changes/bug28115
deleted file mode 100644
index e3e29968eb..0000000000
--- a/changes/bug28115
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (portability):
-    - Make the OPE code (which is used for v3 onion services) run correctly
-      on big-endian platforms. Fixes bug 28115; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28127 b/changes/bug28127
deleted file mode 100644
index 541128c88e..0000000000
--- a/changes/bug28127
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (onion services):
-    - Unless we have explicitly set HiddenServiceVersion, detect the onion
-      service version and then look for invalid options. Previously, we
-      did the reverse, but that broke existing configs which were pointed
-      to a v2 hidden service and had options like HiddenServiceAuthorizeClient
-      set Fixes bug 28127; bugfix on 0.3.5.1-alpha. Patch by Neel Chauhan.
-
diff --git a/changes/bug28183 b/changes/bug28183
deleted file mode 100644
index 8d35dcdc01..0000000000
--- a/changes/bug28183
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (Linux seccomp2 sandbox):
-    - Permit the "shutdown()" system call, which is apparently
-      used by OpenSSL under some circumstances. Fixes bug 28183;
-      bugfix on 0.2.5.1-alpha.
diff --git a/changes/bug28202 b/changes/bug28202
deleted file mode 100644
index 182daac4f1..0000000000
--- a/changes/bug28202
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (C correctness):
-    - Avoid undefined behavior in an end-of-string check when parsing the
-      BEGIN line in a directory object.  Fixes bug 28202; bugfix on
-      0.2.0.3-alpha.
diff --git a/changes/bug28245 b/changes/bug28245
deleted file mode 100644
index d7e6deb810..0000000000
--- a/changes/bug28245
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (OpenSSL, portability):
-    - Fix our usage of named groups when running as a TLS 1.3 client in
-      OpenSSL 1.1.1. Previously, we only initialized EC groups when running
-      as a server, which caused clients to fail to negotiate TLS 1.3 with
-      relays. Fixes bug 28245; bugfix on 0.2.9.15 when TLS 1.3 support was
-      added.
diff --git a/changes/bug28298 b/changes/bug28298
deleted file mode 100644
index 8db340f3df..0000000000
--- a/changes/bug28298
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (configuration):
-    - Resume refusing to start with relative file paths and RunAsDaemon
-      set (regression from the fix for bug 22731). Fixes bug 28298;
-      bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug28303 b/changes/bug28303
deleted file mode 100644
index 80f1302e5e..0000000000
--- a/changes/bug28303
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix a pair of missing headers on OpenBSD. Fixes bug 28303;
-      bugfix on 0.3.5.1-alpha. Patch from  Kris Katterjohn.
diff --git a/changes/bug28348_034 b/changes/bug28348_034
deleted file mode 100644
index 3913c03a4c..0000000000
--- a/changes/bug28348_034
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (embedding, main loop):
-    - When DisableNetwork becomes set, actually disable periodic events that
-      are already enabled. (Previously, we would refrain from enabling new
-      ones, but we would leave the old ones turned on.)
-      Fixes bug 28348; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28399 b/changes/bug28399
deleted file mode 100644
index 9096db70b0..0000000000
--- a/changes/bug28399
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (continuous integration, Windows):
-    - Stop using an external OpenSSL install, and stop installing MSYS2
-      packages, when building using mingw on Appveyor Windows CI.
-      Fixes bug 28399; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28413 b/changes/bug28413
deleted file mode 100644
index 4c88bea7e7..0000000000
--- a/changes/bug28413
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Initialize a variable in aes_new_cipher(), since some compilers
-      cannot tell that we always initialize it before use. Fixes bug 28413;
-      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug28419 b/changes/bug28419
deleted file mode 100644
index 52ceb0a2a7..0000000000
--- a/changes/bug28419
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (memory leaks):
-    - Fix a harmless memory leak in libtorrunner.a. Fixes bug 28419;
-      bugfix on 0.3.3.1-alpha.  Patch from Martin Kepplinger.
\ No newline at end of file
diff --git a/changes/bug28435 b/changes/bug28435
deleted file mode 100644
index 2a886cb8b7..0000000000
--- a/changes/bug28435
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (documentation):
-    - Make Doxygen work again after the 0.3.5 source tree moves.
-      Fixes bug 28435; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28441 b/changes/bug28441
deleted file mode 100644
index d259b9f742..0000000000
--- a/changes/bug28441
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (logging):
-    - Stop talking about the Named flag in log messages. Clients have
-      ignored the Named flag since 0.3.2. Fixes bug 28441;
-      bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug28454 b/changes/bug28454
deleted file mode 100644
index ca46ae2777..0000000000
--- a/changes/bug28454
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (continuous integration, Windows):
-    - Manually configure the zstd compiler options, when building using
-      mingw on Appveyor Windows CI. The MSYS2 mingw zstd package does not
-      come with a pkg-config file. Fixes bug 28454; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug28485 b/changes/bug28485
deleted file mode 100644
index a8309ae21f..0000000000
--- a/changes/bug28485
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Add missing dependency on libgdi32.dll for tor-print-ed-signing-cert.exe
-      on Windows. Fixes bug 28485; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28524 b/changes/bug28524
deleted file mode 100644
index 1cad700422..0000000000
--- a/changes/bug28524
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (restart-in-process, boostrap):
-    - Add missing resets of bootstrap tracking state when shutting
-      down (regression caused by ticket 27169).  Fixes bug 28524;
-      bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28525 b/changes/bug28525
deleted file mode 100644
index 988ffb2192..0000000000
--- a/changes/bug28525
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor features (address selection):
-    - Make Tor aware of the RFC 6598 (Carrier Grade NAT) IP range, which is the
-      subnet 100.64.0.0/10. This is deployed by many ISPs as an alternative to
-      RFC 1918 that does not break existing internal networks. This patch fixes
-      security issues caused by RFC 6518 by blocking control ports on these
-      addresses and warns users if client ports or ExtORPorts are listening on
-      a RFC 6598 address. Closes ticket 28525. Patch by Neel Chauhan.
diff --git a/changes/bug28554 b/changes/bug28554
deleted file mode 100644
index 9a0b281406..0000000000
--- a/changes/bug28554
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (unit tests, guard selection):
-    - Stop leaking memory in an entry guard unit test. Fixes bug 28554;
-      bugfix on 0.3.0.1-alpha.
diff --git a/changes/bug28562 b/changes/bug28562
deleted file mode 100644
index e14362164d..0000000000
--- a/changes/bug28562
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (testing):
-    - Use a separate DataDirectory for the test_rebind script.
-      Previously, this script would run using the default DataDirectory,
-      and sometimes fail. Fixes bug 28562; bugfix on 0.3.5.1-alpha.
-      Patch from Taylor R Campbell.
diff --git a/changes/bug28568 b/changes/bug28568
deleted file mode 100644
index 919ec08903..0000000000
--- a/changes/bug28568
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Stop running stem's unit tests as part of "make test-stem". But continue
-      to run stem's unit and online tests during "make test-stem-full".
-      Fixes bug 28568; bugfix on 0.2.6.3-alpha.
diff --git a/changes/bug28569 b/changes/bug28569
deleted file mode 100644
index 45a57a80ae..0000000000
--- a/changes/bug28569
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (unit tests, directory clients):
-    - Mark outdated dirservers when Tor only has a reasonably live consensus.
-      Fixes bug 28569; bugfix on 0.3.2.5-alpha.
diff --git a/changes/bug28612 b/changes/bug28612
deleted file mode 100644
index 559f254234..0000000000
--- a/changes/bug28612
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (windows services):
-    - Make Tor start correctly as an NT service again: previously it
-      was broken by refactoring.  Fixes bug 28612; bugfix on 0.3.5.3-alpha.
-      
diff --git a/changes/bug28619 b/changes/bug28619
deleted file mode 100644
index 86be8cb2fb..0000000000
--- a/changes/bug28619
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (hidden service v3):
-    - When deleting an ephemeral onion service (DEL_ONION), do not close any
-      rendezvous circuits in order to let the existing client connections
-      finish by themselves or closed by the application. The HS v2 is doing
-      that already so now we have the same behavior for all versions. Fixes
-      bug 28619; bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug28656 b/changes/bug28656
deleted file mode 100644
index d3a13d196c..0000000000
--- a/changes/bug28656
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (logging):
-    - Stop logging a BUG() warning when tor is waiting for exit descriptors.
-      Fixes bug 28656; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28698 b/changes/bug28698
deleted file mode 100644
index 716aa0c552..0000000000
--- a/changes/bug28698
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfix (logging):
-    - Avoid logging about relaxing circuits when their time is fixed.
-      Fixes bug 28698; bugfix on 0.2.4.7-alpha
diff --git a/changes/bug28895 b/changes/bug28895
deleted file mode 100644
index 25fb167b2e..0000000000
--- a/changes/bug28895
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (usability):
-    - Stop saying "Your Guard ..." in pathbias_measure_{use,close}_rate()
-      as that confusingly suggests that mentioned guard node is under control
-      and responsibility of end user, which it is not. Fixes bug 28895;
-      bugfix on Tor 0.3.0.1-alpha.
diff --git a/changes/bug28920 b/changes/bug28920
deleted file mode 100644
index e698686a6d..0000000000
--- a/changes/bug28920
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (logging):
-    - Rework rep_hist_log_link_protocol_counts() to iterate through all link
-      protocol versions when logging incoming/outgoing connection counts. Tor
-      no longer skips version 5 and we don't have to remember to update this
-      function when new link protocol version is developed. Fixes bug 28920;
-      bugfix on 0.2.6.10.
diff --git a/changes/bug28938 b/changes/bug28938
deleted file mode 100644
index de6c5f7b79..0000000000
--- a/changes/bug28938
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix missing headers required for proper detection of
-      OpenBSD.  Fixes bug 28938; bugfix on 0.3.5.1-alpha.
-      Patch from Kris Katterjohn.
diff --git a/changes/bug28974 b/changes/bug28974
deleted file mode 100644
index 2d74f5674f..0000000000
--- a/changes/bug28974
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Fix compilation for Android by adding a missing header to
-      freespace.c.  Fixes bug 28974; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug28979 b/changes/bug28979
deleted file mode 100644
index 0625fd5d25..0000000000
--- a/changes/bug28979
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (documentation):
-    - Describe the contents of the v3 onion service client authorization
-      files correctly: They hold public keys, not private keys. Fixes bug
-      28979; bugfix on 0.3.5.1-alpha. Spotted by "Felixix".
diff --git a/changes/bug28981 b/changes/bug28981
deleted file mode 100644
index c0ea92ab35..0000000000
--- a/changes/bug28981
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (misc):
-    - The amount of total available physical memory is now determined
-      using the sysctl identifier HW_PHYSMEM (rather than HW_USERMEM)
-      when it is defined and a 64-bit variant is not available.  Fixes
-      bug 28981; bugfix on 0.2.5.4-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug28995 b/changes/bug28995
deleted file mode 100644
index f76b6a085a..0000000000
--- a/changes/bug28995
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfix (IPv6):
-    Fix tor_ersatz_socketpair on IPv6-only systems.  Previously,
-    the IPv6 socket was bound using an address family of AF_INET
-    instead of AF_INET6.  Fixes bug 28995; bugfix on 0.3.5.1-alpha.
-    Patch from Kris Katterjohn.
diff --git a/changes/bug29017 b/changes/bug29017
deleted file mode 100644
index 5c4a53c43f..0000000000
--- a/changes/bug29017
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (stats):
-    - When ExtraInfoStatistics is 0, stop including PaddingStatistics in
-      relay and bridge extra-info documents. Fixes bug 29017;
-      bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug29029 b/changes/bug29029
deleted file mode 100644
index e100a8c2ed..0000000000
--- a/changes/bug29029
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging, onion services):
-    - Stop logging "Tried to establish rendezvous on non-OR circuit..." as
-      a warning. Instead, log it as a protocol warning, because there is
-      nothing that relay operators can do to fix it. Fixes bug 29029;
-      bugfix on 0.2.5.7-rc.
diff --git a/changes/bug29034 b/changes/bug29034
deleted file mode 100644
index e7aa9af00b..0000000000
--- a/changes/bug29034
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (Onion service reachability):
-    - Properly clean up the introduction point map when circuits change purpose
-      from onion service circuits to pathbias, measurement, or other circuit types.
-      This should fix some service-side instances of introduction point failure.
-      Fixes bug 29034; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug29036 b/changes/bug29036
deleted file mode 100644
index 8b96c5c8fa..0000000000
--- a/changes/bug29036
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfix (continuous integration):
-    - Reset coverage state on disk after Travis CI has finished. This is being
-      done to prevent future gcda file merge errors which causes the test suite
-      for the process subsystem to fail. The process subsystem was introduced
-      in 0.4.0.1-alpha. Fixes bug 29036; bugfix on 0.2.9.15.
diff --git a/changes/bug29040 b/changes/bug29040
deleted file mode 100644
index 0662aaa8a5..0000000000
--- a/changes/bug29040
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (onion services):
-    - Avoid crashing if ClientOnionAuthDir (incorrectly) contains
-      more than one private key for a hidden service. Fixes bug 29040;
-      bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29042 b/changes/bug29042
deleted file mode 100644
index 8d76939cea..0000000000
--- a/changes/bug29042
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Log more information at "warning" level when unable to read a private
-      key; log more information ad "info" level when unable to read a public
-      key. We had warnings here before, but they were lost during our
-      NSS work. Fixes bug 29042; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29135 b/changes/bug29135
deleted file mode 100644
index fd7b1ae80e..0000000000
--- a/changes/bug29135
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (onion services, logging):
-    - In hs_cache_store_as_client() log an HSDesc we failed to parse at Debug
-      loglevel. Tor used to log it at Warning loglevel, which caused
-      very long log lines to appear for some users. Fixes bug 29135; bugfix on
-      0.3.2.1-alpha.
diff --git a/changes/bug29144 b/changes/bug29144
deleted file mode 100644
index 5801224f14..0000000000
--- a/changes/bug29144
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Log the correct port number for listening sockets when "auto" is
-      used to let Tor pick the port number.  Previously, port 0 was
-      logged instead of the actual port number.  Fixes bug 29144;
-      bugfix on 0.3.5.1-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug29145 b/changes/bug29145
deleted file mode 100644
index 40d3da4b91..0000000000
--- a/changes/bug29145
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (compilation, testing):
-    - Silence a compiler warning in test-memwipe.c on OpenBSD.  Fixes
-      bug 29145; bugfix on 0.2.9.3-alpha.  Patch from Kris Katterjohn.
diff --git a/changes/bug29161 b/changes/bug29161
deleted file mode 100644
index 39a638acf6..0000000000
--- a/changes/bug29161
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (tests):
-    - Detect and suppress "bug" warnings from the util/time test on Windows.
-      Fixes bug 29161; bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug29175_035 b/changes/bug29175_035
deleted file mode 100644
index 134c1d9529..0000000000
--- a/changes/bug29175_035
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Major bugfixes (networking):
-    - Gracefully handle empty username/password fields in SOCKS5
-      username/password auth messsage and allow SOCKS5 handshake to
-      continue. Fixes bug 29175; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29241 b/changes/bug29241
deleted file mode 100644
index 7f25e154d1..0000000000
--- a/changes/bug29241
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (NSS, relay):
-    - When running with NSS, disable TLS 1.2 ciphersuites that use SHA384
-      for their PRF. Due to an NSS bug, the TLS key exporters for these
-      ciphersuites don't work -- which caused relays to fail to handshake
-      with one another when these ciphersuites were enabled.
-      Fixes bug 29241; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug29244 b/changes/bug29244
deleted file mode 100644
index 6206a95463..0000000000
--- a/changes/bug29244
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (build, compatibility):
-    - Update Cargo.lock file to match the version made by the latest
-      version of Rust, so that "make distcheck" will pass again.
-      Fixes bug 29244; bugfix on 0.3.3.4-alpha.
diff --git a/changes/bug29530_035 b/changes/bug29530_035
deleted file mode 100644
index 6dfcd51e7b..0000000000
--- a/changes/bug29530_035
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (testing):
-    - Downgrade some LOG_ERR messages in the address/* tests to warnings.
-      The LOG_ERR messages were occurring when we had no configured network.
-      We were failing the unit tests, because we backported 28668 to 0.3.5.8,
-      but did not backport 29530. Fixes bug 29530; bugfix on 0.3.5.8.
diff --git a/changes/bug29599 b/changes/bug29599
deleted file mode 100644
index 14e2f5d077..0000000000
--- a/changes/bug29599
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (memory management, testing):
-    - Stop leaking parts of the shared random state in the shared-random unit
-      tests. Fixes bug 29599; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug29601 b/changes/bug29601
deleted file mode 100644
index c4ba5fbc8b..0000000000
--- a/changes/bug29601
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (Windows, CI):
-    - Skip the Appveyor 32-bit Windows Server 2016 job, and 64-bit Windows
-      Server 2012 R2 job. The remaining 2 jobs still provide coverage of
-      64/32-bit, and Windows Server 2016/2012 R2. Also set fast_finish, so
-      failed jobs terminate the build immediately.
-      Fixes bug 29601; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug29665 b/changes/bug29665
deleted file mode 100644
index d89046faf5..0000000000
--- a/changes/bug29665
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (single onion services):
-    - Allow connections to single onion services to remain idle without
-      being disconnected. Relays acting as rendezvous points for
-      single onion services were mistakenly closing idle established
-      rendezvous circuits after 60 seconds, thinking that they are unused
-      directory-fetching circuits that had served their purpose. Fixes
-      bug 29665; bugfix on 0.2.1.26.
diff --git a/changes/bug29670 b/changes/bug29670
deleted file mode 100644
index 00b0c33327..0000000000
--- a/changes/bug29670
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (configuration, proxies):
-    - Fix a bug that prevented us from supporting SOCKS5 proxies that want
-      authentication along with configued (but unused!)
-      ClientTransportPlugins. Fixes bug 29670; bugfix on 0.2.6.1-alpha.
diff --git a/changes/bug29703 b/changes/bug29703
deleted file mode 100644
index 0e17ee45e6..0000000000
--- a/changes/bug29703
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Backport the 0.3.4 src/test/test-network.sh to 0.2.9.
-      We need a recent test-network.sh to use new chutney features in CI.
-      Fixes bug 29703; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug29706_minimal b/changes/bug29706_minimal
deleted file mode 100644
index 9d4a43326c..0000000000
--- a/changes/bug29706_minimal
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory management, testing):
-    - Stop leaking parts of the shared random state in the shared-random unit
-      tests. The previous fix in 29599 was incomplete.
-      Fixes bug 29706; bugfix on 0.2.9.1-alpha.
diff --git a/changes/bug29875 b/changes/bug29875
deleted file mode 100644
index 58a1c871cd..0000000000
--- a/changes/bug29875
+++ /dev/null
@@ -1,11 +0,0 @@
-  o Major bugfixes (bridges):
-    - Do not count previously configured working bridges towards our total of
-      working bridges. Previously, when Tor's list of bridges changed, it
-      would think that the old bridges were still usable, and delay fetching
-      router descriptors for the new ones.  Fixes part of bug 29875; bugfix
-      on 0.3.0.1-alpha.
-    - Consider our directory information to have changed when our list of
-      bridges changes. Previously, Tor would not re-compute the status of its
-      directory information when bridges changed, and therefore would not
-      realize that it was no longer able to build circuits. Fixes part of bug
-      29875.
diff --git a/changes/bug29922 b/changes/bug29922
deleted file mode 100644
index dacb951097..0000000000
--- a/changes/bug29922
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing, windows):
-    - Fix a test failure caused by an unexpected bug warning in
-      our test for tor_gmtime_r(-1). Fixes bug 29922;
-      bugfix on 0.2.9.3-alpha.
diff --git a/changes/bug30011 b/changes/bug30011
deleted file mode 100644
index 4c9069e291..0000000000
--- a/changes/bug30011
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (CI):
-    - Terminate test-stem if it takes more than 9.5 minutes to run.
-      (Travis terminates the job after 10 minutes of no output.)
-      Diagnostic for 29437. Fixes bug 30011; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug30021 b/changes/bug30021
deleted file mode 100644
index 2a887f3cf2..0000000000
--- a/changes/bug30021
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Minor bugfixes (TLS protocol, integration tests):
-    - When classifying a client's selection of TLS ciphers, if the client
-      ciphers are not yet available, do not cache the result. Previously,
-      we had cached the unavailability of the cipher list and never looked
-      again, which in turn led us to assume that the client only supported
-      the ancient V1 link protocol.  This, in turn, was causing Stem
-      integration tests to stall in some cases.
-      Fixes bug 30021; bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug30040 b/changes/bug30040
deleted file mode 100644
index 7d80528a10..0000000000
--- a/changes/bug30040
+++ /dev/null
@@ -1,9 +0,0 @@
-  o Minor bugfixes (security):
-    - Fix a potential double free bug when reading huge bandwidth files. The
-      issue is not exploitable in the current Tor network because the
-      vulnerable code is only reached when directory authorities read bandwidth
-      files, but bandwidth files come from a trusted source (usually the
-      authorities themselves). Furthermore, the issue is only exploitable in
-      rare (non-POSIX) 32-bit architectures which are not used by any of the
-      current authorities. Fixes bug 30040; bugfix on 0.3.5.1-alpha. Bug found
-      and fixed by Tobias Stoeckmann.
diff --git a/changes/bug30041 b/changes/bug30041
deleted file mode 100644
index 801c8f67ac..0000000000
--- a/changes/bug30041
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (hardening):
-    - Verify in more places that we are not about to create a buffer
-      with more than INT_MAX bytes, to avoid possible OOB access in the event
-      of bugs.  Fixes bug 30041; bugfix on 0.2.0.16.  Found and fixed by
-      Tobias Stoeckmann.
diff --git a/changes/bug30148 b/changes/bug30148
deleted file mode 100644
index 7d0257e3fe..0000000000
--- a/changes/bug30148
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory leak):
-    - Avoid a minor memory leak that could occur on relays when
-      creating a keys directory failed. Fixes bug 30148; bugfix on
-      0.3.3.1-alpha.
diff --git a/changes/bug30189 b/changes/bug30189
deleted file mode 100644
index f8c932a5f9..0000000000
--- a/changes/bug30189
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation, unusual configuration):
-    - Avoid failures when building with ALL_BUGS_ARE_FAILED due to
-      missing declarations of abort(), and prevent other such failures
-      in the future. Fixes bug 30189; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug30190 b/changes/bug30190
deleted file mode 100644
index e2352c3b9c..0000000000
--- a/changes/bug30190
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (lib):
-    do not log a warning for OpenSSL versions that should be compatible
-    Fixes bug 30190; bugfix on 0.2.4.2-alpha
diff --git a/changes/bug30316 b/changes/bug30316
deleted file mode 100644
index 3e396318ad..0000000000
--- a/changes/bug30316
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (directory authority):
-    - Move the "bandwidth-file-headers" line in directory authority votes
-      so that it conforms to dir-spec.txt. Fixes bug 30316; bugfix on
-      0.3.5.1-alpha.
diff --git a/changes/bug30452 b/changes/bug30452
deleted file mode 100644
index 2bb401d87d..0000000000
--- a/changes/bug30452
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (compile-time modules):
-    - Add a --list-modules command to print a list of which compile-time
-      modules are enabled. Closes ticket 30452.
diff --git a/changes/bug30475 b/changes/bug30475
deleted file mode 100644
index 839597b885..0000000000
--- a/changes/bug30475
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes ():
-    - Avoid a GCC 9.1.1 warning (and possible crash depending on libc
-      implemenation) when failing to load a hidden service client authorization
-      file.  Fixes bug 30475; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug30561 b/changes/bug30561
deleted file mode 100644
index afb3f02c62..0000000000
--- a/changes/bug30561
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfixes (portability):
-    - Avoid crashing in our tor_vasprintf() implementation on systems that
-      define neither vasprintf() nor _vscprintf(). (This bug has been here
-      long enough that we question whether people are running Tor on such
-      systems, but we're applying the fix out of caution.) Fixes bug 30561;
-      bugfix on 0.2.8.2-alpha. Found and fixed by Tobias Stoeckmann.
diff --git a/changes/bug30713 b/changes/bug30713
deleted file mode 100644
index e00b98da65..0000000000
--- a/changes/bug30713
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (testing):
-    - Skip test_rebind when the TOR_SKIP_TEST_REBIND environmental variable is
-      set. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
-    - Skip test_rebind on macOS in Travis, because it is unreliable on
-      macOS on Travis. Fixes bug 30713; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug30744 b/changes/bug30744
deleted file mode 100644
index 9f07d4855f..0000000000
--- a/changes/bug30744
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (continuous integration):
-    - Allow the test-stem job to fail in Travis, because it sometimes hangs.
-      Fixes bug 30744; bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug30781 b/changes/bug30781
deleted file mode 100644
index 7c7adf470e..0000000000
--- a/changes/bug30781
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (directory authorities):
-    - Stop crashing after parsing an unknown descriptor purpose annotation.
-      We think this bug can only be triggered by modifying a local file.
-      Fixes bug 30781; bugfix on 0.2.0.8-alpha.
diff --git a/changes/bug30894 b/changes/bug30894
deleted file mode 100644
index 64c14c4e6d..0000000000
--- a/changes/bug30894
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (memory leaks):
-    - Fix a trivial memory leak when parsing an invalid value
-      from a download schedule in the configuration. Fixes bug
-      30894; bugfix on 0.3.4.1-alpha.
diff --git a/changes/bug30916 b/changes/bug30916
deleted file mode 100644
index b006bfc75d..0000000000
--- a/changes/bug30916
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (relay):
-    - Avoid crashing when starting with a corrupt keys directory where
-      the old ntor key and the new ntor key are identical. Fixes bug 30916;
-      bugfix on 0.2.4.8-alpha.
diff --git a/changes/bug31003 b/changes/bug31003
deleted file mode 100644
index 6c75163380..0000000000
--- a/changes/bug31003
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (crash on exit):
-    - Avoid a set of possible code paths that could use try to use freed memory
-      in routerlist_free() while Tor was exiting.  Fixes bug 31003; bugfix on
-      0.1.2.2-alpha.
diff --git a/changes/bug31107 b/changes/bug31107
deleted file mode 100644
index 9652927c30..0000000000
--- a/changes/bug31107
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (logging, protocol violations):
-    - Do not log a nonfatal assertion failure when receiving a VERSIONS
-      cell on a connection using the obsolete v1 link protocol. Log a
-      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.
diff --git a/changes/bug31335 b/changes/bug31335
new file mode 100644
index 0000000000..f633cf8b24
--- /dev/null
+++ b/changes/bug31335
@@ -0,0 +1,3 @@
+  o Minor bugfixes (code quality):
+    - Fix "make check-includes" so it runs correctly on out-of-tree builds.
+      Fixes bug 31335; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug31343 b/changes/bug31343
deleted file mode 100644
index 17a8057ead..0000000000
--- a/changes/bug31343
+++ /dev/null
@@ -1,9 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Avoid using labs() on time_t, which can cause compilation warnings
-      on 64-bit Windows builds.  Fixes bug 31343; bugfix on 0.2.4.4-alpha.
-
-  o Minor bugfixes (clock skew detection):
-    - Don't believe clock skew results from NETINFO cells that appear to
-      arrive before the VERSIONS cells they are responding to were sent.
-      Previously, we would accept them up to 3 minutes "in the past".
-      Fixes bug 31343; bugfix on 0.2.4.4-alpha.
diff --git a/changes/bug31408 b/changes/bug31408
deleted file mode 100644
index 3e4ffa927d..0000000000
--- a/changes/bug31408
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (torrc):
-    - Stop ignoring torrc options after an %include directive, when the
-      included directory ends with a file that does not contain any config
-      options. (But does contain comments or whitespace.)
-      Fixes bug 31408; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug31463 b/changes/bug31463
deleted file mode 100644
index d85c0887c3..0000000000
--- a/changes/bug31463
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (rust):
-    - Correctly exclude a redundant rust build job in Travis. Fixes bug 31463;
-      bugfix on 0.3.5.4-alpha.
diff --git a/changes/bug31571 b/changes/bug31571
deleted file mode 100644
index 86de3537ba..0000000000
--- a/changes/bug31571
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor bugfixes (error handling):
-    - Report the tor version whenever an assertion fails. Previously, we only
-      reported the Tor version on some crashes, and some non-fatal assertions.
-      Fixes bug 31571; bugfix on 0.3.5.1-alpha.
-    - On abort, try harder to flush the output buffers of log messages. On
-      some platforms (macOS), log messages can be discarded when the process
-      terminates. Fixes bug 31571; bugfix on 0.3.5.1-alpha.
diff --git a/changes/bug31652 b/changes/bug31652
new file mode 100644
index 0000000000..c4eca7994a
--- /dev/null
+++ b/changes/bug31652
@@ -0,0 +1,5 @@
+  o Minor bugfixes (onion services):
+    - When we clean up intro circuits for a v3 onion service, don't remove
+      circuits that have an established or pending circuit even if ran out of
+      retries. This way, we don't cleanup the circuit of the last retry. Fixes
+      bug 31652; bugfix on 0.3.2.1-alpha.
diff --git a/changes/bug31657 b/changes/bug31657
deleted file mode 100644
index 08e9d95fdf..0000000000
--- a/changes/bug31657
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (guards):
-    - When tor is missing descriptors for some primary entry guards, make the
-      log message less alarming. It's normal for descriptors to expire, as long
-      as tor fetches new ones soon after. Fixes bug 31657;
-      bugfix on 0.3.3.1-alpha.
diff --git a/changes/bug31734 b/changes/bug31734
new file mode 100644
index 0000000000..ce989ea5db
--- /dev/null
+++ b/changes/bug31734
@@ -0,0 +1,3 @@
+  o Minor bugfixes (error handling):
+    - Always lock the backtrace buffer before it is used.
+      Fixes bug 31734; bugfix on 0.2.5.3-alpha.
diff --git a/changes/bug31810 b/changes/bug31810
new file mode 100644
index 0000000000..628d12f09b
--- /dev/null
+++ b/changes/bug31810
@@ -0,0 +1,4 @@
+  o Minor bugfixes (process management):
+    - Remove assertion in the Unix process backend. This assertion would trigger
+      when a new process is spawned where the executable is not found leading to
+      a stack trace from the child process. Fixes bug 31810; bugfix on 0.4.0.1-alpha.
diff --git a/changes/bug31884 b/changes/bug31884
deleted file mode 100644
index ddb6c50d74..0000000000
--- a/changes/bug31884
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (Appveyor CI):
-    - Avoid spurious errors when Appveyor CI fails before the install step.
-      Fixes bug 31884; bugfix on 0.3.4.2-alpha.
diff --git a/changes/bug31922 b/changes/bug31922
new file mode 100644
index 0000000000..e6f31ce66a
--- /dev/null
+++ b/changes/bug31922
@@ -0,0 +1,4 @@
+  o Minor bugfixes (configuration):
+    - When pkg-config is not installed, or a library that depends on
+      pkg-config is not found, tell the user what to do to fix the
+      problem. Fixes bug 31922; bugfix on 0.3.1.1-alpha.
diff --git a/changes/bug31995 b/changes/bug31995
new file mode 100644
index 0000000000..c7ddd437a6
--- /dev/null
+++ b/changes/bug31995
@@ -0,0 +1,3 @@
+  o Minor bugfixes (testing):
+    - Avoid intermittent test failures due to a test that had relied on
+      inconsistent timing sources. Fixes bug 31995; bugfix on 0.3.1.3-alpha.
diff --git a/changes/bug32106 b/changes/bug32106
new file mode 100644
index 0000000000..c6e8e95860
--- /dev/null
+++ b/changes/bug32106
@@ -0,0 +1,5 @@
+  o Minor features (documentation):
+    - Make clear in the man page, in both the bandwidth section and the
+      accountingmax section, that Tor counts in powers of two, not
+      powers of ten: 1 GByte is 1024*1024*1024 bytes, not one billion
+      bytes. Resolves ticket 32106.
diff --git a/changes/bug32108 b/changes/bug32108
new file mode 100644
index 0000000000..2806fa3e5d
--- /dev/null
+++ b/changes/bug32108
@@ -0,0 +1,8 @@
+  o Major bugfixes (relay):
+    - Relays now respect their AccountingMax bandwidth again. When relays
+      entered "soft" hibernation (which typically starts when we've hit
+      90% of our AccountingMax), we had stopped checking whether we should
+      enter hard hibernation. Soft hibernation refuses new connections and
+      new circuits, but the existing circuits can continue, meaning that
+      relays could have exceeded their configured AccountingMax. Fixes
+      bug 32108; bugfix on 0.4.0.1-alpha.
diff --git a/changes/bug32124 b/changes/bug32124
new file mode 100644
index 0000000000..164b33c7e3
--- /dev/null
+++ b/changes/bug32124
@@ -0,0 +1,7 @@
+  o Minor bugfixes (build system):
+    - Stop failing when jemalloc is requested, but tcmalloc is not found.
+      Fixes bug 32124; bugfix on 0.3.5.1-alpha.
+    - Interpret --disable-module-dirauth=no correctly.
+      Fixes bug 32124; bugfix on 0.3.4.1-alpha.
+    - Interpret --with-tcmalloc=no correctly.
+      Fixes bug 32124; bugfix on 0.2.0.20-rc.
diff --git a/changes/bug32295 b/changes/bug32295
new file mode 100644
index 0000000000..e5e5a4399d
--- /dev/null
+++ b/changes/bug32295
@@ -0,0 +1,3 @@
+  o Minor bugfixes (configuration):
+    - Log the option name when skipping an obsolete option.
+      Fixes bug 32295; bugfix on 0.4.2.1-alpha.
diff --git a/changes/bug32402 b/changes/bug32402
new file mode 100644
index 0000000000..0654389be3
--- /dev/null
+++ b/changes/bug32402
@@ -0,0 +1,3 @@
+  o Minor bugfixes (shellcheck):
+    - Start checking most scripts for shellcheck errors again.
+      Fixes bug 32402; bugfix on 0.4.2.1-alpha.
diff --git a/changes/bug32402_git_scripts b/changes/bug32402_git_scripts
new file mode 100644
index 0000000000..2b10a8998a
--- /dev/null
+++ b/changes/bug32402_git_scripts
@@ -0,0 +1,3 @@
+  o Minor bugfixes (shellcheck):
+    - Fix minor shellcheck errors in the git-*.sh scripts.
+      Fixes bug 32402; bugfix on 0.4.2.1-alpha.
diff --git a/changes/bug32771 b/changes/bug32771
new file mode 100644
index 0000000000..606bcf4be4
--- /dev/null
+++ b/changes/bug32771
@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging, crash):
+    - Avoid a possible crash when trying to log a (fatal) assertion failure
+      about mismatched magic numbers in configuration objects. Fixes bug 32771;
+      bugfix on 0.4.2.1-alpha.
diff --git a/changes/bug32778 b/changes/bug32778
new file mode 100644
index 0000000000..ccb6104692
--- /dev/null
+++ b/changes/bug32778
@@ -0,0 +1,3 @@
+  o Minor bugfixes (windows service):
+    - Initialize publish/subscribe system when running as a windows service.
+      Fixes bug 32778; bugfix on 0.4.1.1-alpha.
diff --git a/changes/bug32841 b/changes/bug32841
new file mode 100644
index 0000000000..48568f6a61
--- /dev/null
+++ b/changes/bug32841
@@ -0,0 +1,4 @@
+  o Minor bugfixes (linux seccomp sandbox):
+    - Fix crash when reloading logging configuration while the
+      experimental sandbox is enabled. Fixes bug 32841; bugfix
+      on 0.4.1.7. Patch by Peter Gerber.
diff --git a/changes/bug33087 b/changes/bug33087
new file mode 100644
index 0000000000..7acf72a835
--- /dev/null
+++ b/changes/bug33087
@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging):
+    - Stop closing stderr and stdout during shutdown. Closing these file
+      descriptors can hide sanitiser logs.
+      Fixes bug 33087; bugfix on 0.4.1.6.
diff --git a/changes/bug33095_041 b/changes/bug33095_041
new file mode 100644
index 0000000000..7d1f04e279
--- /dev/null
+++ b/changes/bug33095_041
@@ -0,0 +1,5 @@
+  o Minor bugfixes (logging, bug reporting):
+    - When logging a bug, do not say "Future instances of this warning
+      will be silenced" unless we are actually going to do
+      so. Previously we would say this whenever a BUG() check failed in
+      the code.  Fixes bug 33095; bugfix on 0.4.1.1-alpha.
diff --git a/changes/bug33104 b/changes/bug33104
new file mode 100644
index 0000000000..b5478df108
--- /dev/null
+++ b/changes/bug33104
@@ -0,0 +1,4 @@
+  o Minor bugfixes (controller):
+    - When receiving "ACTIVE" or "DORMANT" signals on the control port,
+      report them as SIGNAL events. Fixes bug 33104; bugfix on
+      0.4.0.1-alpha.
diff --git a/changes/bug34077 b/changes/bug34077
new file mode 100644
index 0000000000..29458bd9de
--- /dev/null
+++ b/changes/bug34077
@@ -0,0 +1,3 @@
+  o Minor bugfixes (compiler warnings):
+    - Fix compilation warnings with GCC 10.0.1. Fixes bug 34077; bugfix on
+      0.4.0.3-alpha.
diff --git a/changes/chutney_ci b/changes/chutney_ci
deleted file mode 100644
index b17d587329..0000000000
--- a/changes/chutney_ci
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (continuous integration):
-    - Our Travis configuration now uses Chutney to run some network
-      integration tests automatically. Closes ticket 29280.
diff --git a/changes/cid1444119 b/changes/cid1444119
deleted file mode 100644
index bb6854e66f..0000000000
--- a/changes/cid1444119
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (C correctness):
-    - Fix an unlikely memory leak in consensus_diff_apply(). Fixes bug 29824;
-      bugfix on 0.3.1.1-alpha. This is Coverity warning CID 1444119.
diff --git a/changes/doc31089 b/changes/doc31089
deleted file mode 100644
index 2fc0ba4f7d..0000000000
--- a/changes/doc31089
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation:
-    - Use RFC 2397 data URL scheme to embed image into tor-exit-notice.html
-      so that operators would no longer have to host it themselves.
-      Closes ticket 31089.
diff --git a/changes/geoip-2018-11-06 b/changes/geoip-2018-11-06
deleted file mode 100644
index 5c18ea4244..0000000000
--- a/changes/geoip-2018-11-06
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the November 6 2018 Maxmind GeoLite2
-      Country database. Closes ticket 28395.
-
diff --git a/changes/geoip-2018-12-05 b/changes/geoip-2018-12-05
deleted file mode 100644
index 20ccf2d8a5..0000000000
--- a/changes/geoip-2018-12-05
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the December 5 2018 Maxmind GeoLite2
-      Country database. Closes ticket 28744.
-
diff --git a/changes/geoip-2019-01-03 b/changes/geoip-2019-01-03
deleted file mode 100644
index 27ffb7f460..0000000000
--- a/changes/geoip-2019-01-03
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the January 3 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29012.
-
diff --git a/changes/geoip-2019-02-05 b/changes/geoip-2019-02-05
deleted file mode 100644
index 78ee6d4242..0000000000
--- a/changes/geoip-2019-02-05
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the February 5 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29478.
-
diff --git a/changes/geoip-2019-03-04 b/changes/geoip-2019-03-04
deleted file mode 100644
index c8ce5dad5d..0000000000
--- a/changes/geoip-2019-03-04
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the March 4 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29666.
-
diff --git a/changes/geoip-2019-04-02 b/changes/geoip-2019-04-02
deleted file mode 100644
index 7302d939f6..0000000000
--- a/changes/geoip-2019-04-02
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the April 2 2019 Maxmind GeoLite2
-      Country database. Closes ticket 29992.
-
diff --git a/changes/geoip-2019-05-13 b/changes/geoip-2019-05-13
deleted file mode 100644
index 0a2fa18971..0000000000
--- a/changes/geoip-2019-05-13
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the May 13 2019 Maxmind GeoLite2
-      Country database. Closes ticket 30522.
-
diff --git a/changes/geoip-2019-06-10 b/changes/geoip-2019-06-10
deleted file mode 100644
index 2d1e065649..0000000000
--- a/changes/geoip-2019-06-10
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the June 10 2019 Maxmind GeoLite2
-      Country database. Closes ticket 30852.
-
diff --git a/changes/geoip-2019-10-01 b/changes/geoip-2019-10-01
deleted file mode 100644
index c7ed17b5c4..0000000000
--- a/changes/geoip-2019-10-01
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (geoip):
-    - Update geoip and geoip6 to the October 1 2019 Maxmind GeoLite2
-      Country database. Closes ticket 31931.
-
diff --git a/changes/rust_asan b/changes/rust_asan
deleted file mode 100644
index 1ca7ae6888..0000000000
--- a/changes/rust_asan
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Major bugfixes (compilation, rust):
-    - Rust tests can now build and run successfully with the
-      --enable-fragile-hardening option enabled.
-      Doing this currently requires the rust beta channel; it will
-      be possible with stable rust as of rust version 1.31 is out.
-      Patch from Alex Crichton.
-      Fixes bugs 27272, 27273, and 27274.
-      Bugfix on 0.3.1.1-alpha.
diff --git a/changes/ticket19566 b/changes/ticket19566
deleted file mode 100644
index bf7071e660..0000000000
--- a/changes/ticket19566
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Code simplification and refactoring (shared random, dirauth):
-    - Change many tor_assert() to use BUG() instead. The idea is to not crash
-      a dirauth but rather scream loudly with a stacktrace and let it continue
-      run. The shared random subsystem is very resilient and if anything wrong
-      happens with it, at worst a non coherent value will be put in the vote
-      and discarded by the other authorities. Closes ticket 19566.
diff --git a/changes/ticket24803 b/changes/ticket24803
deleted file mode 100644
index e76a9eeab9..0000000000
--- a/changes/ticket24803
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features (fallback directory list):
-    - Replace the 150 fallbacks originally introduced in Tor 0.3.3.1-alpha in
-      January 2018 (of which ~115 were still functional), with a list of
-      157 fallbacks (92 new, 65 existing, 85 removed) generated in
-      December 2018.  Closes ticket 24803.
diff --git a/changes/ticket27471 b/changes/ticket27471
deleted file mode 100644
index ffe77d268e..0000000000
--- a/changes/ticket27471
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (hidden service v3, client):
-    - When replacing a descriptor in the client cache with a newer descriptor,
-      make sure to close all client introduction circuits of the old
-      descriptor so we don't end up with unusable leftover circuits. Fixes bug
-      27471; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27751 b/changes/ticket27751
deleted file mode 100644
index 593c473b61..0000000000
--- a/changes/ticket27751
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Minor features (continuous integration):
-    - Add a Travis CI build for --enable-nss on Linux gcc. Closes ticket 27751.
diff --git a/changes/ticket27838 b/changes/ticket27838
deleted file mode 100644
index 1699730d7a..0000000000
--- a/changes/ticket27838
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (hidden service v3):
-    - Build the service descriptor signing key certificate before uploading so
-      we always have a fresh one leaving no chances for it to expire service
-      side. Fixes bug 27838; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket27913 b/changes/ticket27913
deleted file mode 100644
index 81ce725932..0000000000
--- a/changes/ticket27913
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing:
-    - Add new CI job to Travis configuration that runs stem-based
-      integration tests. Closes ticket 27913.
diff --git a/changes/ticket27995 b/changes/ticket27995
deleted file mode 100644
index 8c75425749..0000000000
--- a/changes/ticket27995
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (hidden service v3, client authorization):
-    - Fix an assert() when adding a client authorization for the first time
-      and then sending a HUP signal to the service. Before that, tor would
-      stop abruptly. Fixes bug 27995; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket28026 b/changes/ticket28026
deleted file mode 100644
index a6911c2cab..0000000000
--- a/changes/ticket28026
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Documentation (hidden service manpage):
-    - Improve HSv3 client authorization by making some options more explicit
-      and detailed. Closes ticket 28026. Patch by "mtigas".
diff --git a/changes/ticket28113 b/changes/ticket28113
deleted file mode 100644
index 30dd825a9b..0000000000
--- a/changes/ticket28113
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (relay shutdown, systemd):
-    - Notify systemd of ShutdownWaitLength so it can be set to longer than
-      systemd's TimeoutStopSec. In tor's systemd service file, set
-      TimeoutSec to 60 seconds, to allow tor some time to shut down.
-      Fixes bug 28113; bugfix on 0.2.6.2-alpha.
diff --git a/changes/ticket28128 b/changes/ticket28128
deleted file mode 100644
index 6d08c74242..0000000000
--- a/changes/ticket28128
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation (hidden service manpage, sandbox):
-    - Document in the man page that changing ClientOnionAuthDir value or
-      adding a new file in the directory will not work at runtime upon sending
-      a HUP if Sandbox 1. Closes ticket 28128.
diff --git a/changes/ticket28229_diag b/changes/ticket28229_diag
deleted file mode 100644
index cd02b81faa..0000000000
--- a/changes/ticket28229_diag
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing:
-    - Increase logging and tag all log entries with timestamps
-      in test_rebind.py. Provides diagnostics for issue 28229.
diff --git a/changes/ticket28275 b/changes/ticket28275
deleted file mode 100644
index eadca86b7b..0000000000
--- a/changes/ticket28275
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Documentation (hidden service v3, man page):
-    - Note in the man page that the only real way to fully revoke an onion
-      service v3 client authorization is by restarting the tor process. Closes
-      ticket 28275.
diff --git a/changes/ticket28318 b/changes/ticket28318
deleted file mode 100644
index 24dc1e9580..0000000000
--- a/changes/ticket28318
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (Windows, continuous integration):
-    - Build tor on Windows Server 2012 R2 and Windows Server 2016 using
-      Appveyor's CI. Closes ticket 28318.
diff --git a/changes/ticket28459 b/changes/ticket28459
deleted file mode 100644
index 6b5839b52b..0000000000
--- a/changes/ticket28459
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (continuous integration, Windows):
-    - Always show the configure and test logs, and upload them as build
-      artifacts, when building for Windows using Appveyor CI.
-      Implements 28459.
diff --git a/changes/ticket28574 b/changes/ticket28574
deleted file mode 100644
index 562810f511..0000000000
--- a/changes/ticket28574
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (continuous integration, Windows):
-    - Explicitly specify the path to the OpenSSL library and do not download
-      OpenSSL from Pacman, but instead use the library that is already provided
-      by AppVeyor. Fixes bug 28574; bugfix on master.
diff --git a/changes/ticket28668 b/changes/ticket28668
deleted file mode 100644
index 6386e0051f..0000000000
--- a/changes/ticket28668
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (testing):
-    - Treat all unexpected ERR and BUG messages as test failures.
-      Closes ticket 28668.
diff --git a/changes/ticket28669 b/changes/ticket28669
deleted file mode 100644
index 32c6114ffc..0000000000
--- a/changes/ticket28669
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor bugfix (hidden service v3, client):
-    - Avoid a BUG() stacktrace in case a SOCKS connection is found waiting for
-      the descriptor while we do have it in the cache. There is a rare case
-      when this can happen. Now, tor will recover and retry the descriptor.
-      Fixes bug 28669; bugfix on 0.3.2.4-alpha.
-    
diff --git a/changes/ticket28795 b/changes/ticket28795
deleted file mode 100644
index 6ae72562bf..0000000000
--- a/changes/ticket28795
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor features (fallback directory list):
-    - Replace the 157 fallbacks originally introduced in Tor 0.3.5.6-rc
-      in December 2018 (of which ~122 were still functional), with a
-      list of 148 fallbacks (70 new, 78 existing, 79 removed) generated
-      in June 2019. Closes ticket 28795.
diff --git a/changes/ticket28838 b/changes/ticket28838
deleted file mode 100644
index 6c290bf82b..0000000000
--- a/changes/ticket28838
+++ /dev/null
@@ -1,8 +0,0 @@
-  o Minor features (performance):
-    - Remove about 96% of the work from the function that we run at
-      startup to test our curve25519_basepoint implementation.  Since
-      this function has yet to find an actual failure, we'll only
-      run it for 8 iterations instead of 200. Based on our profile
-      information, this change should save around 8% of our startup
-      time on typical desktops, and may have a similar effect on
-      other platforms.  Closes ticket 28838.
diff --git a/changes/ticket28851 b/changes/ticket28851
deleted file mode 100644
index bab0318662..0000000000
--- a/changes/ticket28851
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (performance):
-    - Stop re-validating our hardcoded Diffie-Hellman parameters on every
-      startup. Doing this wasted time and cycles, especially on low-powered
-      devices. Closes ticket 28851.
diff --git a/changes/ticket28879 b/changes/ticket28879
deleted file mode 100644
index 126420f6ca..0000000000
--- a/changes/ticket28879
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (correctness):
-    - Fix an unreached code-path where we checked the value of "hostname"
-      inside send_resolved_hostnam_cell(). Previously, we used it before
-      checking it; now we check it first. Fixes bug 28879; bugfix on
-      0.1.2.7-alpha.
diff --git a/changes/ticket28881 b/changes/ticket28881
deleted file mode 100644
index 1b015a6c37..0000000000
--- a/changes/ticket28881
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Code simplification and refactoring:
-    -  When parsing a port configuration, make it more
-       obvious to static analyzer tools that we will always initialize the
-       address. Closes ticket 28881.
diff --git a/changes/ticket28883 b/changes/ticket28883
deleted file mode 100644
index 1d8b6cb416..0000000000
--- a/changes/ticket28883
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (testing):
-    - Make sure that test_rebind.py actually obeys its timeout, even
-      when it receives a large number of log messages. Fixes bug 28883;
-      bugfix on 0.3.5.4-alpha.
diff --git a/changes/ticket28912 b/changes/ticket28912
deleted file mode 100644
index 4119b778bc..0000000000
--- a/changes/ticket28912
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (relay, directory):
-    - A connection serving directory information wouldn't get reactivated after
-      the first chunk of data was sent (usually 32KB). Tor now always activate
-      the main loop event that goes through these connections as long as at
-      least one connection is still active. Fixes bug 28912; bugfix on
-      0.3.4.1-alpha. Patch by "cypherpunks3".
diff --git a/changes/ticket28924 b/changes/ticket28924
deleted file mode 100644
index 055a6cf285..0000000000
--- a/changes/ticket28924
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (compilation):
-    - When possible, place our warning flags in a separate file, to avoid
-      flooding verbose build logs to an unacceptable amount. Closes ticket
-      28924.
diff --git a/changes/ticket28973 b/changes/ticket28973
deleted file mode 100644
index b1d208ee51..0000000000
--- a/changes/ticket28973
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Minor features (OpenSSL bug workaround):
-    - Work around a bug in OpenSSL 1.1.1a, which prevented the TLS 1.3
-      key export function from handling long labels.  When this bug
-      is detected, Tor will disable TLS 1.3.  We recommend upgrading to
-      a version of OpenSSL without this bug when it becomes available.
-      Closes ticket 28973.
diff --git a/changes/ticket29026 b/changes/ticket29026
deleted file mode 100644
index 1db873dfcf..0000000000
--- a/changes/ticket29026
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (compilation):
-    - Compile correctly when OpenSSL is built with engine support
-      disabled, or with deprecated APIs disabled. Closes ticket
-      29026. Patches from "Mangix".
diff --git a/changes/ticket29160 b/changes/ticket29160
deleted file mode 100644
index 8e11183064..0000000000
--- a/changes/ticket29160
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (tests):
-    - Do not log an error-level message if we fail to find an IPv6
-      network interface from the unit tests. Fixes bug 29160; bugfix on
-      0.2.7.3-rc.
diff --git a/changes/ticket29168 b/changes/ticket29168
deleted file mode 100644
index 65c5232f65..0000000000
--- a/changes/ticket29168
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Major bugfixes (cell scheduler, KIST):
-    - Make KIST to always take into account the outbuf length when computing
-      what we can actually put in the outbuf. This could lead to the outbuf
-      being filled up and thus a possible memory DoS vector. TROVE-2019-001.
-      Fixes bug 29168; bugfix on 0.3.2.1-alpha.
diff --git a/changes/ticket29435 b/changes/ticket29435
deleted file mode 100644
index d48ae98e4b..0000000000
--- a/changes/ticket29435
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor bugfixes (testing):
-    - Fix our gcov wrapper script to look for object files at the
-      correct locations. Fixes bug 29435; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket29617 b/changes/ticket29617
deleted file mode 100644
index 4d50ea9627..0000000000
--- a/changes/ticket29617
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (out-of-memory handler):
-    - When purging the DNS cache because of an out-of-memory condition,
-      try purging just the older entries at first.  Previously, we would
-      purge the whole thing. Fixes bug 29617; bugfix on 0.3.5.1-alpha.
diff --git a/changes/ticket29669 b/changes/ticket29669
new file mode 100644
index 0000000000..f7e98a16ce
--- /dev/null
+++ b/changes/ticket29669
@@ -0,0 +1,3 @@
+  o Minor feature (hidden service, control port):
+    - The ADD_ONION key blob keyword "BEST" now defaults from RSA1024 (v2) to
+      ED25519-V3 (v3). Closes ticket 29669.
diff --git a/changes/ticket29702 b/changes/ticket29702
deleted file mode 100644
index e1cc1f867b..0000000000
--- a/changes/ticket29702
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Testing:
-    - Specify torrc paths (with empty files) when launching tor in
-      integration tests; refrain from reading user and system torrcs.
-      Resolves issue 29702.
diff --git a/changes/ticket29806 b/changes/ticket29806
deleted file mode 100644
index 6afefd4c04..0000000000
--- a/changes/ticket29806
+++ /dev/null
@@ -1,7 +0,0 @@
-  o Minor features (bandwidth authority):
-    - Make bandwidth authorities to ignore relays that are reported in the
-      bandwidth file with the key-value "vote=0".
-      This change allows to report the relays that were not measured due
-      some failure and diagnose the reasons without the bandwidth being included in the
-      bandwidth authorities vote.
-      Closes ticket 29806.
diff --git a/changes/ticket29962 b/changes/ticket29962
deleted file mode 100644
index e36cc0cf9a..0000000000
--- a/changes/ticket29962
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (continuous integration):
-    - On Travis Rust builds, cleanup Rust registry and refrain from caching
-      target/ directory to speed up builds. Resolves issue 29962.
diff --git a/changes/ticket30117 b/changes/ticket30117
deleted file mode 100644
index 5b6e6dabf7..0000000000
--- a/changes/ticket30117
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, tell timelimit to use stem's backtrace signals. And launch
-      python directly from timelimit, so python receives the signals from
-      timelimit, rather than make. Closes ticket 30117.
diff --git a/changes/ticket30213 b/changes/ticket30213
deleted file mode 100644
index acb7614807..0000000000
--- a/changes/ticket30213
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (continuous integration):
-    - Remove sudo configuration lines from .travis.yml as they are no longer
-      needed with current Travis build environment. Resolves issue 30213.
diff --git a/changes/ticket30234 b/changes/ticket30234
deleted file mode 100644
index 5a0076bad2..0000000000
--- a/changes/ticket30234
+++ /dev/null
@@ -1,2 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, show stem's tor log after failure. Closes ticket 30234.
diff --git a/changes/ticket30454 b/changes/ticket30454
deleted file mode 100644
index 77c45d0feb..0000000000
--- a/changes/ticket30454
+++ /dev/null
@@ -1,10 +0,0 @@
-  o Major bugfixes (hidden service v3):
-    - An intro point could try to send an INTRODUCE_ACK with a status code
-      that it wasn't able to encode leading to a hard assert() of the relay.
-      Fortunately, that specific code path can not be reached thus this issue
-      can't be triggered. We've consolidated the ABI values into trunnel now.
-      Fixes bug 30454; bugfix on 0.3.0.1-alpha.
-    - HSv3 client will now be able to properly handle unknown status code from
-      a INTRODUCE_ACK cell (nack) even if they do not know it. The NACK
-      behavior will stay the same. This will allow us to extend status code if
-      we want in the future without breaking the normal client behavior.
diff --git a/changes/ticket30591 b/changes/ticket30591
deleted file mode 100644
index f97c024009..0000000000
--- a/changes/ticket30591
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, make stem log a controller trace to the console. And tail
-      stem's tor log after failure. Closes ticket 30591.
diff --git a/changes/ticket30694 b/changes/ticket30694
deleted file mode 100644
index 70dbf6481a..0000000000
--- a/changes/ticket30694
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Testing (continuous integration):
-    - In Travis, only run the stem tests that use a tor binary.
-      Closes ticket 30694.
diff --git a/changes/ticket30871 b/changes/ticket30871
deleted file mode 100644
index 81c076bb02..0000000000
--- a/changes/ticket30871
+++ /dev/null
@@ -1,6 +0,0 @@
-  o Major bugfixes (circuit build, guard):
-    - When considering upgrading circuits from "waiting for guard" to "open",
-      always ignore the ones that are mark for close. Else, we can end up in
-      the situation where a subsystem is notified of that circuit opening but
-      still marked for close leading to undesirable behavior. Fixes bug 30871;
-      bugfix on 0.3.0.1-alpha.
diff --git a/changes/ticket31091 b/changes/ticket31091
new file mode 100644
index 0000000000..3cb9a2c37b
--- /dev/null
+++ b/changes/ticket31091
@@ -0,0 +1,3 @@
+  o Minor bugfixes (pluggable transports):
+    - Remove overly strict assertions that triggers when a pluggable transport
+      is spawned in an unsuccessful manner. Fixes bug 31091; bugfix on 0.4.0.1-alpha.
diff --git a/changes/ticket31189 b/changes/ticket31189
new file mode 100644
index 0000000000..318941c794
--- /dev/null
+++ b/changes/ticket31189
@@ -0,0 +1,3 @@
+  o Documentation:
+    - Correct the description of "GuardLifetime". Fixes bug 31189; bugfix on
+      0.3.0.1-alpha.
diff --git a/changes/ticket31372_appveyor b/changes/ticket31372_appveyor
deleted file mode 100644
index e7bb03182e..0000000000
--- a/changes/ticket31372_appveyor
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (continuous integration):
-    - When building on Appveyor, pass the "-k" flag to make, so that
-      we are informed of all compilation failures, not just the first
-      one or two. Closes part of ticket 31372.
diff --git a/changes/ticket31372_travis b/changes/ticket31372_travis
deleted file mode 100644
index 403869b2ed..0000000000
--- a/changes/ticket31372_travis
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (continuous integration):
-    - When building on Travis, pass the "-k" flag to make, so that
-      we are informed of all compilation failures, not just the first
-      one or two. Closes part of ticket 31372.
diff --git a/changes/ticket31374 b/changes/ticket31374
deleted file mode 100644
index e8eef9cd49..0000000000
--- a/changes/ticket31374
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation warning):
-    - Fix a compilation warning on Windows about casting a function
-      pointer for GetTickCount64(). Fixes bug 31374; bugfix on
-      0.2.9.1-alpha.
diff --git a/changes/ticket31406 b/changes/ticket31406
deleted file mode 100644
index 0ebe6f6c47..0000000000
--- a/changes/ticket31406
+++ /dev/null
@@ -1,3 +0,0 @@
-  o Minor features (directory authority):
-    - A new IP address the directory authority "dizum" has been changed. Closes
-      ticket 31406;
diff --git a/changes/ticket31466 b/changes/ticket31466
deleted file mode 100644
index e535b4502e..0000000000
--- a/changes/ticket31466
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (logging):
-    - Rate-limit our the logging message about the obsolete .exit notation.
-      Previously, there was no limit on this warning, which could potentially
-      be triggered many times by a hostile website. Fixes bug 31466;
-      bugfix on 0.2.2.1-alpha.
diff --git a/changes/ticket31554 b/changes/ticket31554
deleted file mode 100644
index 73f4159ff3..0000000000
--- a/changes/ticket31554
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor features (stem tests):
-    - Change "make test-stem" so it only runs the stem tests that use tor.
-      This change makes test-stem faster and more reliable.
-      Closes ticket 31554.
diff --git a/changes/ticket31673 b/changes/ticket31673
deleted file mode 100644
index 3b2bb4a46e..0000000000
--- a/changes/ticket31673
+++ /dev/null
@@ -1,3 +0,0 @@
-  o New system requirements (build system):
-    - Do not include the deprecated <sys/sysctl.h> on Linux or Windows system.
-      Closes 31673;
diff --git a/changes/ticket31682 b/changes/ticket31682
new file mode 100644
index 0000000000..9777dec1f3
--- /dev/null
+++ b/changes/ticket31682
@@ -0,0 +1,3 @@
+  o Minor bugfixes (hidden service v3, coverity):
+    - Fix an implicit conversion from ssize_t to size_t discovered by Coverity.
+      Fixes bug 31682; bugfix on 0.4.2.1-alpha.
diff --git a/changes/ticket31687_1 b/changes/ticket31687_1
deleted file mode 100644
index 2f4d440974..0000000000
--- a/changes/ticket31687_1
+++ /dev/null
@@ -1,4 +0,0 @@
-  o Minor bugfixes (compilation):
-    - Suppress spurious float-conversion warnings from GCC when calling
-      floating-point classifier functions on FreeBSD.  Fixes part of bug
-      31687; bugfix on 0.3.1.5-alpha.
diff --git a/changes/ticket31687_2 b/changes/ticket31687_2
deleted file mode 100644
index eadc698275..0000000000
--- a/changes/ticket31687_2
+++ /dev/null
@@ -1,5 +0,0 @@
-  o Minor bugfixes (FreeBSD, PF-based proxy, IPv6):
-    - When extracting an IPv6 address from a PF-based proxy, verify
-      that we are actually configured to receive an IPv6 address,
-      and log an internal error if not. Fixes part of bug 31687;
-      bugfix on 0.2.3.4-alpha.
diff --git a/changes/ticket31841 b/changes/ticket31841
new file mode 100644
index 0000000000..6e7fbc1da1
--- /dev/null
+++ b/changes/ticket31841
@@ -0,0 +1,5 @@
+  o Minor features (testing):
+    - When running tests that attempt to look up hostname, replace the libc
+      name lookup functions with ones that do not actually touch the network.
+      This way, the tests complete more quickly in the presence of a slow or
+      missing DNS resolver. Closes ticket 31841.
diff --git a/changes/ticket31958 b/changes/ticket31958
new file mode 100644
index 0000000000..8206064dfe
--- /dev/null
+++ b/changes/ticket31958
@@ -0,0 +1,5 @@
+  o Minor bugfixes (directory):
+    - When checking if a directory connection is anonymous, test if the circuit
+      was marked for close before looking at its channel. This avoids a BUG()
+      stacktrace in case it was previously closed. Fixes bug 31958; bugfix on
+      0.4.2.1-alpha.
diff --git a/changes/ticket32063 b/changes/ticket32063
new file mode 100644
index 0000000000..2c0246917c
--- /dev/null
+++ b/changes/ticket32063
@@ -0,0 +1,3 @@
+  o Minor bugfixes (hs-v3, memory leak):
+    - Fix memory leak in unlikely error code path when encoding HS DoS establish
+      intro extension cell. Fixes bug 32063; bugfix on 0.4.2.1-alpha.
diff --git a/changes/ticket32191 b/changes/ticket32191
new file mode 100644
index 0000000000..6988328115
--- /dev/null
+++ b/changes/ticket32191
@@ -0,0 +1,3 @@
+  o Minor features (build system):
+    - Make pkg-config use --prefix when cross-compiling, if PKG_CONFIG_PATH
+      is not set. Closes ticket 32191.
diff --git a/changes/ticket32705_disable b/changes/ticket32705_disable
new file mode 100644
index 0000000000..6d5b0779ab
--- /dev/null
+++ b/changes/ticket32705_disable
@@ -0,0 +1,7 @@
+  o Minor bugfixes (testing):
+    - When TOR_DISABLE_PRACTRACKER is set, do not apply it to the
+      test_practracker.sh script.  Doing so caused a test failure.
+      Fixes bug 32705; bugfix on 0.4.2.1-alpha.
+    - When TOR_DISABLE_PRACTRACKER is set, log a notice to stderr
+      when skipping practracker checks.
+      Fixes bug 32705; bugfix on 0.4.2.1-alpha.
diff --git a/changes/ticket32765 b/changes/ticket32765
new file mode 100644
index 0000000000..a9663a5df3
--- /dev/null
+++ b/changes/ticket32765
@@ -0,0 +1,4 @@
+  o Minor bugfixes (correctness checks):
+    - Use GCC/Clang's printf-checking feature to make sure that
+      tor_assertf() arguments are correctly typed. Fixes bug 32765;
+      bugfix on 0.4.1.1-alpha.
diff --git a/changes/ticket33029 b/changes/ticket33029
new file mode 100644
index 0000000000..c32ee4ad84
--- /dev/null
+++ b/changes/ticket33029
@@ -0,0 +1,5 @@
+  o Major bugfixes (directory authority):
+    - Directory authorities will now send a 503 (not enough bandwidth) code to
+      clients when under bandwidth pressure. Known relays and other authorities
+      will always be answered regardless of the bandwidth situation. Fixes bug
+      33029; bugfix on 0.1.2.5-alpha.
diff --git a/changes/ticket33290 b/changes/ticket33290
new file mode 100644
index 0000000000..882764020e
--- /dev/null
+++ b/changes/ticket33290
@@ -0,0 +1,4 @@
+  o Minor features (diagnostic):
+    - Improve assertions and add some memory-poisoning code to try to track
+      down possible causes of a rare crash (32564) in the EWMA code.
+      Closes ticket 33290.
diff --git a/changes/ticket33619 b/changes/ticket33619
new file mode 100644
index 0000000000..3c52858b35
--- /dev/null
+++ b/changes/ticket33619
@@ -0,0 +1,5 @@
+  o Major bugfixes (circuit padding, memory leaks):
+    - Avoid a remotely triggered memory leak in the case that a circuit
+      padding machine is somehow negotiated twice on the same circuit. Fixes
+      bug 33619; bugfix on 0.4.0.1-alpha.  Found by Tobias Pulls.  This is
+      also tracked as TROVE-2020-004.
diff --git a/changes/ticket33623 b/changes/ticket33623
new file mode 100644
index 0000000000..528af3ca02
--- /dev/null
+++ b/changes/ticket33623
@@ -0,0 +1,2 @@
+  o Minor feature (sendme, flow control):
+    - Default on sending SENDME version 1 cells. Closes ticket 33623.
diff --git a/configure.ac b/configure.ac
index f30a7efa91..af143e44ad 100644
--- a/configure.ac
+++ b/configure.ac
@@ -4,10 +4,19 @@ dnl Copyright (c) 2007-2019, The Tor Project, Inc.
 dnl See LICENSE for licensing information
 
 AC_PREREQ([2.63])
-AC_INIT([tor],[0.3.5.11-dev])
+AC_INIT([tor],[0.4.2.8-dev])
 AC_CONFIG_SRCDIR([src/app/main/tor_main.c])
 AC_CONFIG_MACRO_DIR([m4])
 
+# DO NOT EDIT THIS DEFINITION BY HAND UNLESS YOU KNOW WHAT YOU'RE DOING.
+#
+# The update_versions.py script updates this definition when the
+# version number changes.  Tor uses it to make sure that it
+# only shuts down for missing "required protocols" when those protocols
+# are listed as required by a consensus after this date.
+AC_DEFINE(APPROX_RELEASE_DATE, ["2020-07-09"], # for 0.4.2.8-dev
+          [Approximate date when this software was released. (Updated when the version changes.)])
+
 # "foreign" means we don't follow GNU package layout standards
 # "1.11" means we require automake version 1.11 or newer
 # "subdir-objects" means put .o files in the same directory as the .c files
@@ -20,6 +29,18 @@ AC_USE_SYSTEM_EXTENSIONS
 AC_CANONICAL_HOST
 
 PKG_PROG_PKG_CONFIG
+if test "x$PKG_CONFIG" = "x" ; then
+    pkg_config_user_action="install pkg-config, and check the PKG_CONFIG_PATH environment variable"
+    AC_MSG_NOTICE([Some libraries need pkg-config, including systemd, nss, lzma, zstd, and custom mallocs.])
+    AC_MSG_NOTICE([To use those libraries, $pkg_config_user_action.])
+else
+    pkg_config_user_action="check the PKG_CONFIG_PATH environment variable"
+fi
+
+if test "x$PKG_CONFIG_PATH" = "x" && test "x$prefix" != "xNONE" && test "$host" != "$build"; then
+   export PKG_CONFIG_PATH=$prefix/lib/pkgconfig
+   AC_MSG_NOTICE([set PKG_CONFIG_PATH=$PKG_CONFIG_PATH to support cross-compiling])
+fi
 
 AC_ARG_ENABLE(openbsd-malloc,
    AS_HELP_STRING(--enable-openbsd-malloc, [use malloc code from OpenBSD.  Linux only. Deprecated: see --with-malloc]))
@@ -96,6 +117,12 @@ if test "$enable_memory_sentinels" = "no"; then
            [Defined if we're turning off memory safety code to look for bugs])
 fi
 
+AC_ARG_ENABLE(manpage,
+              AS_HELP_STRING(--disable-manpage, [Disable manpage generation.]))
+
+AC_ARG_ENABLE(html-manual,
+              AS_HELP_STRING(--disable-html-manual, [Disable HTML documentation.]))
+
 AC_ARG_ENABLE(asciidoc,
      AS_HELP_STRING(--disable-asciidoc, [don't use asciidoc (disables building of manpages)]),
      [case "${enableval}" in
@@ -145,13 +172,13 @@ if test "x$have_systemd" = "xyes"; then
     TOR_SYSTEMD_CFLAGS="${SYSTEMD_CFLAGS}"
     TOR_SYSTEMD_LIBS="${SYSTEMD_LIBS}"
     PKG_CHECK_MODULES(LIBSYSTEMD209, [libsystemd >= 209],
-         [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or more])], [])
+         [AC_DEFINE(HAVE_SYSTEMD_209,1,[Have systemd v209 or greater])], [])
 fi
 AC_SUBST(TOR_SYSTEMD_CFLAGS)
 AC_SUBST(TOR_SYSTEMD_LIBS)
 
 if test "x$enable_systemd" = "xyes" -a "x$have_systemd" != "xyes" ; then
-    AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found])
+    AC_MSG_ERROR([Explicitly requested systemd support, but systemd not found, $pkg_config_user_action, or set SYSTEMD_CFLAGS and SYSTEMD_LIBS.])
 fi
 
 case "$host" in
@@ -236,11 +263,11 @@ m4_define(MODULES, dirauth)
 dnl Directory Authority module.
 AC_ARG_ENABLE([module-dirauth],
               AS_HELP_STRING([--disable-module-dirauth],
-                             [Do not build tor with the dirauth module]),
-              [], dnl Action if-given
-              AC_DEFINE([HAVE_MODULE_DIRAUTH], [1],
-                        [Compile with Directory Authority feature support]))
-AM_CONDITIONAL(BUILD_MODULE_DIRAUTH, [test "x$enable_module_dirauth" != "xno"])
+                             [Build tor without the Directory Authority module: tor can not run as a directory authority or bridge authority]))
+AM_CONDITIONAL(BUILD_MODULE_DIRAUTH,[test "x$enable_module_dirauth" != "xno"])
+AM_COND_IF(BUILD_MODULE_DIRAUTH,
+           AC_DEFINE([HAVE_MODULE_DIRAUTH], [1],
+                     [Compile with Directory Authority feature support]))
 
 dnl Helper variables.
 TOR_MODULES_ALL_ENABLED=
@@ -290,6 +317,8 @@ AC_PATH_PROG([ASCIIDOC], [asciidoc], none)
 AC_PATH_PROGS([A2X], [a2x a2x.py], none)
 
 AM_CONDITIONAL(USE_ASCIIDOC, test "x$asciidoc" = "xtrue")
+AM_CONDITIONAL(BUILD_MANPAGE, [test "x$enable_manpage" != "xno"])
+AM_CONDITIONAL(BUILD_HTML_DOCS, [test "x$enable_html_manual" != "xno"])
 
 AM_PROG_CC_C_O
 AC_PROG_CC_C99
@@ -519,15 +548,6 @@ if test "x$enable_rust" = "xyes"; then
     fi
   fi
 
-  dnl This is a workaround for #46797
-  dnl (a.k.a https://github.com/rust-lang/rust/issues/46797 ).  Once the
-  dnl upstream bug is fixed, we can remove this workaround.
-  case "$host_os" in
-      darwin*)
-        TOR_RUST_EXTRA_LIBS="-lresolv"
-	;;
-  esac
-
   dnl For now both MSVC and MinGW rust libraries will output static libs with
   dnl the MSVC naming convention.
   if test "$bwin32" = "true"; then
@@ -566,14 +586,12 @@ if test "x$enable_rust" = "xyes"; then
   if test "x$RUSTC_VERSION_MAJOR" = "x" -o "x$RUSTC_VERSION_MINOR" = "x"; then
     AC_MSG_ERROR([rustc version couldn't be identified])
   fi
-  if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 14; then
-    AC_MSG_ERROR([rustc must be at least version 1.14])
+  if test "$RUSTC_VERSION_MAJOR" -lt 2 -a "$RUSTC_VERSION_MINOR" -lt 31; then
+    AC_MSG_ERROR([rustc must be at least version 1.31.0])
   fi
   AC_MSG_RESULT([$RUSTC_VERSION])
 fi
 
-AC_SUBST(TOR_RUST_EXTRA_LIBS)
-
 AC_SEARCH_LIBS(socket, [socket network])
 AC_SEARCH_LIBS(gethostbyname, [nsl])
 AC_SEARCH_LIBS(dlopen, [dl])
@@ -623,8 +641,10 @@ AC_CHECK_FUNCS(
 	llround \
 	localtime_r \
 	lround \
+	madvise \
 	memmem \
 	memset_s \
+	minherit \
 	mmap \
 	pipe \
 	pipe2 \
@@ -828,6 +848,8 @@ fi
 dnl Now check for particular libevent functions.
 AC_CHECK_FUNCS([evutil_secure_rng_set_urandom_device_file \
                 evutil_secure_rng_add_bytes \
+                evdns_base_get_nameserver_addr \
+
 ])
 
 LIBS="$save_LIBS"
@@ -876,7 +898,7 @@ if test "x$enable_nss" = "xyes"; then
   PKG_CHECK_MODULES(NSS,
      [nss],
      [have_nss=yes],
-     [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it.])])
+     [have_nss=no; AC_MSG_ERROR([You asked for NSS but I can't find it, $pkg_config_user_action, or set NSS_CFLAGS and NSS_LIBS.])])
   AC_SUBST(NSS_CFLAGS)
   AC_SUBST(NSS_LIBS)
 fi
@@ -963,21 +985,24 @@ AC_CHECK_MEMBERS([struct ssl_method_st.get_cipher_by_char], , ,
 [#include <openssl/ssl.h>
 ])
 
+dnl OpenSSL functions which we might not have.  In theory, we could just
+dnl check the openssl version number, but in practice that gets pretty
+dnl confusing with LibreSSL, OpenSSL, and various distributions' patches
+dnl to them.
 AC_CHECK_FUNCS([ \
 		ERR_load_KDF_strings \
+		EVP_PBE_scrypt \
+		EVP_sha3_256 \
+		SSL_CIPHER_find \
+		SSL_CTX_set1_groups_list \
+		SSL_CTX_set_security_level \
 		SSL_SESSION_get_master_key \
+		SSL_get_client_ciphers \
+		SSL_get_client_random \
 		SSL_get_server_random \
-                SSL_get_client_ciphers \
-                SSL_get_client_random \
-                SSL_CTX_set1_groups_list \
-		SSL_CIPHER_find \
-                SSL_CTX_set_security_level \
-		TLS_method
+		TLS_method \
 	       ])
 
-dnl Check if OpenSSL has scrypt implementation.
-AC_CHECK_FUNCS([ EVP_PBE_scrypt ])
-
 dnl Check if OpenSSL structures are opaque
 AC_CHECK_MEMBERS([SSL.state], , ,
 [#include <openssl/ssl.h>
@@ -989,6 +1014,15 @@ AC_CHECK_SIZEOF(SHA_CTX, , [AC_INCLUDES_DEFAULT()
 
 fi # enable_nss
 
+dnl We will someday make KECCAK_TINY optional, but for now we still need
+dnl it for SHAKE, since OpenSSL's SHAKE can't be squeezed more than
+dnl once.  See comment in the definition of crypto_xof_t.
+
+dnl AM_CONDITIONAL(BUILD_KECCAK_TINY,
+dnl   test "x$ac_cv_func_EVP_sha3_256" != "xyes")
+
+AM_CONDITIONAL(BUILD_KECCAK_TINY, true)
+
 dnl ======================================================================
 dnl Can we use KIST?
 
@@ -1070,7 +1104,7 @@ else
                       have_lzma=no)
 
     if test "x$have_lzma" = "xno" ; then
-        AC_MSG_WARN([Unable to find liblzma.])
+        AC_MSG_WARN([Unable to find liblzma, $pkg_config_user_action, or set LZMA_CFLAGS and LZMA_LIBS.])
     fi
 fi
 
@@ -1102,7 +1136,7 @@ else
                       have_zstd=no)
 
     if test "x$have_zstd" = "xno" ; then
-        AC_MSG_WARN([Unable to find libzstd.])
+        AC_MSG_WARN([Unable to find libzstd, $pkg_config_user_action, or set ZSTD_CFLAGS and ZSTD_LIBS.])
     fi
 fi
 
@@ -1190,6 +1224,17 @@ m4_ifdef([AS_VAR_IF],[
        TOR_CHECK_LDFLAGS(-pie, "$all_ldflags_for_check", "$all_libs_for_check")
     fi
     TOR_TRY_COMPILE_WITH_CFLAGS(-fwrapv, also_link, CFLAGS_FWRAPV="-fwrapv", true)
+
+   AC_MSG_CHECKING([whether we can run hardened binaries])
+   AC_RUN_IFELSE([AC_LANG_PROGRAM([], [return 0;])],
+        [AC_MSG_RESULT([yes])],
+        [AC_MSG_RESULT([no])
+         AC_MSG_ERROR([dnl
+ We can link with compiler hardening options, but we can't run with them.
+ That's a bad sign! If you must, you can pass --disable-gcc-hardening to
+ configure, but it would be better to figure out what the underlying problem
+ is.])],
+        [AC_MSG_RESULT([cross])])
 fi
 
 if test "$fragile_hardening" = "yes"; then
@@ -1212,6 +1257,11 @@ if test "$fragile_hardening" = "yes"; then
       AC_MSG_ERROR([The compiler supports -fsanitize=undefined, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*])
     fi
 
+   TOR_TRY_COMPILE_WITH_CFLAGS([-fno-sanitize=float-divide-by-zero], also_link, CFLAGS_UBSAN="-fno-sanitize=float-divide-by-zero", true)
+    if test "$tor_cv_cflags__fno_sanitize_float_divide_by_zero" = "yes" && test "$tor_can_link__fno_sanitize_float_divide_by_zero" != "yes"; then
+      AC_MSG_ERROR([The compiler supports -fno-sanitize=float-divide-by-zero, but for some reason I was not able to link when using it. Are you missing run-time support? With GCC you need libasan.*, and with Clang you need libclang_rt.ubsan*])
+    fi
+
 TOR_CHECK_CFLAGS([-fno-omit-frame-pointer])
 fi
 
@@ -1466,6 +1516,7 @@ AC_CHECK_HEADERS([errno.h \
 		  inttypes.h \
 		  limits.h \
 		  linux/types.h \
+		  mach/vm_inherit.h \
 		  machine/limits.h \
 		  malloc.h \
 		  malloc/malloc.h \
@@ -1601,6 +1652,7 @@ AC_CHECK_MEMBERS([struct timeval.tv_sec], , ,
 AC_CHECK_SIZEOF(char)
 AC_CHECK_SIZEOF(short)
 AC_CHECK_SIZEOF(int)
+AC_CHECK_SIZEOF(unsigned int)
 AC_CHECK_SIZEOF(long)
 AC_CHECK_SIZEOF(long long)
 AC_CHECK_SIZEOF(__int64)
@@ -1863,8 +1915,7 @@ if test "$tor_cv_uint8_uchar" = "no"; then
 fi
 
 AC_ARG_WITH(tcmalloc,
-AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library. Deprecated; see --with-malloc]),
-[ tcmalloc=yes ], [ tcmalloc=no ])
+AS_HELP_STRING(--with-tcmalloc, [use tcmalloc memory allocation library. Deprecated; see --with-malloc]))
 
 default_malloc=system
 
@@ -1873,7 +1924,7 @@ if test "x$enable_openbsd_malloc" = "xyes" ; then
   default_malloc=openbsd
 fi
 
-if test "x$tcmalloc" = "xyes"; then
+if test "x$with_tcmalloc" = "xyes"; then
   AC_MSG_NOTICE([The --with-tcmalloc argument is deprecated; use --with-malloc=tcmalloc instead.])
   default_malloc=tcmalloc
 fi
@@ -1891,7 +1942,7 @@ AS_CASE([$malloc],
 			have_tcmalloc=no)
 
       if test "x$have_tcmalloc" = "xno" ; then
-          AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc.])
+          AC_MSG_ERROR([Unable to find tcmalloc requested by --with-malloc, $pkg_config_user_action, or set TCMALLOC_CFLAGS and TCMALLOC_LIBS.])
       fi
 
       CFLAGS="$CFLAGS $TCMALLOC_CFLAGS"
@@ -1904,8 +1955,8 @@ AS_CASE([$malloc],
 			have_jemalloc=yes,
 			have_jemalloc=no)
 
-      if test "x$have_tcmalloc" = "xno" ; then
-          AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc.])
+      if test "x$have_jemalloc" = "xno" ; then
+          AC_MSG_ERROR([Unable to find jemalloc requested by --with-malloc, $pkg_config_user_action, or set JEMALLOC_CFLAGS and JEMALLOC_LIBS.])
       fi
 
       CFLAGS="$CFLAGS $JEMALLOC_CFLAGS"
@@ -2452,16 +2503,12 @@ AC_CONFIG_FILES([
 	Doxyfile
 	Makefile
 	config.rust
-	contrib/dist/suse/tor.sh
 	contrib/operator-tools/tor.logrotate
-	contrib/dist/tor.sh
-	contrib/dist/torctl
 	contrib/dist/tor.service
 	src/config/torrc.sample
 	src/config/torrc.minimal
 	src/rust/.cargo/config
 	scripts/maint/checkOptionDocs.pl
-	scripts/maint/updateVersions.pl
         warning_flags
 ])
 
diff --git a/contrib/README b/contrib/README
index 3a94bb5016..735fcf4c9f 100644
--- a/contrib/README
+++ b/contrib/README
@@ -34,8 +34,6 @@ tools.  Everybody likes to write init scripts differently, it seems.
 
 tor.service is a sample service file for use with systemd.
 
-The suse/ subdirectory contains files used by the suse distribution.
-
 operator-tools/ -- Tools for Tor relay operators
 ------------------------------------------------
 
diff --git a/contrib/client-tools/torify b/contrib/client-tools/torify
index 54acfed654..ac4c9b5c7f 100755
--- a/contrib/client-tools/torify
+++ b/contrib/client-tools/torify
@@ -53,7 +53,7 @@ pathfind() {
 
 if pathfind torsocks; then
     exec torsocks "$@"
-    echo "$0: Failed to exec torsocks $@" >&2
+    echo "$0: Failed to exec torsocks $*" >&2
     exit 1
 else
     echo "$0: torsocks not found in your PATH.  Perhaps it isn't installed?  (tsocks is no longer supported, for security reasons.)" >&2
diff --git a/contrib/dirauth-tools/nagios-check-tor-authority-cert b/contrib/dirauth-tools/nagios-check-tor-authority-cert
index 46dc7284b7..75ff479a53 100755
--- a/contrib/dirauth-tools/nagios-check-tor-authority-cert
+++ b/contrib/dirauth-tools/nagios-check-tor-authority-cert
@@ -49,12 +49,12 @@ DIRSERVERS="$DIRSERVERS 80.190.246.100:80"      # gabelmoo
 DIRSERVERS="$DIRSERVERS 194.109.206.212:80"     # dizum
 DIRSERVERS="$DIRSERVERS 213.73.91.31:80"        # dannenberg
 
-TMPFILE="`tempfile`"
+TMPFILE=$(mktemp)
 trap 'rm -f "$TMPFILE"' 0
 
 for dirserver in $DIRSERVERS; do
-	wget -q -O "$TMPFILE" "http://$dirserver/tor/keys/fp/$identity"
-	if [ "$?" = 0 ]; then
+	if wget -q -O "$TMPFILE" "http://$dirserver/tor/keys/fp/$identity"
+        then
 		break
 	else
 		cat /dev/null > "$TMPFILE"
@@ -74,10 +74,10 @@ now=$(date +%s)
 if [ "$now" -ge "$expiryunix" ]; then
 	echo "CRITICAL: Certificate expired $expirydate (authority $identity)."
 	exit 2
-elif [ "$(( $now + 7*24*60*60 ))" -ge "$expiryunix" ]; then
+elif [ "$(( now + 7*24*60*60 ))" -ge "$expiryunix" ]; then
 	echo "CRITICAL: Certificate expires $expirydate (authority $identity)."
 	exit 2
-elif [ "$(( $now + 30*24*60*60 ))" -ge "$expiryunix" ]; then
+elif [ "$(( now + 30*24*60*60 ))" -ge "$expiryunix" ]; then
 	echo "WARNING: Certificate expires $expirydate (authority $identity)."
 	exit 1
 else
diff --git a/contrib/dist/suse/tor.sh.in b/contrib/dist/suse/tor.sh.in
deleted file mode 100644
index b7e9005eb5..0000000000
--- a/contrib/dist/suse/tor.sh.in
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/bin/sh
-#
-# Copyright (c) 2006-2007 Andrew Lewman
-#
-# tor    The Onion Router
-#
-# Startup/shutdown script for tor. This is a wrapper around torctl;
-# torctl does the actual work in a relatively system-independent, or at least
-# distribution-independent, way, and this script deals with fitting the
-# whole thing into the conventions of the particular system at hand.
-#
-# These next couple of lines "declare" tor for the "chkconfig" program,
-# originally from SGI, used on Red Hat/Fedora and probably elsewhere.
-#
-# chkconfig: 2345 90 10
-# description: Onion Router - A low-latency anonymous proxy
-#
-
-### BEGIN INIT INFO
-# Provides: tor
-# Required-Start: $remote_fs $network
-# Required-Stop: $remote_fs $network
-# Default-Start: 3 5
-# Default-Stop: 0 1 2 6
-# Short-Description: Start the tor daemon
-# Description:  Start the tor daemon:  the anon-proxy server
-### END INIT INFO
-
-. /etc/rc.status
-
-# Shell functions sourced from /etc/rc.status:
-#      rc_check         check and set local and overall rc status
-#      rc_status        check and set local and overall rc status
-#      rc_status -v     ditto but be verbose in local rc status
-#      rc_status -v -r  ditto and clear the local rc status
-#      rc_failed        set local and overall rc status to failed
-#      rc_reset         clear local rc status (overall remains)
-#      rc_exit          exit appropriate to overall rc status
-
-# First reset status of this service
-rc_reset
-
-# Increase open file descriptors a reasonable amount
-ulimit -n 8192
-
-TORCTL=@BINDIR@/torctl
-
-# torctl will use these environment variables
-TORUSER=@TORUSER@
-export TORUSER
-TORGROUP=@TORGROUP@
-export TORGROUP
-
-TOR_DAEMON_PID_DIR="@LOCALSTATEDIR@/run/tor"
-
-if [ -x /bin/su ] ; then
-    SUPROG=/bin/su
-elif [ -x /sbin/su ] ; then
-    SUPROG=/sbin/su
-elif [ -x /usr/bin/su ] ; then
-    SUPROG=/usr/bin/su
-elif [ -x /usr/sbin/su ] ; then
-    SUPROG=/usr/sbin/su
-else
-    SUPROG=/bin/su
-fi
-
-case "$1" in
-
-    start)
-    echo "Starting tor daemon"
-
-    if [ ! -d $TOR_DAEMON_PID_DIR ] ; then
-        mkdir -p $TOR_DAEMON_PID_DIR
-        chown $TORUSER:$TORGROUP $TOR_DAEMON_PID_DIR
-    fi
-
-    ## Start daemon with startproc(8). If this fails
-    ## the echo return value is set appropriate.
-
-    startproc -f $TORCTL start
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    stop)
-    echo "Stopping tor daemon" 
-    startproc -f $TORCTL stop
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    restart)
-    echo "Restarting tor daemon" 
-    startproc -f $TORCTL restart
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    reload)
-    echo "Reloading tor daemon" 
-    startproc -f $TORCTL reload
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    status)
-    startproc -f $TORCTL status
-    # Remember status and be verbose
-    rc_status -v
-    ;;
-
-    *)
-    echo "Usage: $0 (start|stop|restart|reload|status)"
-    RETVAL=1
-esac
-
-rc_exit
diff --git a/contrib/dist/tor.sh.in b/contrib/dist/tor.sh.in
deleted file mode 100644
index 92f890681f..0000000000
--- a/contrib/dist/tor.sh.in
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/bin/sh
-#
-# tor    The Onion Router
-#
-# Startup/shutdown script for tor. This is a wrapper around torctl;
-# torctl does the actual work in a relatively system-independent, or at least
-# distribution-independent, way, and this script deals with fitting the
-# whole thing into the conventions of the particular system at hand.
-# This particular script is written for Red Hat/Fedora Linux, and may
-# also work on Mandrake, but not SuSE.
-#
-# These next couple of lines "declare" tor for the "chkconfig" program,
-# originally from SGI, used on Red Hat/Fedora and probably elsewhere.
-#
-# chkconfig: 2345 90 10
-# description: Onion Router - A low-latency anonymous proxy
-#
-
-PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
-DAEMON=/usr/sbin/tor
-NAME=tor
-DESC="tor daemon"
-TORPIDDIR=/var/run/tor
-TORPID=$TORPIDDIR/tor.pid
-WAITFORDAEMON=60
-ARGS=""
-
-# Library functions
-if [ -f /etc/rc.d/init.d/functions ]; then
-   . /etc/rc.d/init.d/functions
-elif [ -f /etc/init.d/functions ]; then
-   . /etc/init.d/functions
-fi
-
-TORCTL=@BINDIR@/torctl
-
-# torctl will use these environment variables
-TORUSER=@TORUSER@
-export TORUSER
-
-if [ -x /bin/su ] ; then
-    SUPROG=/bin/su
-elif [ -x /sbin/su ] ; then
-    SUPROG=/sbin/su
-elif [ -x /usr/bin/su ] ; then
-    SUPROG=/usr/bin/su
-elif [ -x /usr/sbin/su ] ; then
-    SUPROG=/usr/sbin/su
-else
-    SUPROG=/bin/su
-fi
-
-# Raise ulimit based on number of file descriptors available (thanks, Debian)
-
-if [ -r /proc/sys/fs/file-max ]; then
-	system_max=`cat /proc/sys/fs/file-max`
-	if [ "$system_max" -gt "80000" ] ; then
-		MAX_FILEDESCRIPTORS=32768
-	elif [ "$system_max" -gt "40000" ] ; then
-		MAX_FILEDESCRIPTORS=16384
-	elif [ "$system_max" -gt "10000" ] ; then
-		MAX_FILEDESCRIPTORS=8192
-	else
-		MAX_FILEDESCRIPTORS=1024
-		cat << EOF
-
-Warning: Your system has very few filedescriptors available in total.
-
-Maybe you should try raising that by adding 'fs.file-max=100000' to your
-/etc/sysctl.conf file.  Feel free to pick any number that you deem appropriate.
-Then run 'sysctl -p'.  See /proc/sys/fs/file-max for the current value, and
-file-nr in the same directory for how many of those are used at the moment.
-
-EOF
-	fi
-else
-	MAX_FILEDESCRIPTORS=8192
-fi
-
-NICE=""
-
-case "$1" in
-
-    start)
-	if [ -n "$MAX_FILEDESCRIPTORS" ]; then
-		echo -n "Raising maximum number of filedescriptors (ulimit -n) to $MAX_FILEDESCRIPTORS"
-		if ulimit -n "$MAX_FILEDESCRIPTORS" ; then
-			echo "."
-		else
-			echo ": FAILED."
-		fi
-	fi
-
-    action $"Starting tor:" $TORCTL start
-    RETVAL=$?
-    ;;
-
-    stop)
-    action $"Stopping tor:" $TORCTL stop
-    RETVAL=$?
-    ;;
-
-    restart)
-    action $"Restarting tor:" $TORCTL restart
-    RETVAL=$?
-    ;;
-
-    reload)
-    action $"Reloading tor:" $TORCTL reload
-    RETVAL=$?
-    ;;
-
-    status)
-    $TORCTL status
-    RETVAL=$?
-    ;;
-
-    *)
-    echo "Usage: $0 (start|stop|restart|reload|status)"
-    RETVAL=1
-esac
-
-exit $RETVAL
diff --git a/contrib/dist/torctl.in b/contrib/dist/torctl.in
deleted file mode 100644
index 4cc137da46..0000000000
--- a/contrib/dist/torctl.in
+++ /dev/null
@@ -1,195 +0,0 @@
-#!/bin/sh
-#
-# TOR control script designed to allow an easy command line interface
-# to controlling The Onion Router
-#
-# The exit codes returned are:
-#       0 - operation completed successfully. For "status", tor running.
-#       1 - For "status", tor not running.
-#       2 - Command not supported
-#       3 - Could not be started or reloaded
-#       4 - Could not be stopped
-#       5 -
-#       6 -
-#       7 -
-#       8 -
-#
-# When multiple arguments are given, only the error from the _last_
-# one is reported.
-#
-#
-# |||||||||||||||||||| START CONFIGURATION SECTION  ||||||||||||||||||||
-# --------------------                              --------------------
-# Name of the executable
-EXEC=tor
-#
-# the path to your binary, including options if necessary
-TORBIN="@BINDIR@/$EXEC"
-#
-# the path to the configuration file
-TORCONF="@CONFDIR@/torrc"
-#
-# the path to your PID file
-PIDFILE="@LOCALSTATEDIR@/run/tor/tor.pid"
-#
-# The path to the log file
-LOGFILE="@LOCALSTATEDIR@/log/tor/tor.log"
-#
-# The path to the datadirectory
-TORDATA="@LOCALSTATEDIR@/lib/tor"
-#
-TORARGS="--pidfile $PIDFILE --log \"notice file $LOGFILE\" --runasdaemon 1"
-TORARGS="$TORARGS --datadirectory $TORDATA"
-
-# If user name is set in the environment, then use it;
-# otherwise run as the invoking user (or whatever user the config
-# file says)... unless the invoking user is root. The idea here is to
-# let an unprivileged user run tor for her own use using this script,
-# while still providing for it to be used as a system daemon.
-if [ "x`id -u`" = "x0" ]; then
-    TORUSER=@TORUSER@
-fi
-
-if [ "x$TORUSER" != "x" ]; then
-    TORARGS="$TORARGS --user $TORUSER"
-fi
-
-# We no longer wrap the Tor daemon startup in an su when running as
-# root, because it's too painful to make the use of su portable.
-# Just let the daemon set the UID and GID.
-START="$TORBIN -f $TORCONF $TORARGS"
-
-#
-# --------------------                              --------------------
-# ||||||||||||||||||||   END CONFIGURATION SECTION  ||||||||||||||||||||
-
-ERROR=0
-ARGV="$@"
-if [ "x$ARGV" = "x" ] ; then
-    ARGS="help"
-fi
-
-checkIfRunning ( ) {
-    # check for pidfile
-    PID=unknown
-    if [ -f $PIDFILE ] ; then
-        PID=`/bin/cat $PIDFILE`
-        if [ "x$PID" != "x" ] ; then
-            if kill -0 $PID 2>/dev/null ; then
-               STATUS="$EXEC (pid $PID) running"
-               RUNNING=1
-            else
-                STATUS="PID file ($PIDFILE) present, but $EXEC ($PID) not running"
-                RUNNING=0
-            fi
-        else
-            STATUS="$EXEC (pid $PID?) not running"
-            RUNNING=0
-        fi
-    else
-       STATUS="$EXEC apparently not running (no pid file)"
-       RUNNING=0
-    fi
-    return
-}
-
-for ARG in $@ $ARGS
-do
-    checkIfRunning
-
-    case $ARG in
-    start)
-        if [ $RUNNING -eq 1 ]; then
-            echo "$0 $ARG: $EXEC (pid $PID) already running"
-            continue
-        fi
-        if eval "$START" ; then
-            echo "$0 $ARG: $EXEC started"
-            # Make sure it stayed up!
-            /bin/sleep 1
-            checkIfRunning
-            if [ $RUNNING -eq 0 ]; then
-                echo "$0 $ARG: $EXEC (pid $PID) quit unexpectedly"
-            fi
-        else
-            echo "$0 $ARG: $EXEC could not be started"
-            ERROR=3
-        fi
-        ;;
-    stop)
-        if [ $RUNNING -eq 0 ]; then
-            echo "$0 $ARG: $STATUS"
-            continue
-        fi
-        if kill -15 $PID ; then
-            echo "$0 $ARG: $EXEC stopped"
-        else
-            /bin/sleep 1
-            if kill -9 $PID ; then
-                echo "$0 $ARG: $EXEC stopped"
-            else
-                echo "$0 $ARG: $EXEC could not be stopped"
-                ERROR=4
-            fi
-        fi
-        # Make sure it really died!
-        /bin/sleep 1
-        checkIfRunning
-        if [ $RUNNING -eq 1 ]; then
-            echo "$0 $ARG: $EXEC (pid $PID) unexpectedly still running"
-            ERROR=4
-        fi
-        ;;
-    restart)
-        $0 stop start
-        ;;
-    reload)
-        if [ $RUNNING -eq 0 ]; then
-            echo "$0 $ARG: $STATUS"
-            continue
-        fi
-        if kill -1 $PID; then
-            /bin/sleep 1
-            echo "$EXEC (PID $PID) reloaded"
-        else
-            echo "Can't reload $EXEC"
-            ERROR=3
-        fi
-        ;;
-    status)
-        echo $STATUS
-        if [ $RUNNING -eq 1 ]; then
-            ERROR=0
-        else
-            ERROR=1
-        fi
-        ;;
-    log)
-        cat $LOGFILE
-        ;;
-    help)
-        echo "usage: $0 (start|stop|restart|status|help)"
-        /bin/cat <<EOF
-
-start      - start $EXEC
-stop       - stop $EXEC
-restart    - stop and restart $EXEC if running or start if not running
-reload     - cause the running process to reinitialize itself
-status     - tell whether $EXEC is running or not
-log        - display the contents of the log file
-help       - this text
-
-EOF
-        ERROR=0
-        ;;
-    *)
-        $0 help
-        ERROR=2
-        ;;
-
-    esac
-
-done
-
-exit $ERROR
-
diff --git a/contrib/include.am b/contrib/include.am
index a23e82d6da..784f5427b8 100644
--- a/contrib/include.am
+++ b/contrib/include.am
@@ -3,11 +3,7 @@ EXTRA_DIST+= \
 	contrib/README					\
 	contrib/client-tools/torify			\
 	contrib/dist/rc.subr				\
-	contrib/dist/suse/tor.sh.in			\
-	contrib/dist/tor.sh				\
-	contrib/dist/torctl				\
 	contrib/dist/tor.service.in			\
-	contrib/operator-tools/linux-tor-prio.sh	\
 	contrib/operator-tools/tor-exit-notice.html	\
 	contrib/or-tools/exitlist			\
 	contrib/win32build/tor-mingw.nsi.in		\
diff --git a/contrib/operator-tools/linux-tor-prio.sh b/contrib/operator-tools/linux-tor-prio.sh
deleted file mode 100644
index 30ea5fc659..0000000000
--- a/contrib/operator-tools/linux-tor-prio.sh
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/bin/bash
-# Written by Marco Bonetti & Mike Perry
-# Based on instructions from Dan Singletary's ADSL BW Management HOWTO:
-# http://www.faqs.org/docs/Linux-HOWTO/ADSL-Bandwidth-Management-HOWTO.html
-# This script is Public Domain.
-
-############################### README #################################
-
-# This script provides prioritization of Tor traffic below other
-# traffic on a Linux server. It has two modes of operation: UID based 
-# and IP based. 
-
-# UID BASED PRIORITIZATION
-#
-# The UID based method requires that Tor be launched from 
-# a specific user ID. The "User" Tor config setting is
-# insufficient, as it sets the UID after the socket is created.
-# Here is a C wrapper you can use to execute Tor and drop privs before 
-# it creates any sockets. 
-#
-# Compile with:
-# gcc -DUID=`id -u tor` -DGID=`id -g tor` tor_wrap.c -o tor_wrap
-#
-# #include <unistd.h>
-# int main(int argc, char **argv) {
-# if(initgroups("tor", GID) == -1) { perror("initgroups"); return 1; }
-# if(setresgid(GID, GID, GID) == -1) { perror("setresgid"); return 1; }
-# if(setresuid(UID, UID, UID) == -1) { perror("setresuid"); return 1; }
-# execl("/bin/tor", "/bin/tor", "-f", "/etc/tor/torrc", NULL);
-# perror("execl"); return 1;
-# }
-
-# IP BASED PRIORITIZATION
-#
-# The IP setting requires that a separate IP address be dedicated to Tor. 
-# Your Torrc should be set to bind to this IP for "OutboundBindAddress", 
-# "ListenAddress", and "Address".
-
-# GENERAL USAGE
-#
-# You should also tune the individual connection rate parameters below
-# to your individual connection. In particular, you should leave *some* 
-# minimum amount of bandwidth for Tor, so that Tor users are not 
-# completely choked out when you use your server's bandwidth. 30% is 
-# probably a reasonable choice. More is better of course.
-#
-# To start the shaping, run it as: 
-#   ./linux-tor-prio.sh 
-#
-# To get status information (useful to verify packets are getting marked
-# and prioritized), run:
-#   ./linux-tor-prio.sh status
-#
-# And to stop prioritization:
-#   ./linux-tor-prio.sh stop
-#
-########################################################################
-
-# BEGIN USER TUNABLE PARAMETERS
-
-DEV=eth0
-
-# NOTE! You must START Tor under this UID. Using the Tor User
-# config setting is NOT sufficient. See above.
-TOR_UID=$(id -u tor)
-
-# If the UID mechanism doesn't work for you, you can set this parameter
-# instead. If set, it will take precedence over the UID setting. Note that
-# you need multiple IPs with one specifically devoted to Tor for this to
-# work.
-#TOR_IP="42.42.42.42"
-
-# Average ping to most places on the net, milliseconds
-RTT_LATENCY=40
-
-# RATE_UP must be less than your connection's upload capacity in
-# kbits/sec. If it is larger, then the bottleneck will be at your
-# router's queue, which you do not control. This will cause congestion
-# and a revert to normal TCP fairness no matter what the queing
-# priority is.
-RATE_UP=5000
-
-# RATE_UP_TOR is the minimum speed your Tor connections will have in
-# kbits/sec.  They will have at least this much bandwidth for upload.
-# In general, you probably shouldn't set this too low, or else Tor
-# users who use your node will be completely choked out whenever your
-# machine does any other network activity. That is not very fun.
-RATE_UP_TOR=1500
-
-# RATE_UP_TOR_CEIL is the maximum rate allowed for all Tor traffic in
-# kbits/sec.
-RATE_UP_TOR_CEIL=5000
-
-CHAIN=OUTPUT
-#CHAIN=PREROUTING
-#CHAIN=POSTROUTING
-
-MTU=1500
-AVG_PKT=900 # should be more like 600 for non-exit nodes
-
-# END USER TUNABLE PARAMETERS
-
-
-
-# The queue size should be no larger than your bandwidth-delay
-# product. This is RT latency*bandwidth/MTU/2
-
-BDP=$(expr $RTT_LATENCY \* $RATE_UP / $AVG_PKT)
-
-# Further research indicates that the BDP calculations should use
-# RTT/sqrt(n) where n is the expected number of active connections..
-
-BDP=$(expr $BDP / 4)
-
-if [ "$1" = "status" ]
-then
-	echo "[qdisc]"
-	tc -s qdisc show dev $DEV
-	tc -s qdisc show dev imq0
-	echo "[class]"
-	tc -s class show dev $DEV
-	tc -s class show dev imq0
-	echo "[filter]"
-	tc -s filter show dev $DEV
-	tc -s filter show dev imq0
-	echo "[iptables]"
-	iptables -t mangle -L TORSHAPER-OUT -v -x 2> /dev/null
-	exit
-fi
-
-
-# Reset everything to a known state (cleared)
-tc qdisc del dev $DEV root 2> /dev/null > /dev/null
-tc qdisc del dev imq0 root 2> /dev/null > /dev/null
-iptables -t mangle -D POSTROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -D PREROUTING -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -D OUTPUT -o $DEV -j TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -F TORSHAPER-OUT 2> /dev/null > /dev/null
-iptables -t mangle -X TORSHAPER-OUT 2> /dev/null > /dev/null
-ip link set imq0 down 2> /dev/null > /dev/null
-rmmod imq 2> /dev/null > /dev/null
-
-if [ "$1" = "stop" ]
-then
-	echo "Shaping removed on $DEV."
-	exit
-fi
-
-# Outbound Shaping (limits total bandwidth to RATE_UP)
-
-ip link set dev $DEV qlen $BDP
-
-# Add HTB root qdisc, default is high prio
-tc qdisc add dev $DEV root handle 1: htb default 20
-
-# Add main rate limit class
-tc class add dev $DEV parent 1: classid 1:1 htb rate ${RATE_UP}kbit
-
-# Create the two classes, giving Tor at least RATE_UP_TOR kbit and capping
-# total upstream at RATE_UP so the queue is under our control.
-tc class add dev $DEV parent 1:1 classid 1:20 htb rate $(expr $RATE_UP - $RATE_UP_TOR)kbit ceil ${RATE_UP}kbit prio 0
-tc class add dev $DEV parent 1:1 classid 1:21 htb rate $[$RATE_UP_TOR]kbit ceil ${RATE_UP_TOR_CEIL}kbit prio 10
-
-# Start up pfifo
-tc qdisc add dev $DEV parent 1:20 handle 20: pfifo limit $BDP
-tc qdisc add dev $DEV parent 1:21 handle 21: pfifo limit $BDP
-
-# filter traffic into classes by fwmark
-tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 20 fw flowid 1:20
-tc filter add dev $DEV parent 1:0 prio 0 protocol ip handle 21 fw flowid 1:21
-
-# add TORSHAPER-OUT chain to the mangle table in iptables
-iptables -t mangle -N TORSHAPER-OUT
-iptables -t mangle -I $CHAIN -o $DEV -j TORSHAPER-OUT
-
-
-# Set firewall marks
-# Low priority to Tor
-if [ ""$TOR_IP == "" ]
-then
-	echo "Using UID-based QoS. UID $TOR_UID marked as low priority."
-	iptables -t mangle -A TORSHAPER-OUT -m owner --uid-owner $TOR_UID -j MARK --set-mark 21
-else
-	echo "Using IP-based QoS. $TOR_IP marked as low priority."
-	iptables -t mangle -A TORSHAPER-OUT -s $TOR_IP -j MARK --set-mark 21
-fi
-
-# High prio for everything else
-iptables -t mangle -A TORSHAPER-OUT -m mark --mark 0 -j MARK --set-mark 20
-
-echo "Outbound shaping added to $DEV.  Rate for Tor upload at least: ${RATE_UP_TOR}Kbyte/sec."
-
diff --git a/contrib/or-tools/check-tor b/contrib/or-tools/check-tor
deleted file mode 100755
index e981a35fcc..0000000000
--- a/contrib/or-tools/check-tor
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/sh
-
-## Originally written by Peter Palfrader.
-
-## This script lets you quickly check if a given router (by nickname)
-## will let you do a TLS handshake, or will let you download a directory.
-
-## Usage: check-tor nickname
-
-#set -x
-
-router="$1"
-dirserver="http://belegost.seul.org:80/tor/"
-
-lines=$( wget -q $dirserver --proxy=off -O - | grep -A5 '^router '"$router"' ' )
-line=$( echo "$lines" | head -n1 )
-
-if [ -z "$line" ]; then
-	echo "Not found" >&2
-	exit 1
-fi
-
-echo "$lines"
-echo
-
-ipor=$( echo "$line" | awk '{printf "%s:%s", $3, $4}' )
-
-op=$( echo "$line" | awk '{printf $6}' )
-ipop=$( echo "$line" | awk '{printf "%s:%s", $3, $6}' )
-
-echo
-echo ">>" openssl s_client -connect "$ipor"
-timeout 5 openssl s_client -connect "$ipor" < /dev/null
-if [ "$op" != "0" ]; then
-	echo
-	echo ">>" wget --proxy=off -O - http://$ipop/tor/
-	timeout 5 wget --proxy=off -O - http://$ipop/tor/ | head -n3
-fi
-
-echo
-echo -n "$router "; echo "$lines" | grep 'fingerprint' | sed -e 's/^opt //' -e 's/^fingerprint //';
diff --git a/contrib/win32build/tor-mingw.nsi.in b/contrib/win32build/tor-mingw.nsi.in
index 638754153b..ababaa38fc 100644
--- a/contrib/win32build/tor-mingw.nsi.in
+++ b/contrib/win32build/tor-mingw.nsi.in
@@ -8,7 +8,7 @@
 !include "LogicLib.nsh"
 !include "FileFunc.nsh"
 !insertmacro GetParameters
-!define VERSION "0.3.5.11-dev"
+!define VERSION "0.4.2.8-dev"
 !define INSTALLER "tor-${VERSION}-win32.exe"
 !define WEBSITE "https://www.torproject.org/"
 !define LICENSE "LICENSE"
diff --git a/doc/HACKING/CodeStructure.md b/doc/HACKING/CodeStructure.md
index 736d6cd484..fffafcaed1 100644
--- a/doc/HACKING/CodeStructure.md
+++ b/doc/HACKING/CodeStructure.md
@@ -2,128 +2,121 @@
 TODO: revise this to talk about how things are, rather than how things
 have changed.
 
-TODO: Make this into good markdown.
-
-
-
-For quite a while now, the program "tor" has been built from source
-code in just two directories: src/common and src/or.
+For quite a while now, the program *tor* has been built from source
+code in just two directories: **src/common** and **src/or**.
 
 This has become more-or-less untenable, for a few reasons -- most
 notably of which is that it has led our code to become more
 spaghetti-ish than I can endorse with a clean conscience.
 
 So to fix that, we've gone and done a huge code movement in our git
-master branch, which will land in a release once Tor 0.3.5.1-alpha is
+master branch, which will land in a release once Tor `0.3.5.1-alpha` is
 out.
 
 Here's what we did:
 
-  * src/common has been turned into a set of static libraries.  These
-all live in the "src/lib/*" directories.  The dependencies between
+  * **src/common** has been turned into a set of static libraries.  These
+all live in the **src/lib/*** directories.  The dependencies between
 these libraries should have no cycles.  The libraries are:
 
-    arch -- Headers to handle architectural differences
-    cc -- headers to handle differences among compilers
-    compress -- wraps zlib, zstd, lzma
-    container -- high-level container types
-    crypt_ops -- Cryptographic operations. Planning to split this into
+    - **arch** -- Headers to handle architectural differences
+    - **cc** -- headers to handle differences among compilers
+    - **compress** -- wraps zlib, zstd, lzma
+    - **container** -- high-level container types
+    - **crypt_ops** -- Cryptographic operations. Planning to split this into
 a higher and lower level library
-    ctime -- Operations that need to run in constant-time. (Properly,
+    - **ctime** -- Operations that need to run in constant-time. (Properly,
 data-invariant time)
-    defs -- miscelaneous definitions needed throughout Tor.
-    encoding -- transforming one data type into another, and various
+    - **defs** -- miscelaneous definitions needed throughout Tor.
+    - **encoding** -- transforming one data type into another, and various
 data types into strings.
-    err -- lowest-level error handling, in cases where we can't use
+    - **err** -- lowest-level error handling, in cases where we can't use
 the logs because something that the logging system needs has broken.
-    evloop -- Generic event-loop handling logic
-    fdio -- Low-level IO wrapper functions for file descriptors.
-    fs -- Operations on the filesystem
-    intmath -- low-level integer math and misc bit-twiddling hacks
-    lock -- low-level locking code
-    log -- Tor's logging module.  This library sits roughly halfway up
+    - **evloop** -- Generic event-loop handling logic
+    - **fdio** -- Low-level IO wrapper functions for file descriptors.
+    - **fs** -- Operations on the filesystem
+    - **intmath** -- low-level integer math and misc bit-twiddling hacks
+    - **lock** -- low-level locking code
+    - **log** -- Tor's logging module.  This library sits roughly halfway up
 the library dependency diagram, since everything it depends on has to
 be carefully crafted to *not* log.
-    malloc -- Low-level wrappers for the platform memory allocation functions.
-    math -- Higher-level mathematical functions, and floating-point math
-    memarea -- An arena allocator
-    meminfo -- Functions for querying the current process's memory
+    - **malloc** -- Low-level wrappers for the platform memory allocation functions.
+    - **math** -- Higher-level mathematical functions, and floating-point math
+    - **memarea** -- An arena allocator
+    - **meminfo** -- Functions for querying the current process's memory
 status and resources
-    net -- Networking compatibility and convenience code
-    osinfo -- Querying information about the operating system
-    process -- Launching and querying the status of other processes
-    sandbox -- Backend for the linux seccomp2 sandbox
-    smartlist_core -- The lowest-level of the smartlist_t data type.
+    - **net** -- Networking compatibility and convenience code
+    - **osinfo** -- Querying information about the operating system
+    - **process** -- Launching and querying the status of other processes
+    - **sandbox** -- Backend for the linux seccomp2 sandbox
+    - **smartlist_core** -- The lowest-level of the smartlist_t data type.
 Separated from the rest of the containers library because the logging
 subsystem depends on it.
-    string -- Compatibility and convenience functions for manipulating
+    - **string** -- Compatibility and convenience functions for manipulating
 C strings.
-    term -- Terminal-related functions (currently limited to a getpass
+    - **term** -- Terminal-related functions (currently limited to a getpass
 function).
-    testsupport -- Macros for mocking, unit tests, etc.
-    thread -- Higher-level thread compatibility code
-    time -- Higher-level time management code, including format
+    - **testsupport** -- Macros for mocking, unit tests, etc.
+    - **thread** -- Higher-level thread compatibility code
+    - **time** -- Higher-level time management code, including format
 conversions and monotonic time
-    tls -- Our wrapper around our TLS library
-    trace -- Formerly src/trace -- a generic event tracing API
-    wallclock -- Low-level time code, used by the log module.
+    - **tls** -- Our wrapper around our TLS library
+    - **trace** -- Formerly src/trace -- a generic event tracing API
+    - **wallclock** -- Low-level time code, used by the log module.
 
-  * To ensure that the dependency graph in src/common remains under
-control, there is a tool that you can run called "make
-check-includes".  It verifies that each module in Tor only includes
+  * To ensure that the dependency graph in **src/common** remains under
+control, there is a tool that you can run called `make
+check-includes`.  It verifies that each module in Tor only includes
 the headers that it is permitted to include, using a per-directory
-".may_include" file.
+*.may_include* file.
 
-  * The src/or/or.h header has been split into numerous smaller
+  * The **src/or/or.h** header has been split into numerous smaller
 headers.  Notably, many important structures are now declared in a
-header called foo_st.h, where "foo" is the name of the structure.
+header called *foo_st.h*, where "foo" is the name of the structure.
 
-  * The src/or directory, which had most of Tor's code, had been split
+  * The **src/or** directory, which had most of Tor's code, had been split
 up into several directories.  This is still a work in progress:  This
 code has not itself been refactored, and its dependency graph is still
 a tangled web.  I hope we'll be working on that over the coming
 releases, but it will take a while to do.
 
-    The new top-level source directories are:
-
-     src/core -- Code necessary to actually perform or use onion routing.
-     src/feature -- Code used only by some onion routing
+    - The new top-level source directories are:
+        - **src/core** -- Code necessary to actually perform or use onion routing.
+        - **src/feature** -- Code used only by some onion routing
 configurations, or only for a special purpose.
-     src/app -- Top-level code to run, invoke, and configure the
+        - **src/app** -- Top-level code to run, invoke, and configure the
 lower-level code
 
-   The new second-level source directories are:
-     src/core/crypto -- High-level cryptographic protocols used in Tor
-     src/core/mainloop -- Tor's event loop, connection-handling, and
+    - The new second-level source directories are:
+        - **src/core/crypto** -- High-level cryptographic protocols used in Tor
+        - **src/core/mainloop** -- Tor's event loop, connection-handling, and
 traffic-routing code.
-     src/core/or -- Parts related to handling onion routing itself
-     src/core/proto -- support for encoding and decoding different
+        - **src/core/or** -- Parts related to handling onion routing itself
+        - **src/core/proto** -- support for encoding and decoding different
 wire protocols
-
-     src/feature/api -- Support for making Tor embeddable
-     src/feature/client -- Functionality which only Tor clients need
-     src/feature/control -- Controller implementation
-     src/feature/dirauth -- Directory authority
-     src/feature/dircache -- Directory cache
-     src/feature/dirclient -- Directory client
-     src/feature/dircommon -- Shared code between the other directory modules
-     src/feature/hibernate -- Hibernating when Tor is out of bandwidth
+        - **src/feature/api** -- Support for making Tor embeddable
+        - **src/feature/client** -- Functionality which only Tor clients need
+        - **src/feature/control** -- Controller implementation
+        - **src/feature/dirauth** -- Directory authority
+        - **src/feature/dircache** -- Directory cache
+        - **src/feature/dirclient** -- Directory client
+        - **src/feature/dircommon** -- Shared code between the other directory modules
+        - **src/feature/hibernate** -- Hibernating when Tor is out of bandwidth
 or shutting down
-     src/feature/hs -- v3 onion service implementation
-     src/feature/hs_common -- shared code between both onion service
+        - **src/feature/hs** -- v3 onion service implementation
+        - **src/feature/hs_common** -- shared code between both onion service
 implementations
-     src/feature/nodelist -- storing and accessing the list of relays on
+        - **src/feature/nodelist** -- storing and accessing the list of relays on
 the network.
-     src/feature/relay -- code that only relay servers and exit servers need.
-     src/feature/rend -- v2 onion service implementation
-     src/feature/stats -- statistics and history
-
-     src/app/config -- configuration and state for Tor
-     src/app/main -- Top-level functions to invoke the rest or Tor.
+        - **src/feature/relay** -- code that only relay servers and exit servers need.
+        - **src/feature/rend** -- v2 onion service implementation
+        - **src/feature/stats** -- statistics and history
+        - **src/app/config** -- configuration and state for Tor
+        - **src/app/main** -- Top-level functions to invoke the rest or Tor.
 
-  * The "tor" executable is now built in src/app/tor rather than src/or/tor.
+  * The `tor` executable is now built in **src/app/tor** rather than **src/or/tor**.
 
   * There are more static libraries than before that you need to build
 into your application if you want to embed Tor.  Rather than
-maintaining this list yourself, I recommend that you run "make
-show-libs" to have Tor emit a list of what you need to link.
+maintaining this list yourself, I recommend that you run `make
+show-libs` to have Tor emit a list of what you need to link.
diff --git a/doc/HACKING/CodingStandards.md b/doc/HACKING/CodingStandards.md
index 4f229348e4..2c273910d1 100644
--- a/doc/HACKING/CodingStandards.md
+++ b/doc/HACKING/CodingStandards.md
@@ -42,6 +42,7 @@ If you have changed build system components:
    - For example, if you have changed Makefiles, autoconf files, or anything
      else that affects the build system.
 
+
 License issues
 ==============
 
@@ -58,7 +59,6 @@ Some compatible licenses include:
   - CC0 Public Domain Dedication
 
 
-
 How we use Git branches
 =======================
 
@@ -99,29 +99,65 @@ When you do a commit that needs a ChangeLog entry, add a new file to
 the `changes` toplevel subdirectory.  It should have the format of a
 one-entry changelog section from the current ChangeLog file, as in
 
-- Major bugfixes:
+  o Major bugfixes (security):
     - Fix a potential buffer overflow. Fixes bug 99999; bugfix on
       0.3.1.4-beta.
+  o Minor features (performance):
+    - Make tor faster. Closes ticket 88888.
 
 To write a changes file, first categorize the change.  Some common categories
-are: Minor bugfixes, Major bugfixes, Minor features, Major features, Code
-simplifications and refactoring.  Then say what the change does.  If
-it's a bugfix, mention what bug it fixes and when the bug was
-introduced.  To find out which Git tag the change was introduced in,
-you can use `git describe --contains <sha1 of commit>`.
-
-If at all possible, try to create this file in the same commit where you are
-making the change.  Please give it a distinctive name that no other branch will
-use for the lifetime of your change. To verify the format of the changes file,
-you can use `make check-changes`.  This is run automatically as part of
-`make check` -- if it fails, we must fix it before we release.  These
-checks are implemented in `scripts/maint/lintChanges.py`.
+are:
+  o Minor bugfixes (subheading):
+  o Major bugfixes (subheading):
+  o Minor features (subheading):
+  o Major features (subheading):
+  o Code simplifications and refactoring:
+  o Testing:
+  o Documentation:
+
+The subheading is a particular area within Tor.  See the ChangeLog for
+examples.
+
+Then say what the change does.  If it's a bugfix, mention what bug it fixes
+and when the bug was introduced. To find out which Git tag the change was
+introduced in, you can use `git describe --contains <sha1 of commit>`.
+If you don't know the commit, you can search the git diffs (-S) for the first
+instance of the feature (--reverse).
+
+For example, for #30224, we wanted to know when the bridge-distribution-request
+feature was introduced into Tor:
+    $ git log -S bridge-distribution-request --reverse
+    commit ebab521525
+    Author: Roger Dingledine <arma@torproject.org>
+    Date:   Sun Nov 13 02:39:16 2016 -0500
+
+        Add new BridgeDistribution config option
+
+    $ git describe --contains ebab521525
+    tor-0.3.2.3-alpha~15^2~4
+
+If you need to know all the Tor versions that contain a commit, use:
+    $ git tag --contains 9f2efd02a1 | sort -V
+    tor-0.2.5.16
+    tor-0.2.8.17
+    tor-0.2.9.14
+    tor-0.2.9.15
+    ...
+    tor-0.3.0.13
+    tor-0.3.1.9
+    tor-0.3.1.10
+    ...
+
+If at all possible, try to create the changes file in the same commit where
+you are making the change.  Please give it a distinctive name that no other
+branch will use for the lifetime of your change. We usually use "ticketNNNNN"
+or "bugNNNNN", where NNNNN is the ticket number. To verify the format of the
+changes file, you can use `make check-changes`.  This is run automatically as
+part of `make check` -- if it fails, we must fix it as soon as possible, so
+that our CI passes.  These checks are implemented in
+`scripts/maint/lintChanges.py`.
 
 Changes file style guide:
-  * Changes files begin with "  o Header (subheading):".  The header
-    should usually be "Minor/Major bugfixes/features". The subheading is a
-    particular area within Tor.  See the ChangeLog for examples.
-
   * Make everything terse.
 
   * Write from the user's point of view: describe the user-visible changes
@@ -314,7 +350,6 @@ for more information about trunnel.
 
 For information on adding new trunnel code to Tor, see src/trunnel/README
 
-
 Calling and naming conventions
 ------------------------------
 
@@ -422,7 +457,6 @@ to use it as a function callback), define it with a name like
       abc_free_(obj);
     }
 
-
 Doxygen comment conventions
 ---------------------------
 
diff --git a/doc/HACKING/CodingStandardsRust.md b/doc/HACKING/CodingStandardsRust.md
index fc562816db..b570e10dc7 100644
--- a/doc/HACKING/CodingStandardsRust.md
+++ b/doc/HACKING/CodingStandardsRust.md
@@ -256,7 +256,7 @@ Here are some additional bits of advice and rules:
    or 2) should fail (i.e. in a unittest).
 
    You SHOULD NOT use `unwrap()` anywhere in which it is possible to handle the
-   potential error with either `expect()` or the eel operator, `?`.
+   potential error with the eel operator, `?` or another non panicking way.
    For example, consider a function which parses a string into an integer:
 
         fn parse_port_number(config_string: &str) -> u16 {
@@ -264,12 +264,12 @@ Here are some additional bits of advice and rules:
         }
 
    There are numerous ways this can fail, and the `unwrap()` will cause the
-   whole program to byte the dust!  Instead, either you SHOULD use `expect()`
+   whole program to byte the dust!  Instead, either you SHOULD use `ok()`
    (or another equivalent function which will return an `Option` or a `Result`)
    and change the return type to be compatible:
 
         fn parse_port_number(config_string: &str) -> Option<u16> {
-            u16::from_str_radix(config_string, 10).expect("Couldn't parse port into a u16")
+            u16::from_str_radix(config_string, 10).ok()
         }
 
    or you SHOULD use `or()` (or another similar method):
diff --git a/doc/HACKING/EndOfLifeTor.md b/doc/HACKING/EndOfLifeTor.md
new file mode 100644
index 0000000000..2fece2ca9d
--- /dev/null
+++ b/doc/HACKING/EndOfLifeTor.md
@@ -0,0 +1,50 @@
+
+End of Life on an old release series
+------------------------------------
+
+Here are the steps that the maintainer should take when an old Tor release
+series reaches End of Life.  Note that they are _only_ for entire series that
+have reached their planned EOL: they do not apply to security-related
+deprecations of individual versions.
+
+### 0. Preliminaries
+
+0. A few months before End of Life:
+   Write a deprecation announcement.
+   Send the announcement out with every new release announcement.
+
+1. A month before End of Life:
+   Send the announcement to tor-announce, tor-talk, tor-relays, and the
+   packagers.
+
+### 1. On the day
+
+1. Open tickets to remove the release from:
+   - the jenkins builds
+   - tor's Travis CI cron jobs
+   - chutney's Travis CI tests (#)
+   - stem's Travis CI tests (#)
+
+2. Close the milestone in Trac. To do this, go to Trac, log in,
+   select "Admin" near the top of the screen, then select "Milestones" from
+   the menu on the left.  Click on the milestone for this version, and
+   select the "Completed" checkbox. By convention, we select the date as
+   the End of Life date.
+
+3. Replace NNN-backport with NNN-unreached-backport in all open trac tickets.
+
+4. If there are any remaining tickets in the milestone:
+     - merge_ready tickets are for backports:
+       - if there are no supported releases for the backport, close the ticket
+       - if there is an earlier (LTS) release for the backport, move the ticket
+         to that release
+     - other tickets should be closed (if we won't fix them) or moved to a
+       supported release (if we will fix them)
+
+5. Mail the end of life announcement to tor-announce, the packagers list,
+   and tor-relays. The current list of packagers is in ReleasingTor.md.
+
+6. Ask at least two of weasel/arma/Sebastian to remove the old version
+   number from their approved versions list.
+
+7. Update the CoreTorReleases wiki page.
diff --git a/doc/HACKING/Fuzzing.md b/doc/HACKING/Fuzzing.md
index 2039d6a4c0..c2db7e9853 100644
--- a/doc/HACKING/Fuzzing.md
+++ b/doc/HACKING/Fuzzing.md
@@ -1,6 +1,6 @@
-= Fuzzing Tor
+# Fuzzing Tor
 
-== The simple version (no fuzzing, only tests)
+## The simple version (no fuzzing, only tests)
 
 Check out fuzzing-corpora, and set TOR_FUZZ_CORPORA to point to the place
 where you checked it out.
@@ -12,7 +12,7 @@ This won't actually fuzz Tor!  It will just run all the fuzz binaries
 on our existing set of testcases for the fuzzer.
 
 
-== Different kinds of fuzzing
+## Different kinds of fuzzing
 
 Right now we support three different kinds of fuzzer.
 
@@ -37,7 +37,7 @@ In all cases, you'll need some starting examples to give the fuzzer when it
 starts out.  There's a set in the "fuzzing-corpora" git repository.  Try
 setting TOR_FUZZ_CORPORA to point to a checkout of that repository
 
-== Writing Tor fuzzers
+## Writing Tor fuzzers
 
 A tor fuzzing harness should have:
 * a fuzz_init() function to set up any necessary global state.
@@ -52,7 +52,7 @@ bug, or accesses memory it shouldn't. This helps fuzzing frameworks detect
 "interesting" cases.
 
 
-== Guided Fuzzing with AFL
+## Guided Fuzzing with AFL
 
 There is no HTTPS, hash, or signature for American Fuzzy Lop's source code, so
 its integrity can't be verified. That said, you really shouldn't fuzz on a
@@ -101,7 +101,7 @@ macOS (OS X) requires slightly more preparation, including:
 * using afl-clang (or afl-clang-fast from the llvm directory)
 * disabling external crash reporting (AFL will guide you through this step)
 
-== Triaging Issues
+## Triaging Issues
 
 Crashes are usually interesting, particularly if using AFL_HARDEN=1 and --enable-expensive-hardening. Sometimes crashes are due to bugs in the harness code.
 
@@ -115,7 +115,7 @@ To see what fuzz-http is doing with a test case, call it like this:
 
 (Logging is disabled while fuzzing to increase fuzzing speed.)
 
-== Reporting Issues
+## Reporting Issues
 
 Please report any issues discovered using the process in Tor's security issue
 policy:
diff --git a/doc/HACKING/HelpfulTools.md b/doc/HACKING/HelpfulTools.md
index d499238526..cba57e875d 100644
--- a/doc/HACKING/HelpfulTools.md
+++ b/doc/HACKING/HelpfulTools.md
@@ -371,3 +371,18 @@ source code. Here's how to use it:
 
   6. See the Doxygen manual for more information; this summary just
      scratches the surface.
+
+Style and best-pratices checking
+--------------------------------
+
+We use scripts to check for various problems in the formatting and style
+of our source code.  The "check-spaces" test detects a bunch of violations
+of our coding style on the local level.  The "check-best-practices" test
+looks for violations of some of our complexity guidelines.
+
+You can tell the tool about exceptions to the complexity guidelines via its
+exceptions file (scripts/maint/practracker/exceptions.txt).  But before you
+do this, consider whether you shouldn't fix the underlying problem.  Maybe
+that file really _is_ too big.  Maybe that function really _is_ doing too
+much.  (On the other hand, for stable release series, it is sometimes better
+to leave things unrefactored.)
diff --git a/doc/HACKING/HowToReview.md b/doc/HACKING/HowToReview.md
index 2d1f3d1c9e..2325e70175 100644
--- a/doc/HACKING/HowToReview.md
+++ b/doc/HACKING/HowToReview.md
@@ -63,7 +63,7 @@ Let's look at the code!
 Let's look at the documentation!
 --------------------------------
 
-- Does the documentation confirm to CodingStandards.txt?
+- Does the documentation conform to CodingStandards.txt?
 
 - Does it make sense?
 
diff --git a/doc/HACKING/Maintaining.md b/doc/HACKING/Maintaining.md
new file mode 100644
index 0000000000..4d5a7f6b76
--- /dev/null
+++ b/doc/HACKING/Maintaining.md
@@ -0,0 +1,113 @@
+# Maintaining Tor
+
+This document details the duties and processes on maintaining the Tor code
+base.
+
+The first section describes who is the current Tor maintainer and what are the
+responsibilities. Tor has one main single maintainer but does have many
+committers and subsystem maintainers.
+
+The second third section describes how the **alpha and master** branches are
+maintained and by whom.
+
+Finally, the last section describes how the **stable** branches are maintained
+and by whom.
+
+This document does not cover how Tor is released, please see
+[ReleasingTor.md](ReleasingTor.md) for that information.
+
+## Tor Maintainer
+
+The current maintainer is Nick Mathewson <nickm@torproject.org>.
+
+The maintainer takes final decisions in terms of engineering, architecture and
+protocol design. Releasing Tor falls under their responsibility.
+
+## Alpha and Master Branches
+
+The Tor repository always has at all times a **master** branch which contains
+the upstream ongoing development.
+
+It may also contain a branch for a released feature freezed version which is
+called the **alpha** branch. The git tag and version number is always
+postfixed with `-alpha[-dev]`. For example: `tor-0.3.5.0-alpha-dev` or
+`tor-0.3.5.3-alpha`.
+
+Tor is separated into subsystems and some of those are maintained by other
+developers than the main maintainer. Those people have commit access to the
+code base but only commit (in most cases) into the subsystem they maintain.
+
+Upstream merges are restricted to the alpha and master branches. Subsystem
+maintainers should never push a patch into a stable branch which is the
+responsibility of the [stable branch maintainer](#stable-branches).
+
+### Who
+
+In alphabetical order, the following people have upstream commit access and
+maintain the following subsystems:
+
+- David Goulet <dgoulet@torproject.org>
+  * Onion Service (including Shared Random).  
+    ***keywords:*** *[tor-hs]*
+  * Channels, Circuitmux, Connection, Scheduler.  
+    ***keywords:*** *[tor-chan, tor-cmux, tor-sched, tor-conn]*
+  * Cell Logic (Handling/Parsing).  
+    ***keywords:*** *[tor-cell]*
+  * Threading backend.
+    ***keywords:*** *[tor-thread]*  
+
+- George Kadianakis <asn@torproject.org>
+  * Onion Service (including Shared Random).  
+    ***keywords:*** *[tor-hs]*
+  * Guard.  
+    ***keywords:*** *[tor-guard]*
+  * Pluggable Transport (excluding Bridge networking).  
+    ***keywords:*** *[tor-pt]*
+
+### Tasks
+
+These are the tasks of a subsystem maintainer:
+
+1. Regularly go over `merge_ready` tickets relevant to the related subsystem
+   and for the current alpha or development (master branch) Milestone.
+
+2. A subsystem maintainer is expected to contribute to any design changes
+   (including proposals) or large patch set about the subsystem.
+
+3. Leave their ego at the door. Mistakes will be made but they have to be
+   taking care of seriously. Learn and move on quickly.
+
+### Merging Policy
+
+These are few important items to follow when merging code upstream:
+
+1. To merge code upstream, the patch must have passed our CI (currently
+   github.com/torproject), have a corresponding ticket and reviewed by
+   **at least** one person that is not the original coder.
+
+   Example A: If Alice writes a patch then Bob, a Tor network team member,
+   reviews it and flags it `merge_ready`. Then, the maintainer is required
+   to look at the patch and makes a decision.
+
+   Example B: If the maintainer writes a patch then Bob, a Tor network
+   team member, reviews it and flags it `merge_ready`, then the maintainer
+   can merge the code upstream.
+
+2. Maintainer makes sure the commit message should describe what was fixed
+   and, if it applies, how was it fixed. It should also always refer to
+   the ticket number.
+
+3. Trivial patches such as comment change, documentation, syntax issues or
+   typos can be merged without a ticket or reviewers.
+
+4. Tor uses the "merge forward" method, that is, if a patch applies to the
+   alpha branch, it has to be merged there first and then merged forward
+   into master.
+
+5. Maintainer should always consult with the network team about any doubts,
+   mis-understandings or unknowns of a patch. Final word will always go to the
+   main Tor maintainer.
+
+## Stable Branches
+
+(Currently being drafted and reviewed by the network team.)
diff --git a/doc/HACKING/ReleasingTor.md b/doc/HACKING/ReleasingTor.md
index 55a40fc89b..f40e2af573 100644
--- a/doc/HACKING/ReleasingTor.md
+++ b/doc/HACKING/ReleasingTor.md
@@ -5,7 +5,7 @@ Putting out a new release
 Here are the steps that the maintainer should take when putting out a
 new Tor release:
 
-=== 0. Preliminaries
+### 0. Preliminaries
 
 1. Get at least two of weasel/arma/Sebastian to put the new
    version number in their approved versions list.  Give them a few
@@ -18,35 +18,41 @@ new Tor release:
    date of a TB that contains it.  See note below in "commit, upload,
    announce".
 
-=== I. Make sure it works
+### I. Make sure it works
 
-1. Use it for a while, as a client, as a relay, as a hidden service,
-   and as a directory authority. See if it has any obvious bugs, and
-   resolve those.
+1. Make sure that CI passes: have a look at Travis
+   (https://travis-ci.org/torproject/tor/branches), Appveyor
+   (https://ci.appveyor.com/project/torproject/tor/history), and
+   Jenkins (https://jenkins.torproject.org/view/tor/).
+   Make sure you're looking at the right branches.
 
-   As applicable, merge the `maint-X` branch into the `release-X` branch.
-   But you've been doing that all along, right?
+   If there are any unexplained failures, try to fix them or figure them
+   out.
 
-2. Are all of the jenkins builders happy?  See jenkins.torproject.org.
+2. Verify that there are no big outstanding issues.  You might find such
+   issues --
 
-   What about the bsd buildbots?
-         See http://buildbot.pixelminers.net/builders/
+    * On Trac
 
-   What about Coverity Scan?
+    * On coverity scan
 
-   What about clang scan-build?
+    * On OSS-Fuzz
 
-   Does 'make distcheck' complain?
+3. Run checks that aren't covered above, including:
 
-   How about 'make test-stem' and 'make test-network' and
-   `make test-network-full`?
+    * clang scan-build.  (See the script in ./scripts/test/scan_build.sh)
 
-       - Are all those tests still happy with --enable-expensive-hardening ?
+    * make test-network and make test-network-all (with
+      --enable-fragile-hardening)
 
-   Any memory leaks?
+    * Running Tor yourself and making sure that it actually works for you.
 
+    * Running Tor under valgrind.  (Our 'fragile hardening' doesn't cover
+      libevent and openssl, so using valgrind will sometimes find extra
+      memory leaks.)
 
-=== II. Write a changelog
+
+### II. Write a changelog
 
 
 1a. (Alpha release variant)
@@ -55,11 +61,14 @@ new Tor release:
    of them and reordering to focus on what users and funders would find
    interesting and understandable.
 
-   To do this, first run `./scripts/maint/lintChanges.py changes/*` and
-   fix as many warnings as you can.  Then run `./scripts/maint/sortChanges.py
-   changes/* > changelog.in` to combine headings and sort the entries.
-   After that, it's time to hand-edit and fix the issues that lintChanges
-   can't find:
+   To do this, run
+      `./scripts/maint/sortChanges.py changes/* > changelog.in`
+   to combine headings and sort the entries.  Copy the changelog.in file
+   into the ChangeLog.  Run 'format_changelog.py' (see below) to clean
+   up the line breaks.
+
+   After that, it's time to hand-edit and fix the issues that
+   lintChanges can't find:
 
    1. Within each section, sort by "version it's a bugfix on", else by
       numerical ticket order.
@@ -68,8 +77,6 @@ new Tor release:
 
       Make stuff very terse
 
-      Make sure each section name ends with a colon
-
       Describe the user-visible problem right away
 
       Mention relevant config options by name.  If they're rare or unusual,
@@ -79,7 +86,9 @@ new Tor release:
 
       Present and imperative tense: not past.
 
-      'Relays', not 'servers' or 'nodes' or 'Tor relays'.
+      "Relays", not "servers" or "nodes" or "Tor relays".
+
+      "Onion services", not "hidden services".
 
       "Stop FOOing", not "Fix a bug where we would FOO".
 
@@ -100,12 +109,14 @@ new Tor release:
 
    For stable releases that backport things from later, we try to compose
    their releases, we try to make sure that we keep the changelog entries
-   identical to their original versions, with a 'backport from 0.x.y.z'
+   identical to their original versions, with a "backport from 0.x.y.z"
    note added to each section.  So in this case, once you have the items
    from the changes files copied together, don't use them to build a new
    changelog: instead, look up the corrected versions that were merged
    into ChangeLog in the master branch, and use those.
 
+   Add "backport from X.Y.Z" in the section header for these entries.
+
 2. Compose a short release blurb to highlight the user-facing
    changes. Insert said release blurb into the ChangeLog stanza. If it's
    a stable release, add it to the ReleaseNotes file too. If we're adding
@@ -128,47 +139,61 @@ new Tor release:
    text of existing entries, though.)
 
 
-=== III. Making the source release.
+### III. Making the source release.
 
 1. In `maint-0.?.x`, bump the version number in `configure.ac` and run
-   `perl scripts/maint/updateVersions.pl` to update version numbers in other
+   `make update-versions` to update version numbers in other
    places, and commit.  Then merge `maint-0.?.x` into `release-0.?.x`.
 
-   (NOTE: To bump the version number, edit `configure.ac`, and then run
-   either `make`, or `perl scripts/maint/updateVersions.pl`, depending on
-   your version.)
-
    When you merge the maint branch forward to the next maint branch, or into
    master, merge it with "-s ours" to avoid a needless version bump.
 
 2. Make distcheck, put the tarball up in somewhere (how about your
-   homedir on your homedir on people.torproject.org?) , and tell `#tor`
-   about it. Wait a while to see if anybody has problems building it.
-   (Though jenkins is usually pretty good about catching these things.)
+   homedir on your homedir on people.torproject.org?) , and tell `#tor-dev`
+   about it.
+
+   If you want, wait until at least one person has built it
+   successfully.  (We used to say "wait for others to test it", but our
+   CI has successfully caught these kinds of errors for the last several
+   years.)
+
 
-=== IV. Commit, upload, announce
+3. Make sure that the new version is recommended in the latest consensus.
+   (Otherwise, users will get confused when it complains to them
+   about its status.)
+
+   If it is not, you'll need to poke Roger, Weasel, and Sebastian again: see
+   item 0.1 at the start of this document.
+
+### IV. Commit, upload, announce
 
 1. Sign the tarball, then sign and push the git tag:
 
         gpg -ba <the_tarball>
-        git tag -u <keyid> tor-0.3.x.y-status
-        git push origin tag tor-0.3.x.y-status
+        git tag -s tor-0.4.x.y-<status>
+        git push origin tag tor-0.4.x.y-<status>
 
-   (You must do this before you update the website: it relies on finding
-   the version by tag.)
+   (You must do this before you update the website: the website scripts
+   rely on finding the version by tag.)
+
+   (If your default PGP key is not the one you want to sign with, then say
+   "-u <keyid>" instead of "-s".)
 
 2. scp the tarball and its sig to the dist website, i.e.
-   `/srv/dist-master.torproject.org/htdocs/` on dist-master. When you want
-   it to go live, you run "static-update-component dist.torproject.org"
-   on dist-master.
+   `/srv/dist-master.torproject.org/htdocs/` on dist-master. Run
+   "static-update-component dist.torproject.org" on dist-master.
 
-   In the webwml.git repository, `include/versions.wmi` and `Makefile`
-   to note the new version.
+   In the webwml.git repository, `include/versions.wmi` and `Makefile`.
+   In the project/web/tpo.git repository, update `databags/versions.ini`
+   to note the new version.  Push these changes to master.
 
    (NOTE: Due to #17805, there can only be one stable version listed at
    once.  Nonetheless, do not call your version "alpha" if it is stable,
    or people will get confused.)
 
+   (NOTE: It will take a while for the website update scripts to update
+   the website.)
+
 3. Email the packagers (cc'ing tor-team) that a new tarball is up.
    The current list of packagers is:
 
@@ -186,37 +211,47 @@ new Tor release:
 
    Also, email tor-packagers@lists.torproject.org.
 
+   Mention where to download the tarball (https://dist.torproject.org).
+
+   Include a link to the changelog.
+
+
 4. Add the version number to Trac.  To do this, go to Trac, log in,
     select "Admin" near the top of the screen, then select "Versions" from
     the menu on the left.  At the right, there will be an "Add version"
     box.  By convention, we enter the version in the form "Tor:
-    0.2.2.23-alpha" (or whatever the version is), and we select the date as
+    0.4.0.1-alpha" (or whatever the version is), and we select the date as
     the date in the ChangeLog.
 
-5. Double-check: did the version get recommended in the consensus yet?  Is
-   the website updated?  If not, don't announce until they have the
-   up-to-date versions, or people will get confused.
+5. Wait for the download page to be updated. (If you don't do this before you
+   announce, people will be confused.)
 
 6. Mail the release blurb and ChangeLog to tor-talk (development release) or
    tor-announce (stable).
 
    Post the changelog on the blog as well. You can generate a
-   blog-formatted version of the changelog with the -B option to
-   format-changelog.
+   blog-formatted version of the changelog with
+      `./scripts/maint/format_changelog.py --B`
 
    When you post, include an estimate of when the next TorBrowser
    releases will come out that include this Tor release.  This will
    usually track https://wiki.mozilla.org/RapidRelease/Calendar , but it
    can vary.
 
+   For templates to use when announcing, see:
+       https://trac.torproject.org/projects/tor/wiki/org/teams/NetworkTeam/AnnouncementTemplates
 
-=== V. Aftermath and cleanup
+### V. Aftermath and cleanup
 
 1. If it's a stable release, bump the version number in the
     `maint-x.y.z` branch to "newversion-dev", and do a `merge -s ours`
     merge to avoid taking that change into master.
 
-2. Forward-port the ChangeLog (and ReleaseNotes if appropriate).
+2. If there is a new `maint-x.y.z` branch, create a Travis CI cron job that
+   builds the release every week. (It's ok to skip the weekly build if the
+   branch was updated in the last 24 hours.)
 
-3. Keep an eye on the blog post, to moderate comments and answer questions.
+3. Forward-port the ChangeLog (and ReleaseNotes if appropriate) to the
+   master branch.
 
+4. Keep an eye on the blog post, to moderate comments and answer questions.
diff --git a/doc/HACKING/design/00-overview.md b/doc/HACKING/design/00-overview.md
new file mode 100644
index 0000000000..2103a9062a
--- /dev/null
+++ b/doc/HACKING/design/00-overview.md
@@ -0,0 +1,124 @@
+
+## Overview ##
+
+This document describes the general structure of the Tor codebase, how
+it fits together, what functionality is available for extending Tor,
+and gives some notes on how Tor got that way.
+
+Tor remains a work in progress: We've been working on it for more than a
+decade, and we've learned a lot about good coding since we first
+started.  This means, however, that some of the older pieces of Tor will
+have some "code smell" in them that could sure stand a brisk
+refactoring.  So when I describe a piece of code, I'll sometimes give a
+note on how it got that way, and whether I still think that's a good
+idea.
+
+The first drafts of this document were written in the Summer and Fall of
+2015, when Tor 0.2.6 was the most recent stable version, and Tor 0.2.7
+was under development.  If you're reading this far in the future, some
+things may have changed.  Caveat haxxor!
+
+This document is not an overview of the Tor protocol.  For that, see the
+design paper and the specifications at https://spec.torproject.org/ .
+
+For more information about Tor's coding standards and some helpful
+development tools, see doc/HACKING in the Tor repository.
+
+For more information about writing tests, see doc/HACKING/WritingTests.txt
+in the Tor repository.
+
+### The very high level ###
+
+Ultimately, Tor runs as an event-driven network daemon: it responds to
+network events, signals, and timers by sending and receiving things over
+the network.  Clients, relays, and directory authorities all use the
+same codebase: the Tor process will run as a client, relay, or authority
+depending on its configuration.
+
+Tor has a few major dependencies, including Libevent (used to tell which
+sockets are readable and writable), OpenSSL (used for many encryption
+functions, and to implement the TLS protocol), and zlib (used to
+compress and uncompress directory information).
+
+Most of Tor's work today is done in a single event-driven main thread.
+Tor also spawns one or more worker threads to handle CPU-intensive
+tasks.  (Right now, this only includes circuit encryption.)
+
+On startup, Tor initializes its libraries, reads and responds to its
+configuration files, and launches a main event loop.  At first, the only
+events that Tor listens for are a few signals (like TERM and HUP), and
+one or more listener sockets (for different kinds of incoming
+connections).  Tor also configures a timer function to run once per
+second to handle periodic events.  As Tor runs over time, other events
+will open, and new events will be scheduled.
+
+The codebase is divided into a few main subdirectories:
+
+   src/common -- utility functions, not necessarily tor-specific.
+
+   src/or -- implements the Tor protocols.
+
+   src/test -- unit and regression tests
+
+   src/ext -- Code maintained elsewhere that we include in the Tor
+   source distribution.
+
+   src/trunnel -- automatically generated code (from the Trunnel)
+   tool: used to parse and encode binary formats.
+
+### Some key high-level abstractions ###
+
+The most important abstractions at Tor's high-level are Connections,
+Channels, Circuits, and Nodes.
+
+A 'Connection' represents a stream-based information flow.  Most
+connections are TCP connections to remote Tor servers and clients. (But
+as a shortcut, a relay will sometimes make a connection to itself
+without actually using a TCP connection.  More details later on.)
+Connections exist in different varieties, depending on what
+functionality they provide.  The principle types of connection are
+"edge" (eg a socks connection or a connection from an exit relay to a
+destination), "OR" (a TLS stream connecting to a relay), "Directory" (an
+HTTP connection to learn about the network), and "Control" (a connection
+from a controller).
+
+A 'Circuit' is persistent tunnel through the Tor network, established
+with public-key cryptography, and used to send cells one or more hops.
+Clients keep track of multi-hop circuits, and the cryptography
+associated with each hop.  Relays, on the other hand, keep track only of
+their hop of each circuit.
+
+A 'Channel' is an abstract view of sending cells to and from a Tor
+relay.  Currently, all channels are implemented using OR connections.
+If we switch to other strategies in the future, we'll have more
+connection types.
+
+A 'Node' is a view of a Tor instance's current knowledge and opinions
+about a Tor relay orbridge.
+
+### The rest of this document. ###
+
+> **Note**: This section describes the eventual organization of this
+> document, which is not yet complete.
+
+We'll begin with an overview of the various utility functions available
+in Tor's 'common' directory.  Knowing about these is key to writing
+portable, simple code in Tor.
+
+Then we'll go on and talk about the main data-flow of the Tor network:
+how Tor generates and responds to network traffic.  This will occupy a
+chapter for the main overview, with other chapters for special topics.
+
+After that, we'll mention the main modules in Tor, and describe the
+function of each.
+
+We'll cover the directory subsystem next: how Tor learns about other
+relays, and how relays advertise themselves.
+
+Then we'll cover a few specialized modules, such as hidden services,
+sandboxing, hibernation, accounting, statistics, guards, path
+generation, pluggable transports, and how they integrate with the rest of Tor.
+
+We'll close with a meandering overview of important pending issues in
+the Tor codebase, and how they affect the future of the Tor software.
+
diff --git a/doc/HACKING/design/01-common-utils.md b/doc/HACKING/design/01-common-utils.md
new file mode 100644
index 0000000000..79a6a7b7d3
--- /dev/null
+++ b/doc/HACKING/design/01-common-utils.md
@@ -0,0 +1,121 @@
+
+## Utility code in Tor
+
+Most of Tor's utility code is in modules in the src/common subdirectory.
+
+These are divided, broadly, into _compatibility_ functions, _utility_
+functions, _containers_, and _cryptography_.  (Someday in the future, it
+would be great to split these modules into separate directories.  Also, some
+functions are probably put in the wrong modules)
+
+### Compatibility code
+
+These functions live in src/common/compat\*.c; some corresponding macros live
+in src/common/compat\*.h.  They serve as wrappers around platform-specific or
+compiler-specific logic functionality.
+
+In general, the rest of the Tor code *should not* be calling platform-specific
+or otherwise non-portable functions.  Instead, they should call wrappers from
+compat.c, which implement a common cross-platform API.  (If you don't know
+whether a function is portable, it's usually good enough to see whether it
+exists on OSX, Linux, and Windows.)
+
+Other compatibility modules include backtrace.c, which generates stack traces
+for crash reporting; sandbox.c, which implements the Linux seccomp2 sandbox;
+and procmon.c, which handles monitoring a child process.
+
+Parts of address.c are compatibility code for handling network addressing
+issues; other parts are in util.c.
+
+Notable compatibility areas are:
+
+   * mmap support for mapping files into the address space (read-only)
+
+   * Code to work around the intricacies
+
+   * Workaround code for Windows's horrible winsock incompatibilities and
+     Linux's intricate socket extensions.
+
+   * Helpful string functions like memmem, memstr, asprintf, strlcpy, and
+     strlcat that not all platforms have.
+
+   * Locale-ignoring variants of the ctypes functions.
+
+   * Time-manipulation functions
+
+   * File locking function
+
+   * IPv6 functions for platforms that don't have enough IPv6 support
+
+   * Endianness functions
+
+   * OS functions
+
+   * Threading and locking functions.
+
+=== Utility functions
+
+General-purpose utilities are in util.c; they include higher-level wrappers
+around many of the compatibility functions to provide things like
+file-at-once access, memory management functions, math, string manipulation,
+time manipulation, filesystem manipulation, etc.
+
+(Some functionality, like daemon-launching, would be better off in a
+compatibility module.)
+
+In util_format.c, we have code to implement stuff like base-32 and base-64
+encoding.
+
+The address.c module interfaces with the system resolver and implements
+address parsing and formatting functions.  It converts sockaddrs to and from
+a more compact tor_addr_t type.
+
+The di_ops.c module provides constant-time comparison and associative-array
+operations, for side-channel avoidance.
+
+The logging subsystem in log.c supports logging to files, to controllers, to
+stdout/stderr, or to the system log.
+
+The abstraction in memarea.c is used in cases when a large amount of
+temporary objects need to be allocated, and they can all be freed at the same
+time.
+
+The torgzip.c module wraps the zlib library to implement compression.
+
+Workqueue.c provides a simple multithreaded work-queue implementation.
+
+### Containers
+
+The container.c module defines these container types, used throughout the Tor
+codebase.
+
+There is a dynamic array called **smartlist**, used as our general resizeable
+array type.  It supports sorting, searching, common set operations, and so
+on.  It has specialized functions for smartlists of strings, and for
+heap-based priority queues.
+
+There's a bit-array type.
+
+A set of mapping types to map strings, 160-bit digests, and 256-bit digests
+to void \*.  These are what we generally use when we want O(1) lookup.
+
+Additionally, for containers, we use the ht.h and tor_queue.h headers, in
+src/ext.  These provide intrusive hashtable and linked-list macros.
+
+###  Cryptography
+
+Once, we tried to keep our cryptography code in a single "crypto.c" file,
+with an "aes.c" module containing an AES implementation for use with older
+OpenSSLs.
+
+Now, our practice has become to introduce crypto_\*.c modules when adding new
+cryptography backend code.  We have modules for Ed25519, Curve25519,
+secret-to-key algorithms, and password-based boxed encryption.
+
+Our various TLS compatibility code, wrappers, and hacks are kept in
+tortls.c, which is probably too full of Tor-specific kludges.  I'm
+hoping we can eliminate most of those kludges when we finally remove
+support for older versions of our TLS handshake.
+
+
+
diff --git a/doc/HACKING/design/01a-memory.md b/doc/HACKING/design/01a-memory.md
new file mode 100644
index 0000000000..9a20782962
--- /dev/null
+++ b/doc/HACKING/design/01a-memory.md
@@ -0,0 +1,93 @@
+
+## Memory management
+
+### Heap-allocation functions
+
+Tor imposes a few light wrappers over C's native malloc and free
+functions, to improve convenience, and to allow wholescale replacement
+of malloc and free as needed.
+
+You should never use 'malloc', 'calloc', 'realloc, or 'free' on their
+own; always use the variants prefixed with 'tor_'.
+They are the same as the standard C functions, with the following
+exceptions:
+
+   * tor_free(NULL) is a no-op.
+   * tor_free() is a macro that takes an lvalue as an argument and sets it to
+     NULL after freeing it.  To avoid this behavior, you can use tor_free_()
+     instead.
+   * tor_malloc() and friends fail with an assertion if they are asked to
+     allocate a value so large that it is probably an underflow.
+   * It is always safe to tor_malloc(0), regardless of whether your libc
+     allows it.
+   * tor_malloc(), tor_realloc(), and friends are never allowed to fail.
+     Instead, Tor will die with an assertion.  This means that you never
+     need to check their return values.  See the next subsection for
+     information on why we think this is a good idea.
+
+We define additional general-purpose memory allocation functions as well:
+
+   * tor_malloc_zero(x) behaves as calloc(1, x), except the it makes clear
+     the intent to allocate a single zeroed-out value.
+   * tor_reallocarray(x,y) behaves as the OpenBSD reallocarray function.
+     Use it for cases when you need to realloc() in a multiplication-safe
+     way.
+
+And specific-purpose functions as well:
+
+   * tor_strdup() and tor_strndup() behaves as the underlying libc functions,
+     but use tor_malloc() instead of the underlying function.
+   * tor_memdup() copies a chunk of memory of a given size.
+   * tor_memdup_nulterm() copies a chunk of memory of a given size, then
+     NUL-terminates it just to be safe.
+
+#### Why assert on failure?
+
+Why don't we allow tor_malloc() and its allies to return NULL?
+
+First, it's error-prone.  Many programmers forget to check for NULL return
+values, and testing for malloc() failures is a major pain.
+
+Second, it's not necessarily a great way to handle OOM conditions. It's
+probably better (we think) to have a memory target where we dynamically free
+things ahead of time in order to stay under the target.  Trying to respond to
+an OOM at the point of tor_malloc() failure, on the other hand, would involve
+a rare operation invoked from deep in the call stack.  (Again, that's
+error-prone and hard to debug.)
+
+Third, thanks to the rise of Linux and other operating systems that allow
+memory to be overcommitted, you can't actually ever rely on getting a NULL
+from malloc() when you're out of memory; instead you have to use an approach
+closer to tracking the total memory usage.
+
+#### Conventions for your own allocation functions.
+
+Whenever you create a new type, the convention is to give it a pair of
+x_new() and x_free() functions, named after the type.
+
+Calling x_free(NULL) should always be a no-op.
+
+
+### Grow-only memory allocation: memarea.c
+
+It's often handy to allocate a large number of tiny objects, all of which
+need to disappear at the same time.  You can do this in tor using the
+memarea.c abstraction, which uses a set of grow-only buffers for allocation,
+and only supports a single "free" operation at the end.
+
+Using memareas also helps you avoid memory fragmentation.  You see, some libc
+malloc implementations perform badly on the case where a large number of
+small temporary objects are allocated at the same time as a few long-lived
+objects of similar size.  But if you use tor_malloc() for the long-lived ones
+and a memarea for the temporary object, the malloc implementation is likelier
+to do better.
+
+To create a new memarea, use memarea_new().  To drop all the storage from a
+memarea, and invalidate its pointers, use memarea_drop_all().
+
+The allocation functions memarea_alloc(), memarea_alloc_zero(),
+memarea_memdup(), memarea_strdup(), and memarea_strndup() are analogous to
+the similarly-named malloc() functions.  There is intentionally no
+memarea_free() or memarea_realloc().
+
+
diff --git a/doc/HACKING/design/01b-collections.md b/doc/HACKING/design/01b-collections.md
new file mode 100644
index 0000000000..def60b0f15
--- /dev/null
+++ b/doc/HACKING/design/01b-collections.md
@@ -0,0 +1,43 @@
+
+## Collections in tor
+
+### Smartlists: Neither lists, nor especially smart.
+
+For historical reasons, we call our dynamic-allocated array type
+"smartlist_t".  It can grow or shrink as elements are added and removed.
+
+All smartlists hold an array of void \*.  Whenever you expose a smartlist
+in an API you *must* document which types its pointers actually hold.
+
+<!-- It would be neat to fix that, wouldn't it? -NM  -->
+
+Smartlists are created empty with smartlist_new() and freed with
+smartlist_free().  See the containers.h module documentation for more
+information; there are many convenience functions for commonly needed
+operations.
+
+
+### Digest maps, string maps, and more.
+
+Tor makes frequent use of maps from 160-bit digests, 256-bit digests,
+or nul-terminated strings to void \*. These types are digestmap_t,
+digest256map_t, and strmap_t respectively.  See the containers.h
+module documentation for more information.
+
+
+### Intrusive lists and hashtables
+
+For performance-sensitive cases, we sometimes want to use "intrusive"
+collections: ones where the bookkeeping pointers are stuck inside the
+structures that belong to the collection.  If you've used the
+BSD-style sys/queue.h macros, you'll be familiar with these.
+
+Unfortunately, the sys/queue.h macros vary significantly between the
+platforms that have them, so we provide our own variants in
+src/ext/tor_queue.h .
+
+We also provide an intrusive hashtable implementation in src/ext/ht.h
+. When you're using it, you'll need to define your own hash
+functions. If attacker-induced collisions are a worry here, use the
+cryptographic siphash24g function to extract hashes.
+
diff --git a/doc/HACKING/design/01c-time.md b/doc/HACKING/design/01c-time.md
new file mode 100644
index 0000000000..5cd0b354fd
--- /dev/null
+++ b/doc/HACKING/design/01c-time.md
@@ -0,0 +1,75 @@
+
+## Time in tor ##
+
+### What time is it? ###
+
+We have several notions of the current time in Tor.
+
+The *wallclock time* is available from time(NULL) with
+second-granularity and tor_gettimeofday() with microsecond
+granularity.  It corresponds most closely to "the current time and date".
+
+The *monotonic time* is available with the set of monotime_\*
+functions declared in compat_time.h.  Unlike the wallclock time, it
+can only move forward. It does not necessarily correspond to a real
+world time, and it is not portable between systems.
+
+The *coarse monotonic time* is available from the set of
+monotime_coarse_\* functions in compat_time.h. It is the same as
+monotime_\* on some platforms. On others, it gives a monotonic timer
+with less precision, but which it's more efficient to access.
+
+### Cached views of time. ###
+
+On some systems (like Linux), many time functions use a VDSO to avoid
+the overhead of a system call.  But on other systems, gettimeofday()
+and time() can be costly enough that you wouldn't want to call them
+tens of thousands of times.  To get a recent, but not especially
+accurate, view of the current time, see approx_time() and
+tor_gettimeofday_cached().
+
+
+### Parsing and encoding time values ###
+
+Tor has functions to parse and format time in these formats:
+
+   * RFC1123 format. ("Fri, 29 Sep 2006 15:54:20 GMT").  For this,
+     use format_rfc1123_time() and parse_rfc1123_time.
+
+   * ISO8601 format. ("2006-10-29 10:57:20") For this, use
+     format_local_iso_time and format_iso_time.  We also support the
+     variant format "2006-10-29T10:57:20" with format_iso_time_nospace, and
+     "2006-10-29T10:57:20.123456" with format_iso_time_nospace_usec.
+
+   * HTTP format collections (preferably "Mon, 25 Jul 2016 04:01:11
+     GMT" or possibly "Wed Jun 30 21:49:08 1993" or even "25-Jul-16
+     04:01:11 GMT"). For this, use parse_http_time.  Don't generate anything
+     but the first format.
+
+Some of these functions use struct tm. You can use the standard
+tor_localtime_r and tor_gmtime_r() to wrap these in a safe way. We
+also have a tor_timegm() function.
+
+### Scheduling events ###
+
+The main way to schedule a not-too-frequent periodic event with
+respect to the Tor mainloop is via the mechanism in periodic.c.
+There's a big table of periodic_events in main.c, each of which gets
+invoked on its own schedule.  You should not expect more than about
+one second of accuracy with these timers.
+
+You can create an independent timer using libevent directly, or using
+the periodic_timer_new() function.  But you should avoid doing this
+for per-connection or per-circuit timers: Libevent's internal timer
+implementation uses a min-heap, and those tend to start scaling poorly
+once you have a few thousand entries.
+
+If you need to create a large number of fine-grained timers for some
+purpose, you should consider the mechanism in src/common/timers.c,
+which is optimized for the case where you have a large number of
+timers with not-too-long duration, many of which will be deleted
+before they actually expire. These timers should be reasonably
+accurate within a handful of milliseconds -- possibly better on some
+platforms.  (The timers.c module uses William Ahern's timeout.c
+implementation as its backend, which is based on a hierarchical timing
+wheel algorithm. It's cool stuff; check it out.)
diff --git a/doc/HACKING/design/01d-crypto.md b/doc/HACKING/design/01d-crypto.md
new file mode 100644
index 0000000000..d4def947d1
--- /dev/null
+++ b/doc/HACKING/design/01d-crypto.md
@@ -0,0 +1,169 @@
+
+## Lower-level cryptography functionality in Tor ##
+
+Generally speaking, Tor code shouldn't be calling OpenSSL (or any
+other crypto library) directly.  Instead, we should indirect through
+one of the functions in src/common/crypto\*.c or src/common/tortls.c.
+
+Cryptography functionality that's available is described below.
+
+### RNG facilities ###
+
+The most basic RNG capability in Tor is the crypto_rand() family of
+functions. These currently use OpenSSL's RAND_() backend, but may use
+something faster in the future.
+
+In addition to crypto_rand(), which fills in a buffer with random
+bytes, we also have functions to produce random integers in certain
+ranges; to produce random hostnames; to produce random doubles, etc.
+
+When you're creating a long-term cryptographic secret, you might want
+to use crypto_strongest_rand() instead of crypto_rand().  It takes the
+operating system's entropy source and combines it with output from
+crypto_rand().  This is a pure paranoia measure, but it might help us
+someday.
+
+You can use smartlist_choose() to pick a random element from a smartlist
+and smartlist_shuffle() to randomize the order of a smartlist.  Both are
+potentially a bit slow.
+
+### Cryptographic digests and related functions ###
+
+We treat digests as separate types based on the length of their
+outputs.  We support one 160-bit digest (SHA1), two 256-bit digests
+(SHA256 and SHA3-256), and two 512-bit digests (SHA512 and SHA3-512).
+
+You should not use SHA1 for anything new.
+
+The crypto_digest\*() family of functions manipulates digests.  You
+can either compute a digest of a chunk of memory all at once using
+crypto_digest(), crypto_digest256(), or crypto_digest512().  Or you
+can create a crypto_digest_t object with
+crypto_digest{,256,512}_new(), feed information to it in chunks using
+crypto_digest_add_bytes(), and then extract the final digest using
+crypto_digest_get_digest().  You can copy the state of one of these
+objects using crypto_digest_dup() or crypto_digest_assign().
+
+We support the HMAC hash-based message authentication code
+instantiated using SHA256. See crypto_hmac_sha256.  (You should not
+add any HMAC users with SHA1, and HMAC is not necessary with SHA3.)
+
+We also support the SHA3 cousins, SHAKE128 and SHAKE256.  Unlike
+digests, these are extendable output functions (or XOFs) where you can
+get any amount of output.  Use the crypto_xof_\*() functions to access
+these.
+
+We have several ways to derive keys from cryptographically strong secret
+inputs (like diffie-hellman outputs). The old
+crypto_expand_key_material-TAP() performs an ad-hoc KDF based on SHA1 -- you
+shouldn't use it for implementing anything but old versions of the Tor
+protocol.  You can use HKDF-SHA256 (as defined in RFC5869) for more modern
+protocols.  Also consider SHAKE256.
+
+If your input is potentially weak, like a password or passphrase, use a salt
+along with the secret_to_key() functions as defined in crypto_s2k.c.  Prefer
+scrypt over other hashing methods when possible.  If you're using a password
+to encrypt something, see the "boxed file storage" section below.
+
+Finally, in order to store objects in hash tables, Tor includes the
+randomized SipHash 2-4 function.  Call it via the siphash24g() function in
+src/ext/siphash.h whenever you're creating a hashtable whose keys may be
+manipulated by an attacker in order to DoS you with collisions.
+
+
+### Stream ciphers ###
+
+You can create instances of a stream cipher using crypto_cipher_new().
+These are stateful objects of type crypto_cipher_t.  Note that these
+objects only support AES-128 right now; a future version should add
+support for AES-128 and/or ChaCha20.
+
+You can encrypt/decrypt with crypto_cipher_encrypt or
+crypto_cipher_decrypt. The crypto_cipher_crypt_inplace function performs
+an encryption without a copy.
+
+Note that sensible people should not use raw stream ciphers; they should
+probably be using some kind of AEAD. Sorry.
+
+### Public key functionality ###
+
+We support four public key algorithms: DH1024, RSA, Curve25519, and
+Ed25519.
+
+We support DH1024 over two prime groups.  You access these via the
+crypto_dh_\*() family of functions.
+
+We support RSA in many bit sizes for signing and encryption.  You access
+it via the crypto_pk_*() family of functions.  Note that a crypto_pk_t
+may or may not include a private key.  See the crypto_pk_* functions in
+crypto.c for a full list of functions here.
+
+For Curve25519 functionality, see the functions and types in
+crypto_curve25519.c. Curve25519 is generally suitable for when you need
+a secure fast elliptic-curve diffie hellman implementation. When
+designing new protocols, prefer it over DH in Z_p.
+
+For Ed25519 functionality, see the functions and types in
+crypto_ed25519.c. Ed25519 is a generally suitable as a secure fast
+elliptic curve signature method. For new protocols, prefer it over RSA
+signatures.
+
+### Metaformats for storage ###
+
+When OpenSSL manages the storage of some object, we use whatever format
+OpenSSL provides -- typically, some kind of PEM-wrapped base 64 encoding
+that starts with "----- BEGIN CRYPTOGRAPHIC OBJECT ----".
+
+When we manage the storage of some cryptographic object, we prefix the
+object with 32-byte NUL-padded prefix in order to avoid accidental
+object confusion; see the crypto_read_tagged_contents_from_file() and
+crypto_write_tagged_contents_to_file() functions for manipulating
+these. The prefix is "== type: tag ==", where type describes the object
+and its encoding, and tag indicates which one it is.
+
+### Boxed-file storage ###
+
+When managing keys, you frequently want to have some way to write a
+secret object to disk, encrypted with a passphrase.  The crypto_pwbox
+and crypto_unpwbox functions do so in a way that's likely to be
+readable by future versions of Tor.
+
+### Certificates ###
+
+We have, alas, several certificate types in Tor.
+
+The tor_x509_cert_t type represents an X.509 certificate. This document
+won't explain X.509 to you -- possibly, no document can. (OTOH, Peter
+Gutmann's "x.509 style guide", though severely dated, does a good job of
+explaining how awful x.509 can be.)  Do not introduce any new usages of
+X.509. Right now we only use it in places where TLS forces us to do so.
+
+The authority_cert_t type is used only for directory authority keys. It
+has a medium-term signing key (which the authorities actually keep
+online) signed by a long-term identity key (which the authority operator
+had really better be keeping offline).  Don't use it for any new kind of
+certificate.
+
+For new places where you need a certificate, consider tor_cert_t: it
+represents a typed and dated _something_ signed by an Ed25519 key.  The
+format is described in tor-spec. Unlike x.509, you can write it on a
+napkin.
+
+(Additionally, the Tor directory design uses a fairly wide variety of
+documents that include keys and which are signed by keys. You can
+consider these documents to be an additional kind of certificate if you
+want.)
+
+### TLS ###
+
+Tor's TLS implementation is more tightly coupled to OpenSSL than we'd
+prefer.  You can read most of it in tortls.c.
+
+Unfortunately, TLS's state machine and our requirement for nonblocking
+IO support means that using TLS in practice is a bit hairy, since
+logical writes can block on a physical reads, and vice versa.
+
+If you are lucky, you will never have to look at the code here.
+
+
+
diff --git a/doc/HACKING/design/01e-os-compat.md b/doc/HACKING/design/01e-os-compat.md
new file mode 100644
index 0000000000..072e95bc8a
--- /dev/null
+++ b/doc/HACKING/design/01e-os-compat.md
@@ -0,0 +1,50 @@
+
+## OS compatibility functions ##
+
+We've got a bunch of functions to wrap differences between various
+operating systems where we run.
+
+### The filesystem ###
+
+We wrap the most important filesystem functions with load-file,
+save-file, and map-file abstractions declared in util.c or compat.c.  If
+you're messing about with file descriptors yourself, you might be doing
+things wrong.  Most of the time, write_str_to_file() and
+read_str_from_file() are all you need.
+
+Use the check_private_directory() function to create or verify the
+presence of directories, and tor_listdir() to list the files in a
+directory.
+
+Those modules also have functions for manipulating paths a bit.
+
+### Networking ###
+
+Nearly all the world is on a Berkeley sockets API, except for
+windows, whose version of the Berkeley API was corrupted by late-90s
+insistence on backward compatibility with the
+sort-of-berkeley-sort-of-not add-on *thing* that was WinSocks.
+
+What's more, everybody who implemented sockets realized that select()
+wasn't a very good way to do nonblocking IO... and then the various
+implementations all decided to so something different.
+
+You can forget about most of these differences, fortunately: We use
+libevent to hide most of the differences between the various networking
+backends, and we add a few of our own functions to hide the differences
+that Libevent doesn't.
+
+To create a network connection, the right level of abstraction to look
+at is probably the connection_t system in connection.c.  Most of the
+lower level work has already been done for you.  If you need to
+instantiate something that doesn't fit well with connection_t, you
+should see whether you can instantiate it with connection_t anyway -- or
+you might need to refactor connection.c a little.
+
+Whenever possible, represent network addresses as tor_addr_t.
+
+### Process launch and monitoring ###
+
+Launching and/or monitoring a process is tricky business. You can use
+the mechanisms in procmon.c and tor_spawn_background(), but they're both
+a bit wonky.  A refactoring would not be out of order.
diff --git a/doc/HACKING/design/01f-threads.md b/doc/HACKING/design/01f-threads.md
new file mode 100644
index 0000000000..a0dfa2d40e
--- /dev/null
+++ b/doc/HACKING/design/01f-threads.md
@@ -0,0 +1,26 @@
+
+## Threads in Tor ##
+
+Tor is based around a single main thread and one or more worker
+threads.  We aim (with middling success) to use worker threads for
+CPU-intensive activities and the main thread for our networking.
+Fortunately (?) we have enough cryptography that moving what we can of the
+cryptographic processes to the workers should achieve good parallelism under most
+loads.  Unfortunately, we only have a small fraction of our
+cryptography done in our worker threads right now.
+
+Our threads-and-workers abstraction is defined in workqueue.c, which
+combines a work queue with a thread pool, and integrates the
+signalling with libevent.  Tor main instance of a work queue is
+instantiated in cpuworker.c.  It will probably need some refactoring
+as more types of work are added.
+
+On a lower level, we provide locks with tor_mutex_t, conditions with
+tor_cond_t, and thread-local storage with tor_threadlocal_t, all of
+which are specified in compat_threads.h and implemented in an OS-
+specific compat_\*threads.h module.
+
+Try to minimize sharing between threads: it is usually best to simply
+make the worker "own" all the data it needs while the work is in
+progress, and to give up ownership when it's complete.
+
diff --git a/doc/HACKING/design/01g-strings.md b/doc/HACKING/design/01g-strings.md
new file mode 100644
index 0000000000..145a35cd6f
--- /dev/null
+++ b/doc/HACKING/design/01g-strings.md
@@ -0,0 +1,95 @@
+
+## String processing in Tor ##
+
+Since you're reading about a C program, you probably expected this
+section: it's full of functions for manipulating the (notoriously
+dubious) C string abstraction.  I'll describe some often-missed
+highlights here.
+
+### Comparing strings and memory chunks ###
+
+We provide strcmpstart() and strcmpend() to perform a strcmp with the start
+or end of a string.
+
+	tor_assert(!strcmpstart("Hello world","Hello"));
+	tor_assert(!strcmpend("Hello world","world"));
+
+	tor_assert(!strcasecmpstart("HELLO WORLD","Hello"));
+	tor_assert(!strcasecmpend("HELLO WORLD","world"));
+
+To compare two string pointers, either of which might be NULL, use
+strcmp_opt().
+
+To search for a string or a chunk of memory within a non-null
+terminated memory block, use tor_memstr or tor_memmem respectively.
+
+We avoid using memcmp() directly, since it tends to be used in cases
+when having a constant-time operation would be better.  Instead, we
+recommend tor_memeq() and tor_memneq() for when you need a
+constant-time operation.  In cases when you need a fast comparison,
+and timing leaks are not a danger, you can use fast_memeq() and
+fast_memneq().
+
+It's a common pattern to take a string representing one or more lines
+of text, and search within it for some other string, at the start of a
+line.  You could search for "\\ntarget", but that would miss the first
+line.  Instead, use find_str_at_start_of_line.
+
+### Parsing text ###
+
+Over the years, we have accumulated lots of ways to parse text --
+probably too many. Refactoring them to be safer and saner could be a
+good project!  The one that seems most error-resistant is tokenizing
+text with smartlist_split_strings().  This function takes a smartlist,
+a string, and a separator, and splits the string along occurrences of
+the separator, adding new strings for the sub-elements to the given
+smartlist.
+
+To handle time, you can use one of the functions mentioned above in
+"Parsing and encoding time values".
+
+For numbers in general, use the tor_parse_{long,ulong,double,uint64}
+family of functions.  Each of these can be called in a few ways.  The
+most general is as follows:
+
+      const int BASE = 10;
+      const int MINVAL = 10, MAXVAL = 10000;
+      const char *next;
+      int ok;
+      long lng = tor_parse_long("100", BASE, MINVAL, MAXVAL, &ok, &next);
+
+The return value should be ignored if "ok" is set to false.  The input
+string needs to contain an entire number, or it's considered
+invalid... unless the "next" pointer is available, in which case extra
+characters at the end are allowed, and "next" is set to point to the
+first such character.
+
+### Generating blocks of text ###
+
+For not-too-large blocks of text, we provide tor_asprintf(), which
+behaves like other members of the sprintf() family, except that it
+always allocates enough memory on the heap for its output.
+
+For larger blocks: Rather than using strlcat and strlcpy to build
+text, or keeping pointers to the interior of a memory block, we
+recommend that you use the smartlist_* functions to build a smartlist
+full of substrings in order.  Then you can concatenate them into a
+single string with smartlist_join_strings(), which also takes optional
+separator and terminator arguments.
+
+As a convenience, we provide smartlist_add_asprintf(), which combines
+the two methods above together.  Many of the cryptographic digest
+functions also accept a not-yet-concatenated smartlist of strings.
+
+### Logging helpers ###
+
+Often we'd like to log a value that comes from an untrusted source.
+To do this, use escaped() to escape the nonprintable characters and
+other confusing elements in a string, and surround it by quotes.  (Use
+esc_for_log() if you need to allocate a new string.)
+
+It's also handy to put memory chunks into hexadecimal before logging;
+you can use hex_str(memory, length) for that.
+
+The escaped() and hex_str() functions both provide outputs that are
+only valid till they are next invoked; they are not threadsafe.
diff --git a/doc/HACKING/design/02-dataflow.md b/doc/HACKING/design/02-dataflow.md
new file mode 100644
index 0000000000..39f21a908c
--- /dev/null
+++ b/doc/HACKING/design/02-dataflow.md
@@ -0,0 +1,236 @@
+
+## Data flow in the Tor process ##
+
+We read bytes from the network, we write bytes to the network.  For the
+most part, the bytes we write correspond roughly to bytes we have read,
+with bits of cryptography added in.
+
+The rest is a matter of details.
+
+![Diagram of main data flows in Tor](./diagrams/02/02-dataflow.png "Diagram of main data flows in Tor")
+
+### Connections and buffers: reading, writing, and interpreting. ###
+
+At a low level, Tor's networking code is based on "connections".  Each
+connection represents an object that can send or receive network-like
+events.  For the most part, each connection has a single underlying TCP
+stream (I'll discuss counterexamples below).
+
+A connection that behaves like a TCP stream has an input buffer and an
+output buffer.  Incoming data is
+written into the input buffer ("inbuf"); data to be written to the
+network is queued on an output buffer ("outbuf").
+
+Buffers are implemented in buffers.c.  Each of these buffers is
+implemented as a linked queue of memory extents, in the style of classic
+BSD mbufs, or Linux skbufs.
+
+A connection's reading and writing can be enabled or disabled.  Under
+the hood, this functionality is implemented using libevent events: one
+for reading, one for writing.  These events are turned on/off in
+main.c, in the functions connection_{start,stop}_{reading,writing}.
+
+When a read or write event is turned on, the main libevent loop polls
+the kernel, asking which sockets are ready to read or write.  (This
+polling happens in the event_base_loop() call in run_main_loop_once()
+in main.c.)  When libevent finds a socket that's ready to read or write,
+it invokes conn_{read,write}_callback(), also in main.c
+
+These callback functions delegate to connection_handle_read() and
+connection_handle_write() in connection.c, which read or write on the
+network as appropriate, possibly delegating to openssl.
+
+After data is read or written, or other event occurs, these
+connection_handle_read_write() functions call logic functions whose job is
+to respond to the information.  Some examples included:
+
+   * connection_flushed_some() -- called after a connection writes any
+     amount of data from its outbuf.
+   * connection_finished_flushing() -- called when a connection has
+     emptied its outbuf.
+   * connection_finished_connecting() -- called when an in-process connection
+     finishes making a remote connection.
+   * connection_reached_eof() -- called after receiving a FIN from the
+     remote server.
+   * connection_process_inbuf() -- called when more data arrives on
+     the inbuf.
+
+These functions then call into specific implementations depending on
+the type of the connection.  For example, if the connection is an
+edge_connection_t, connection_reached_eof() will call
+connection_edge_reached_eof().
+
+> **Note:** "Also there are bufferevents!"  We have vestigial
+> code for an alternative low-level networking
+> implementation, based on Libevent's evbuffer and bufferevent
+> code.  These two object types take on (most of) the roles of
+> buffers and connections respectively. It isn't working in today's
+> Tor, due to code rot and possible lingering libevent bugs.  More
+> work is needed; it would be good to get this working efficiently
+> again, to have IOCP support on Windows.
+
+
+#### Controlling connections ####
+
+A connection can have reading or writing enabled or disabled for a
+wide variety of reasons, including:
+
+   * Writing is disabled when there is no more data to write
+   * For some connection types, reading is disabled when the inbuf is
+     too full.
+   * Reading/writing is temporarily disabled on connections that have
+     recently read/written enough data up to their bandwidth 
+   * Reading is disabled on connections when reading more data from them
+     would require that data to be buffered somewhere else that is
+     already full.
+
+Currently, these conditions are checked in a diffuse set of
+increasingly complex conditional expressions.  In the future, it could
+be helpful to transition to a unified model for handling temporary
+read/write suspensions.
+
+#### Kinds of connections ####
+
+Today Tor has the following connection and pseudoconnection types.
+For the most part, each type of channel has an associated C module
+that implements its underlying logic.
+
+**Edge connections** receive data from and deliver data to points
+outside the onion routing network.  See `connection_edge.c`. They fall into two types:
+
+**Entry connections** are a type of edge connection. They receive data
+from the user running a Tor client, and deliver data to that user.
+They are used to implement SOCKSPort, TransPort, NATDPort, and so on.
+Sometimes they are called "AP" connections for historical reasons (it
+used to stand for "Application Proxy").
+
+**Exit connections** are a type of edge connection. They exist at an
+exit node, and transmit traffic to and from the network.
+
+(Entry connections and exit connections are also used as placeholders
+when performing a remote DNS request; they are not decoupled from the
+notion of "stream" in the Tor protocol. This is implemented partially
+in `connection_edge.c`, and partially in `dnsserv.c` and `dns.c`.)
+
+**OR connections** send and receive Tor cells over TLS, using some
+version of the Tor link protocol.  Their implementation is spread
+across `connection_or.c`, with a bit of logic in `command.c`,
+`relay.c`, and `channeltls.c`.
+
+**Extended OR connections** are a type of OR connection for use on
+bridges using pluggable transports, so that the PT can tell the bridge
+some information about the incoming connection before passing on its
+data.  They are implemented in `ext_orport.c`.
+
+**Directory connections** are server-side or client-side connections
+that implement Tor's HTTP-based directory protocol.  These are
+instantiated using a socket when Tor is making an unencrypted HTTP
+connection.  When Tor is tunneling a directory request over a Tor
+circuit, directory connections are implemented using a linked
+connection pair (see below).  Directory connections are implemented in
+`directory.c`; some of the server-side logic is implemented in
+`dirserver.c`.
+
+**Controller connections** are local connections to a controller
+process implementing the controller protocol from
+control-spec.txt. These are in `control.c`.
+
+**Listener connections** are not stream oriented!  Rather, they wrap a
+listening socket in order to detect new incoming connections.  They
+bypass most of stream logic.  They don't have associated buffers.
+They are implemented in `connection.c`.
+
+![structure hierarchy for connection types](./diagrams/02/02-connection-types.png "structure hierarchy for connection types")
+
+>**Note**: "History Time!" You might occasionally find reference to a couple types of connections
+> which no longer exist in modern Tor.  A *CPUWorker connection*
+>connected the main Tor process to a thread or process used for
+>computation.  (Nowadays we use in-process communication.)  Even more
+>anciently, a *DNSWorker connection* connected the main tor process to
+>a separate thread or process used for running `gethostbyname()` or
+>`getaddrinfo()`.  (Nowadays we use Libevent's evdns facility to
+>perform DNS requests asynchronously.)
+
+#### Linked connections ####
+
+Sometimes two channels are joined together, such that data which the
+Tor process sends on one should immediately be received by the same
+Tor process on the other.  (For example, when Tor makes a tunneled
+directory connection, this is implemented on the client side as a
+directory connection whose output goes, not to the network, but to a
+local entry connection. And when a directory receives a tunnelled
+directory connection, this is implemented as an exit connection whose
+output goes, not to the network, but to a local directory connection.)
+
+The earliest versions of Tor to support linked connections used
+socketpairs for the purpose.  But using socketpairs forced us to copy
+data through kernelspace, and wasted limited file descriptors.  So
+instead, a pair of connections can be linked in-process.  Each linked
+connection has a pointer to the other, such that data written on one
+is immediately readable on the other, and vice versa.
+
+### From connections to channels ###
+
+There's an abstraction layer above OR connections (the ones that
+handle cells) and below cells called **Channels**.  A channel's
+purpose is to transmit authenticated cells from one Tor instance
+(relay or client) to another.
+
+Currently, only one implementation exists: Channel_tls, which sends
+and receiveds cells over a TLS-based OR connection.
+
+Cells are sent on a channel using
+`channel_write_{,packed_,var_}cell()`. Incoming cells arrive on a
+channel from its backend using `channel_queue*_cell()`, and are
+immediately processed using `channel_process_cells()`.
+
+Some cell types are handled below the channel layer, such as those
+that affect handshaking only.  And some others are passed up to the
+generic cross-channel code in `command.c`: cells like `DESTROY` and
+`CREATED` are all trivial to handle.  But relay cells
+require special handling...
+
+### From channels through circuits ###
+
+When a relay cell arrives on an existing circuit, it is handled in
+`circuit_receive_relay_cell()` -- one of the innermost functions in
+Tor.  This function encrypts or decrypts the relay cell as
+appropriate, and decides whether the cell is intended for the current
+hop of the circuit.
+
+If the cell *is* intended for the current hop, we pass it to
+`connection_edge_process_relay_cell()` in `relay.c`, which acts on it
+based on its relay command, and (possibly) queues its data on an
+`edge_connection_t`.
+
+If the cell *is not* intended for the current hop, we queue it for the
+next channel in sequence with `append cell_to_circuit_queue()`.  This
+places the cell on a per-circuit queue for cells headed out on that
+particular channel.
+
+### Sending cells on circuits: the complicated bit. ###
+
+Relay cells are queued onto circuits from one of two (main) sources:
+reading data from edge connections, and receiving a cell to be relayed
+on a circuit.  Both of these sources place their cells on cell queue:
+each circuit has one cell queue for each direction that it travels.
+
+A naive implementation would skip using cell queues, and instead write
+each outgoing relay cell.  (Tor did this in its earlier versions.)
+But such an approach tends to give poor performance, because it allows
+high-volume circuits to clog channels, and it forces the Tor server to
+send data queued on a circuit even after that circuit has been closed.
+
+So by using queues on each circuit, we can add cells to each channel
+on a just-in-time basis, choosing the cell at each moment based on
+a performance-aware algorithm.
+
+This logic is implemented in two main modules: `scheduler.c` and
+`circuitmux*.c`.  The scheduler code is responsible for determining
+globally, across all channels that could write cells, which one should
+next receive queued cells.  The circuitmux code determines, for all
+of the circuits with queued cells for a channel, which one should
+queue the next cell.
+
+(This logic applies to outgoing relay cells only; incoming relay cells
+are processed as they arrive.)
diff --git a/doc/HACKING/design/03-modules.md b/doc/HACKING/design/03-modules.md
new file mode 100644
index 0000000000..93eb9d3089
--- /dev/null
+++ b/doc/HACKING/design/03-modules.md
@@ -0,0 +1,247 @@
+
+## Tor's modules ##
+
+### Generic modules ###
+
+`buffers.c`
+: Implements the `buf_t` buffered data type for connections, and several
+low-level data handling functions to handle network protocols on it.
+
+`channel.c`
+: Generic channel implementation. Channels handle sending and receiving cells
+among tor nodes.
+
+`channeltls.c`
+: Channel implementation for TLS-based OR connections. Uses `connection_or.c`.
+
+`circuitbuild.c`
+: Code for constructing circuits and choosing their paths.  (*Note*:
+this module could plausibly be split into handling the client side,
+the server side, and the path generation aspects of circuit building.)
+
+`circuitlist.c`
+: Code for maintaining and navigating the global list of circuits.
+
+`circuitmux.c`
+: Generic circuitmux implementation. A circuitmux handles deciding, for a
+particular channel, which circuit should write next.
+
+`circuitmux_ewma.c`
+: A circuitmux implementation based on the EWMA (exponentially
+weighted moving average) algorithm.
+
+`circuituse.c`
+: Code to actually send and receive data on circuits.
+
+`command.c`
+: Handles incoming cells on channels.
+
+`config.c`
+: Parses options from torrc, and uses them to configure the rest of Tor.
+
+`confparse.c`
+: Generic torrc-style parser. Used to parse torrc and state files.
+
+`connection.c`
+: Generic and common connection tools, and implementation for the simpler
+connection types.
+
+`connection_edge.c`
+: Implementation for entry and exit connections.
+
+`connection_or.c`
+: Implementation for OR connections (the ones that send cells over TLS).
+
+`main.c`
+: Principal entry point, main loops, scheduled events, and network
+management for Tor.
+
+`ntmain.c`
+: Implements Tor as a Windows service. (Not very well.)
+
+`onion.c`
+: Generic code for generating and responding to CREATE and CREATED
+cells, and performing the appropriate onion handshakes. Also contains
+code to manage the server-side onion queue.
+
+`onion_fast.c`
+: Implements the old SHA1-based CREATE_FAST/CREATED_FAST circuit
+creation handshake. (Now deprecated.)
+
+`onion_ntor.c`
+: Implements the Curve25519-based NTOR circuit creation handshake.
+
+`onion_tap.c`
+: Implements the old RSA1024/DH1024-based TAP circuit creation handshake. (Now
+deprecated.)
+
+`relay.c`
+: Handles particular types of relay cells, and provides code to receive,
+encrypt, route, and interpret relay cells.
+
+`scheduler.c`
+: Decides which channel/circuit pair is ready to receive the next cell.
+
+`statefile.c`
+: Handles loading and storing Tor's state file.
+
+`tor_main.c`
+: Contains the actual `main()` function.  (This is placed in a separate
+file so that the unit tests can have their own `main()`.)
+
+
+### Node-status modules ###
+
+`directory.c`
+: Implements the HTTP-based directory protocol, including sending,
+receiving, and handling most request types.  (*Note*: The client parts
+of this, and the generic-HTTP parts of this, could plausibly be split
+off.)
+
+`microdesc.c`
+: Implements the compact "microdescriptor" format for keeping track of
+what we know about a router.
+
+`networkstatus.c`
+: Code for fetching, storing, and interpreting consensus vote documents.
+
+`nodelist.c`
+: Higher-level view of our knowledge of which Tor servers exist.  Each
+`node_t` corresponds to a router we know about.
+
+`routerlist.c`
+: Code for storing and retrieving router descriptors and extrainfo
+documents.
+
+`routerparse.c`
+: Generic and specific code for parsing all Tor directory information
+types.
+
+`routerset.c`
+: Parses and interprets a specification for a set of routers (by IP
+range, fingerprint, nickname (deprecated), or country).
+
+
+### Client modules ###
+
+`addressmap.c`
+: Handles client-side associations between one address and another.
+These are used to implement client-side DNS caching (NOT RECOMMENDED),
+MapAddress directives, Automapping, and more.
+
+`circpathbias.c`
+: Path bias attack detection for circuits: tracks whether
+connections made through a particular guard have an unusually high failure rate.
+
+`circuitstats.c`
+: Code to track circuit performance statistics in order to adapt our behavior.
+Notably includes an algorithm to track circuit build times.
+
+`dnsserv.c`
+: Implements DNSPort for clients. (Note that in spite of the word
+"server" in this module's name, it is used for Tor clients.  It
+implements a DNS server, not DNS for servers.)
+
+`entrynodes.c`
+: Chooses, monitors, and remembers guard nodes.  Also contains some
+bridge-related code.
+
+`torcert.c`
+: Code to interpret and generate Ed25519-based certificates.
+
+### Server modules ###
+
+`dns.c`
+: Server-side DNS code.  Handles sending and receiving DNS requests on
+exit nodes, and implements the server-side DNS cache.
+
+`dirserv.c`
+: Implements part of directory caches that handles responding to
+client requests.
+
+`ext_orport.c`
+: Implements the extended ORPort protocol for communication between
+server-side pluggable transports and Tor servers.
+
+`hibernate.c`
+: Performs bandwidth accounting, and puts Tor relays into hibernation
+when their bandwidth is exhausted.
+
+`router.c`
+: Management code for running a Tor server. In charge of RSA key
+maintenance, descriptor generation and uploading.
+
+`routerkeys.c`
+: Key handling code for a Tor server. (Currently handles only the
+Ed25519 keys, but the RSA keys could be moved here too.)
+
+
+### Onion service modules ###
+
+`rendcache.c`
+: Stores onion service descriptors.
+
+`rendclient.c`
+: Client-side implementation of the onion service protocol.
+
+`rendcommon.c`
+: Parts of the onion service protocol that are shared by clients,
+services, and/or Tor servers.
+
+`rendmid.c`
+: Tor-server-side implementation of the onion service protocol. (Handles
+acting as an introduction point or a rendezvous point.)
+
+`rendservice.c`
+: Service-side implementation of the onion service protocol.
+
+`replaycache.c`
+: Backend to check introduce2 requests for replay attempts.
+
+
+### Authority modules ###
+
+`dircollate.c`
+: Helper for `dirvote.c`: Given a set of votes, each containing a list
+of Tor nodes, determines which entries across all the votes correspond
+to the same nodes, and yields them in a useful order.
+
+`dirvote.c`
+: Implements the directory voting algorithms that authorities use.
+
+`keypin.c`
+: Implements a persistent key-pinning mechanism to tie RSA1024
+identities to ed25519 identities.
+
+### Miscellaneous modules ###
+
+`control.c`
+: Implements the Tor controller protocol.
+
+`cpuworker.c`
+: Implements the inner work queue function.  We use this to move the
+work of circuit creation (on server-side) to other CPUs.
+
+`fp_pair.c`
+: Types for handling 2-tuples of 20-byte fingerprints.
+
+`geoip.c`
+: Parses geoip files (which map IP addresses to country codes), and
+performs lookups on the internal geoip table.  Also stores some
+geoip-related statistics.
+
+`policies.c`
+: Parses and implements Tor exit policies.
+
+`reasons.c`
+: Maps internal reason-codes to human-readable strings.
+
+`rephist.c`
+: Tracks Tor servers' performance over time.
+
+`status.c`
+: Writes periodic "heartbeat" status messages about the state of the Tor
+process.
+
+`transports.c`
+: Implements management for the pluggable transports subsystem.
diff --git a/doc/HACKING/design/Makefile b/doc/HACKING/design/Makefile
new file mode 100644
index 0000000000..e126130970
--- /dev/null
+++ b/doc/HACKING/design/Makefile
@@ -0,0 +1,34 @@
+
+
+
+HTML= \
+	00-overview.html \
+	01-common-utils.html \
+	01a-memory.html \
+	01b-collections.html \
+	01c-time.html \
+	01d-crypto.html \
+	01e-os-compat.html \
+	01f-threads.html \
+	01g-strings.html \
+	02-dataflow.html \
+	03-modules.html \
+	this-not-that.html
+
+PNG = \
+	diagrams/02/02-dataflow.png \
+	diagrams/02/02-connection-types.png
+
+all: generated
+
+generated: $(HTML) $(PNG)
+
+%.html: %.md
+	maruku $< -o $@
+
+%.png: %.dia
+	dia $< --export=$@
+
+clean:
+	rm -f $(HTML)
+	rm -f $(PNG)
diff --git a/doc/HACKING/design/this-not-that.md b/doc/HACKING/design/this-not-that.md
new file mode 100644
index 0000000000..815c7b2fbc
--- /dev/null
+++ b/doc/HACKING/design/this-not-that.md
@@ -0,0 +1,51 @@
+
+Don't use memcmp.  Use {tor,fast}_{memeq,memneq,memcmp}.
+
+Don't use assert.  Use tor_assert or tor_assert_nonfatal or BUG.  Prefer
+nonfatal assertions or BUG()s.
+
+Don't use sprintf or snprintf.  Use tor_asprintf or tor_snprintf.
+
+Don't write hand-written binary parsers.  Use trunnel.
+
+Don't use malloc, realloc, calloc, free, strdup, etc. Use tor_malloc,
+tor_realloc, tor_calloc, tor_free, tor_strdup, etc.
+
+Don't use tor_realloc(x, y\*z). Use tor_reallocarray(x, y, z);
+
+Don't say "if (x) foo_free(x)".  Just foo_free(x) and make sure that
+foo_free(NULL) is a no-op.
+
+Don't use toupper or tolower; use TOR_TOUPPER and TOR_TOLOWER.
+
+Don't use isalpha, isalnum, etc.  Instead use TOR_ISALPHA, TOR_ISALNUM, etc.
+
+Don't use strcat, strcpy, strncat, or strncpy. Use strlcat and strlcpy
+instead.
+
+Don't use tor_asprintf then smartlist_add; use smartlist_add_asprintf.
+
+Don't use any of these functions: they aren't portable. Use the
+version prefixed with `tor_` instead: strtok_r, memmem, memstr,
+asprintf, localtime_r, gmtime_r, inet_aton, inet_ntop, inet_pton,
+getpass, ntohll, htonll, strdup,   (This list is incomplete.)
+
+Don't create or close sockets directly. Instead use the wrappers in
+compat.h.
+
+When creating new APIs, only use 'char \*' to represent 'pointer to a
+nul-terminated string'.  Represent 'pointer to a chunk of memory' as
+'uint8_t \*'.  (Many older Tor APIs ignore this rule.)
+
+Don't encode/decode u32, u64, or u16 to byte arrays by casting
+pointers. That can crash if the pointers aren't aligned, and can cause
+endianness problems.  Instead say something more like set_uint32(ptr,
+htonl(foo)) to encode, and ntohl(get_uint32(ptr)) to decode.
+
+Don't declare a 0-argument function with "void foo()".  That's C++
+syntax. In C you say "void foo(void)".
+
+When creating new APIs, use const everywhere you reasonably can.
+
+Sockets should have type tor_socket_t, not int.
+
diff --git a/doc/asciidoc-helper.sh b/doc/asciidoc-helper.sh
index a3ef53f884..765850a125 100755
--- a/doc/asciidoc-helper.sh
+++ b/doc/asciidoc-helper.sh
@@ -19,7 +19,7 @@ if [ "$1" = "html" ]; then
     base=${output%%.html.in}
 
     if [ "$2" != none ]; then
-      TZ=UTC "$2" -d manpage -o $output $input;
+      TZ=UTC "$2" -d manpage -o "$output" "$input";
     else
       echo "==================================";
       echo;
@@ -44,8 +44,8 @@ elif [ "$1" = "man" ]; then
       echo "==================================";
       exit 1;
     fi
-    if "$2" -f manpage $input; then
-      mv $base.1 $output;
+    if "$2" -f manpage "$input"; then
+      mv "$base.1" "$output";
     else
       cat<<EOF
 ==================================
diff --git a/doc/include.am b/doc/include.am
index 0a123aae11..a9d3fa1c98 100644
--- a/doc/include.am
+++ b/doc/include.am
@@ -15,17 +15,32 @@
 all_mans = doc/tor doc/tor-gencert doc/tor-resolve doc/torify doc/tor-print-ed-signing-cert
 
 if USE_ASCIIDOC
-nodist_man1_MANS = $(all_mans:=.1)
-doc_DATA = $(all_mans:=.html)
+txt_in = $(all_mans:=.1.txt)
+
+if BUILD_HTML_DOCS
 html_in = $(all_mans:=.html.in)
+doc_DATA = $(all_mans:=.html)
+else
+html_in =
+doc_DATA =
+endif
+
+if BUILD_MANPAGE
+nodist_man1_MANS = $(all_mans:=.1)
 man_in = $(all_mans:=.1.in)
-txt_in = $(all_mans:=.1.txt)
 else
+nodist_man1_MANS =
+man_in =
+endif
+
+else
+
 html_in =
+doc_DATA =
 man_in =
 txt_in =
 nodist_man1_MANS =
-doc_DATA =
+
 endif
 
 EXTRA_DIST+= doc/asciidoc-helper.sh			\
diff --git a/doc/tor-print-ed-signing-cert.1.txt b/doc/tor-print-ed-signing-cert.1.txt
index 1a3109df95..48a3f095d5 100644
--- a/doc/tor-print-ed-signing-cert.1.txt
+++ b/doc/tor-print-ed-signing-cert.1.txt
@@ -21,6 +21,12 @@ DESCRIPTION
 **tor-print-ed-signing-cert** is utility program for Tor relay operators to
 check expiration date of ed25519 signing certificate.
 
+Expiration date is printed in three formats:
+
+* Human-readable timestamp, localized to current timezone.
+* RFC 1123 timestamp (at GMT timezone).
+* Unix time value.
+
 SEE ALSO
 --------
 **tor**(1) +
diff --git a/doc/tor.1.txt b/doc/tor.1.txt
index 21b482802e..c7c41e7841 100644
--- a/doc/tor.1.txt
+++ b/doc/tor.1.txt
@@ -37,7 +37,7 @@ Project's website.
 
 COMMAND-LINE OPTIONS
 --------------------
-[[opt-h]] **-h**, **-help**::
+[[opt-h]] **-h**, **--help**::
     Display a short help message and exit.
 
 [[opt-f]] **-f** __FILE__::
@@ -93,7 +93,9 @@ COMMAND-LINE OPTIONS
     into Tor.  (Any module not listed is not optional in this version of Tor.)
 
 [[opt-version]] **--version**::
-    Display Tor version and exit.
+    Display Tor version and exit. The output is a single line of the format
+    "Tor version [version number]."  (The version number format
+    is as specified in version-spec.txt.)
 
 [[opt-quiet]] **--quiet**|**--hush**::
     Override the default console log.  By default, Tor starts out logging
@@ -149,8 +151,8 @@ instance, you can tell Tor to start listening for SOCKS connections on port
 9999 by passing --SocksPort 9999 or SocksPort 9999 to it on the command line,
 or by putting "SocksPort 9999" in the configuration file.  You will need to
 quote options with spaces in them: if you want Tor to log all debugging
-messages to debug.log, you will probably need to say --Log 'debug file
-debug.log'.
+messages to debug.log, you will probably need to say **--Log** `"debug file
+debug.log"`.
 
 Options on the command line override those in configuration files. See the
 next section for more information.
@@ -209,10 +211,14 @@ GENERAL OPTIONS
     Note that this option, and other bandwidth-limiting options, apply to TCP
     data only: They do not count TCP headers or DNS traffic. +
  +
+    Tor uses powers of two, not powers of ten, so 1 GByte is
+    1024*1024*1024 bytes as opposed to 1 billion bytes. +
+ +
     With this option, and in other options that take arguments in bytes,
     KBytes, and so on, other formats are also supported. Notably, "KBytes" can
     also be written as "kilobytes" or "kb"; "MBytes" can be written as
     "megabytes" or "MB"; "kbits" can be written as "kilobits"; and so forth.
+    Case doesn't matter.
     Tor also accepts "byte" and "bit" in the singular.
     The prefixes "tera" and "T" are also recognized.
     If no units are given, we default to bytes.
@@ -352,7 +358,7 @@ GENERAL OPTIONS
     all sockets will be set to this limit. Must be a value between 2048 and
     262144, in 1024 byte increments. Default of 8192 is recommended.
 
-[[ControlPort]] **ControlPort** __PORT__|**unix:**__path__|**auto** [__flags__]::
+[[ControlPort]] **ControlPort** \['address':]__port__|**unix:**__path__|**auto** [__flags__]::
     If set, Tor will accept connections on this port and allow those
     connections to control the Tor process using the Tor Control Protocol
     (described in control-spec.txt in
@@ -363,7 +369,8 @@ GENERAL OPTIONS
     methods means either method is sufficient to authenticate to Tor.) This
     option is required for many Tor controllers; most use the value of 9051.
     If a unix domain socket is used, you may quote the path using standard
-    C escape sequences.
+    C escape sequences. You can specify this directive multiple times, to
+    bind to multiple address/port pairs.
     Set it to "auto" to have Tor pick a port for you. (Default: 0) +
  +
     Recognized flags are...
@@ -444,13 +451,18 @@ GENERAL OPTIONS
     setting for DataDirectoryGroupReadable when the CacheDirectory is the
     same as the DataDirectory, and 0 otherwise. (Default: auto)
 
-[[FallbackDir]] **FallbackDir** __ipv4address__:__port__ orport=__port__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]::
-    When we're unable to connect to any directory cache for directory info
-    (usually because we don't know about any yet) we try a directory authority.
-    Clients also simultaneously try a FallbackDir, to avoid hangs on client
-    startup if a directory authority is down. Clients retry FallbackDirs more
-    often than directory authorities, to reduce the load on the directory
-    authorities.
+[[FallbackDir]] **FallbackDir** __ipv4address__:__dirport__ orport=__orport__ id=__fingerprint__ [weight=__num__] [ipv6=**[**__ipv6address__**]**:__orport__]::
+    When tor is unable to connect to any directory cache for directory info
+    (usually because it doesn't know about any yet) it tries a hard-coded
+    directory. Relays try one directory authority at a time. Clients try
+    multiple directory authorities and FallbackDirs, to avoid hangs on
+    startup if a hard-coded directory is down. Clients wait for a few seconds
+    between each attempt, and retry FallbackDirs more often than directory
+    authorities, to reduce the load on the directory authorities.  +
+ +
+    FallbackDirs should be stable relays with stable IP addresses, ports,
+    and identity keys. They must have a DirPort. +
+ +
     By default, the directory authorities are also FallbackDirs. Specifying a
     FallbackDir replaces Tor's default hard-coded FallbackDirs (if any).
     (See the **DirAuthority** entry for an explanation of each flag.)
@@ -460,30 +472,30 @@ GENERAL OPTIONS
     FallbackDir line is present, it replaces the hard-coded FallbackDirs,
     regardless of the value of UseDefaultFallbackDirs.) (Default: 1)
 
-[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__port__ __fingerprint__::
+[[DirAuthority]] **DirAuthority** [__nickname__] [**flags**] __ipv4address__:__dirport__ __fingerprint__::
     Use a nonstandard authoritative directory server at the provided address
     and port, with the specified key fingerprint. This option can be repeated
     many times, for multiple authoritative directory servers. Flags are
     separated by spaces, and determine what kind of an authority this directory
     is. By default, an authority is not authoritative for any directory style
-    or version unless an appropriate flag is given.
+    or version unless an appropriate flag is given. +
+ +
     Tor will use this authority as a bridge authoritative directory if the
-    "bridge" flag is set. If a flag "orport=**port**" is given, Tor will use the
-    given port when opening encrypted tunnels to the dirserver. If a flag
-    "weight=**num**" is given, then the directory server is chosen randomly
-    with probability proportional to that weight (default 1.0). If a
+    "bridge" flag is set. If a flag "orport=**orport**" is given, Tor will
+    use the given port when opening encrypted tunnels to the dirserver. If a
+    flag "weight=**num**" is given, then the directory server is chosen
+    randomly with probability proportional to that weight (default 1.0). If a
     flag "v3ident=**fp**" is given, the dirserver is a v3 directory authority
     whose v3 long-term signing key has the fingerprint **fp**. Lastly,
     if an "ipv6=**[**__ipv6address__**]**:__orport__" flag is present, then
-    the directory
-    authority is listening for IPv6 connections on the indicated IPv6 address
-    and OR Port. +
+    the directory authority is listening for IPv6 connections on the
+    indicated IPv6 address and OR Port. +
  +
     Tor will contact the authority at __ipv4address__ to
-    download directory documents. The provided __port__ value is a dirport;
-    clients ignore this in favor of the specified "orport=" value.  If an
-    IPv6 ORPort is supplied, Tor will
-    also download directory documents at the IPv6 ORPort. +
+    download directory documents. Clients always use the ORPort. Relays
+    usually use the DirPort, but will use the ORPort in some circumstances.
+    If an IPv6 ORPort is supplied, clients will also download directory
+    documents at the IPv6 ORPort, if they are configured to use IPv6. +
  +
     If no **DirAuthority** line is given, Tor will use the default directory
     authorities. NOTE: this option is intended for setting up a private Tor
@@ -602,20 +614,26 @@ GENERAL OPTIONS
     Otherwise the sandbox will be disabled. The option is currently an
     experimental feature. It only works on Linux-based operating systems,
     and only when Tor has been built with the libseccomp library. This option
-    can not be changed while tor is running.
+    can not be changed while tor is running. +
  +
-    When the Sandbox is 1, the following options can not be changed when tor
+    When the **Sandbox** is 1, the following options can not be changed when tor
     is running:
-    Address
-    ConnLimit
-    CookieAuthFile
-    DirPortFrontPage
-    ExtORPortCookieAuthFile
-    Logs
-    ServerDNSResolvConfFile
-    Tor must remain in client or server mode (some changes to ClientOnly and
-    ORPort are not allowed).
-    ClientOnionAuthDir and any files in it won't reload on HUP signal.
+    **Address**,
+    **ConnLimit**,
+    **CookieAuthFile**,
+    **DirPortFrontPage**,
+    **ExtORPortCookieAuthFile**,
+    **Logs**,
+    **ServerDNSResolvConfFile**,
+    **ClientOnionAuthDir** (and any files in it won't reload on HUP signal).
+ +
+    Launching new Onion Services through the control port is not supported
+    with current syscall sandboxing implementation.
+ +
+    Tor must remain in client or server mode (some changes to **ClientOnly**
+    and **ORPort** are not allowed). Currently, if **Sandbox** is 1,
+    **ControlPort** command "GETINFO address" will not work.
+ +
     (Default: 0)
 
 [[Socks4Proxy]] **Socks4Proxy** __host__[:__port__]::
@@ -649,7 +667,16 @@ GENERAL OPTIONS
     debug, info, notice, warn, and err. We advise using "notice" in most cases,
     since anything more verbose may provide sensitive information to an
     attacker who obtains the logs. If only one severity level is given, all
-    messages of that level or higher will be sent to the listed destination.
+    messages of that level or higher will be sent to the listed destination. +
+ +
+    Some low-level logs may be sent from signal handlers, so their destination
+    logs must be signal-safe. These low-level logs include backtraces,
+    logging function errors, and errors in code called by logging functions.
+    Signal-safe logs are always sent to stderr or stdout. They are also sent to
+    a limited number of log files that are configured to log messages at error
+    severity from the bug or general domains. They are never sent as syslogs,
+    android logs, control port log events, or to any API-based log
+    destinations.
 
 [[Log2]] **Log** __minSeverity__[-__maxSeverity__] **file** __FILENAME__::
     As above, but send log messages to the listed filename. The
@@ -673,7 +700,8 @@ GENERAL OPTIONS
  +
     The currently recognized domains are: general, crypto, net, config, fs,
     protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge,
-    acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos.
+    acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos,
+    process, pt, btrack, and mesg.
     Domain names are case-insensitive. +
  +
     For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends
@@ -949,6 +977,20 @@ The following options are useful only for clients (that is, if
     this option. This option should be offered via the UI to mobile users
     for use where bandwidth may be expensive. (Default: 0)
 
+[[CircuitPadding]] **CircuitPadding** **0**|**1**::
+    If set to 0, Tor will not pad client circuits with additional cover
+    traffic. Only clients may set this option. This option should be offered
+    via the UI to mobile users for use where bandwidth may be expensive. If
+    set to 1, padding will be negotiated as per the consensus and relay
+    support (unlike ConnectionPadding, CircuitPadding cannot be force-enabled).
+    (Default: 1)
+
+[[ReducedCircuitPadding]] **ReducedCircuitPadding** **0**|**1**::
+    If set to 1, Tor will only use circuit padding algorithms that have low
+    overhead. Only clients may set this option. This option should be offered
+    via the UI to mobile users for use where bandwidth may be expensive.
+    (Default: 0)
+
 [[ExcludeNodes]] **ExcludeNodes** __node__,__node__,__...__::
     A list of identity fingerprints, country codes, and address
     patterns of nodes to avoid when building a circuit. Country codes are
@@ -1015,6 +1057,26 @@ The following options are useful only for clients (that is, if
     The .exit address notation, if enabled via MapAddress, overrides
     this option.
 
+[[MiddleNodes]] **MiddleNodes** __node__,__node__,__...__::
+    A list of identity fingerprints and country codes of nodes
+    to use for "middle" hops in your normal circuits.
+    Normal circuits include all circuits except for direct connections
+    to directory servers. Middle hops are all hops other than exit and entry. +
++
+    This is an **experimental** feature that is meant to be used by researchers
+    and developers to test new features in the Tor network safely. Using it
+    without care will strongly influence your anonymity. This feature might get
+    removed in the future.
++
+    The HSLayer2Node and HSLayer3Node options override this option for onion
+    service circuits, if they are set. The vanguards addon will read this
+    option, and if set, it will set HSLayer2Nodes and HSLayer3Nodes to nodes
+    from this set.
++
+    The ExcludeNodes option overrides this option: any node listed in both
+    MiddleNodes and ExcludeNodes is treated as excluded. See
+    the **ExcludeNodes** option for more information on how to specify nodes.
+
 [[EntryNodes]] **EntryNodes** __node__,__node__,__...__::
     A list of identity fingerprints and country codes of nodes
     to use for the first hop in your normal circuits.
@@ -1030,14 +1092,15 @@ The following options are useful only for clients (that is, if
 [[StrictNodes]] **StrictNodes** **0**|**1**::
     If StrictNodes is set to 1, Tor will treat solely the ExcludeNodes option
     as a requirement to follow for all the circuits you generate, even if
-    doing so will break functionality for you (StrictNodes applies to neither
-    ExcludeExitNodes nor to ExitNodes).  If StrictNodes is set to 0, Tor will
-    still try to avoid nodes in the ExcludeNodes list, but it will err on the
-    side of avoiding unexpected errors.  Specifically, StrictNodes 0 tells Tor
-    that it is okay to use an excluded node when it is *necessary* to perform
-    relay reachability self-tests, connect to a hidden service, provide a
-    hidden service to a client, fulfill a .exit request, upload directory
-    information, or download directory information.  (Default: 0)
+    doing so will break functionality for you (StrictNodes does not apply to
+    ExcludeExitNodes, ExitNodes, MiddleNodes, or MapAddress).  If StrictNodes
+    is set to 0, Tor will still try to avoid nodes in the ExcludeNodes list,
+    but it will err on the side of avoiding unexpected errors.
+    Specifically, StrictNodes 0 tells Tor that it is okay to use an excluded
+    node when it is *necessary* to perform relay reachability self-tests,
+    connect to a hidden service, provide a hidden service to a client,
+    fulfill a .exit request, upload directory information, or download
+    directory information.  (Default: 0)
 
 [[FascistFirewall]] **FascistFirewall** **0**|**1**::
     If 1, Tor will only create outgoing connections to ORs running on ports
@@ -1083,7 +1146,7 @@ The following options are useful only for clients (that is, if
     information) to port 80.
 
 [[HidServAuth]] **HidServAuth** __onion-address__ __auth-cookie__ [__service-name__]::
-    Client authorization for a hidden service. Valid onion addresses contain 16
+    Client authorization for a v2 hidden service. Valid onion addresses contain 16
     characters in a-z2-7 plus ".onion", and valid auth cookies contain 22
     characters in A-Za-z0-9+/. The service name is only used for internal
     purposes, e.g., for Tor controllers. This option may be used multiple times
@@ -1126,23 +1189,26 @@ The following options are useful only for clients (that is, if
     "MapAddress \*.example.com \*.example.com.torserver.exit". (Note the
     leading "*." in each part of the directive.) You can also redirect all
     subdomains of a domain to a single address. For example, "MapAddress
-    *.example.com www.example.com". +
+    *.example.com www.example.com". If the specified exit is not available,
+    or the exit can not connect to the site, Tor will fail any connections
+    to the mapped address.+
  +
     NOTES:
 
     1. When evaluating MapAddress expressions Tor stops when it hits the most
     recently added expression that matches the requested address. So if you
-    have the following in your torrc, www.torproject.org will map to 1.1.1.1:
+    have the following in your torrc, www.torproject.org will map to
+    198.51.100.1:
 
-     MapAddress www.torproject.org 2.2.2.2
-     MapAddress www.torproject.org 1.1.1.1
+     MapAddress www.torproject.org 192.0.2.1
+     MapAddress www.torproject.org 198.51.100.1
 
     2. Tor evaluates the MapAddress configuration until it finds no matches. So
     if you have the following in your torrc, www.torproject.org will map to
-    2.2.2.2:
+    203.0.113.1:
 
-      MapAddress 1.1.1.1 2.2.2.2
-      MapAddress www.torproject.org 1.1.1.1
+      MapAddress 198.51.100.1 203.0.113.1
+      MapAddress www.torproject.org 198.51.100.1
 
     3. The following MapAddress expression is invalid (and will be
     ignored) because you cannot map from a specific address to a wildcard
@@ -1153,6 +1219,15 @@ The following options are useful only for clients (that is, if
     4. Using a wildcard to match only part of a string (as in *ample.com) is
     also invalid.
 
+    5. Tor maps hostnames and IP addresses separately. If you MapAddress
+       a DNS name, but use an IP address to connect, then Tor will ignore the
+       DNS name mapping.
+
+    6. MapAddress does not apply to redirects in the application protocol.
+       For example, HTTP redirects and alt-svc headers will ignore mappings
+       for the original address. You can use a wildcard mapping to handle
+       redirects within the same site.
+
 [[NewCircuitPeriod]] **NewCircuitPeriod** __NUM__::
     Every NUM seconds consider whether to build a new circuit. (Default: 30
     seconds)
@@ -1202,6 +1277,8 @@ The following options are useful only for clients (that is, if
         information to anybody watching your network, and allow anybody
         to use your computer as an open proxy. +
  +
+    If multiple entries of this option are present in your configuration
+    file, Tor will perform stream isolation between listeners by default.
     The _isolation flags_ arguments give Tor rules for which streams
     received on this SocksPort are allowed to share circuits with one
     another.  Recognized isolation flags are:
@@ -1395,10 +1472,9 @@ The following options are useful only for clients (that is, if
     default to 3 if the consensus parameter isn't set.  (Default: 0)
 
 [[GuardLifetime]] **GuardLifetime**  __N__ **days**|**weeks**|**months**::
-    If nonzero, and UseEntryGuards is set, minimum time to keep a guard before
-    picking a new one. If zero, we use the GuardLifetime parameter from the
-    consensus directory.  No value here may  be less than 1 month or greater
-    than 5 years; out-of-range values are clamped. (Default: 0)
+    If UseEntryGuards is set, minimum time to keep a guard on our guard list
+    before  picking a new one.  If less than one day, we use defaults from the
+    consensus directory.  (Default: 0)
 
 [[SafeSocks]] **SafeSocks** **0**|**1**::
     When this option is enabled, Tor will reject application connections that
@@ -1444,14 +1520,18 @@ The following options are useful only for clients (that is, if
     protocol instead of SOCKS. Set this to
     0 if you don't want to allow "HTTP CONNECT" connections. Set the port
     to "auto" to have Tor pick a port for you. This directive can be
-    specified multiple times to bind to multiple addresses/ports.  See
+    specified multiple times to bind to multiple addresses/ports. If multiple
+    entries of this option are present in your configuration file, Tor will
+    perform stream isolation between listeners by default. See
     SOCKSPort for an explanation of isolation flags. (Default: 0)
 
 [[TransPort]] **TransPort**  \['address':]__port__|**auto** [_isolation flags_]::
     Open this port to listen for transparent proxy connections.  Set this to
     0 if you don't want to allow transparent proxy connections.  Set the port
     to "auto" to have Tor pick a port for you. This directive can be
-    specified multiple times to bind to multiple addresses/ports.  See
+    specified multiple times to bind to multiple addresses/ports. If multiple
+    entries of this option are present in your configuration file, Tor will
+    perform stream isolation between listeners by default.  See
     SOCKSPort for an explanation of isolation flags. +
  +
     TransPort requires OS support for transparent proxies, such as BSDs' pf or
@@ -1488,7 +1568,9 @@ The following options are useful only for clients (that is, if
     included in old versions of FreeBSD, etc) using the NATD protocol.
     Use 0 if you don't want to allow NATD connections.  Set the port
     to "auto" to have Tor pick a port for you. This directive can be
-    specified multiple times to bind to multiple addresses/ports.  See
+    specified multiple times to bind to multiple addresses/ports.  If multiple
+    entries of this option are present in your configuration file, Tor will
+    perform stream isolation between listeners by default. See
     SocksPort for an explanation of isolation flags. +
  +
     This option is only for people who cannot use TransPort. (Default: 0)
@@ -1753,6 +1835,12 @@ The following options are useful only for clients (that is, if
     other clients prefer IPv4. Other things may influence the choice. This
     option breaks a tie to the favor of IPv6. (Default: auto)
 
+[[ClientAutoIPv6ORPort]] **ClientAutoIPv6ORPort** **0**|**1**::
+    If this option is set to 1, Tor clients randomly prefer a node's IPv4 or
+    IPv6 ORPort. The random preference is set every time a node is loaded
+    from a new consensus or bridge config. When this option is set to 1,
+    **ClientPreferIPv6ORPort** is ignored. (Default: 0)
+
 [[PathsNeededToBuildCircuits]] **PathsNeededToBuildCircuits** __NUM__::
     Tor clients don't build circuits for user traffic until they know
     about enough of the network so that they could potentially construct
@@ -1794,6 +1882,43 @@ The following options are useful only for clients (that is, if
     Try this many simultaneous connections to download a consensus before
     waiting for one to complete, timeout, or error out. (Default: 3)
 
+[[DormantClientTimeout]] **DormantClientTimeout**  __N__ **minutes**|**hours**|**days**|**weeks**::
+    If Tor spends this much time without any client activity,
+    enter a dormant state where automatic circuits are not built, and
+    directory information is not fetched.
+    Does not affect servers or onion services. Must be at least 10 minutes.
+    (Default: 24 hours)
+
+[[DormantTimeoutDisabledByIdleStreams]] **DormantTimeoutDisabledByIdleStreams**  **0**|**1**::
+    If true, then any open client stream (even one not reading or writing)
+    counts as client activity for the purpose of DormantClientTimeout.
+    If false, then only network activity counts. (Default: 1)
+
+[[DormantOnFirstStartup]] **DormantOnFirstStartup** **0**|**1**::
+    If true, then the first time Tor starts up with a fresh DataDirectory,
+    it starts in dormant mode, and takes no actions until the user has made
+    a request.  (This mode is recommended if installing a Tor client for a
+    user who might not actually use it.)  If false, Tor bootstraps the first
+    time it is started, whether it sees a user request or not.
+ +
+    After the first time Tor starts, it begins in dormant mode if it was
+    dormant before, and not otherwise. (Default: 0)
+
+[[DormantCanceledByStartup]] **DormantCanceledByStartup** **0**|**1**::
+    By default, Tor starts in active mode if it was active the last time
+    it was shut down, and in dormant mode if it was dormant.  But if
+    this option is true, Tor treats every startup event as user
+    activity, and Tor will never start in Dormant mode, even if it has
+    been unused for a long time on previous runs. (Default: 0)
+ +
+    Note: Packagers and application developers should change the value of
+    this option only with great caution: it has the potential to
+    create spurious traffic on the network.  This option should only
+    be used if Tor is started by an affirmative user activity (like
+    clicking on an applcation or running a command), and not if Tor
+    is launched for some other reason (for example, by a startup
+    process, or by an application that launches itself on every login.)
+
 SERVER OPTIONS
 --------------
 
@@ -1831,10 +1956,6 @@ is non-zero):
     would like its bridge address to be given out. Set it to "none" if
     you want BridgeDB to avoid distributing your bridge address, or "any" to
     let BridgeDB decide. (Default: any)
- +
-    Note: as of Oct 2017, the BridgeDB part of this option is not yet
-    implemented.  Until BridgeDB is updated to obey this option, your
-    bridge will make this request, but it will not (yet) be obeyed.
 
 [[ContactInfo]] **ContactInfo** __email_address__::
     Administrative contact information for this relay or bridge. This line
@@ -1856,13 +1977,14 @@ is non-zero):
     exit according to the ExitPolicy option, the ReducedExitPolicy option,
     or the default ExitPolicy (if no other exit policy option is specified). +
  +
-    If ExitRelay is set to 0, no traffic is allowed to
-    exit, and the ExitPolicy and ReducedExitPolicy options are ignored. +
+    If ExitRelay is set to 0, no traffic is allowed to exit, and the
+    ExitPolicy, ReducedExitPolicy, and IPv6Exit options are ignored. +
  +
-    If ExitRelay is set to "auto", then Tor checks the ExitPolicy and
-    ReducedExitPolicy options. If either is set, Tor behaves as if ExitRelay
-    were set to 1. If neither exit policy option is set, Tor behaves as if
-    ExitRelay were set to 0. (Default: auto)
+    If ExitRelay is set to "auto", then Tor checks the ExitPolicy,
+    ReducedExitPolicy, and IPv6Exit options. If at least one of these options
+    is set, Tor behaves as if ExitRelay were set to 1. If none of these exit
+    policy options are set, Tor behaves as if ExitRelay were set to 0.
+    (Default: auto)
 
 [[ExitPolicy]] **ExitPolicy** __policy__,__policy__,__...__::
     Set an exit policy for this server. Each policy is of the form
@@ -2057,8 +2179,9 @@ is non-zero):
     (Default: 0)
 
 [[IPv6Exit]] **IPv6Exit** **0**|**1**::
-    If set, and we are an exit node, allow clients to use us for IPv6
-    traffic. (Default: 0)
+    If set, and we are an exit node, allow clients to use us for IPv6 traffic.
+    When this option is set and ExitRelay is auto, we act as if ExitRelay
+    is 1. (Default: 0)
 
 [[MaxOnionQueueDelay]] **MaxOnionQueueDelay** __NUM__ [**msec**|**second**]::
     If we have more onionskins queued for processing than we can process in
@@ -2169,9 +2292,9 @@ is non-zero):
     using a given calculation rule (see: AccountingStart, AccountingRule).
     Useful if you need to stay under a specific bandwidth. By default, the
     number used for calculation is the max of either the bytes sent or
-    received. For example, with AccountingMax set to 1 GByte, a server
-    could send 900 MBytes and receive 800 MBytes and continue running.
-    It will only hibernate once one of the two reaches 1 GByte. This can
+    received. For example, with AccountingMax set to 1 TByte, a server
+    could send 900 GBytes and receive 800 GBytes and continue running.
+    It will only hibernate once one of the two reaches 1 TByte. This can
     be changed to use the sum of the both bytes received and sent by setting
     the AccountingRule option to "sum" (total bandwidth in/out). When the
     number of bytes remaining gets low, Tor will stop accepting new connections
@@ -2182,7 +2305,12 @@ is non-zero):
     enabling hibernation is preferable to setting a low bandwidth, since
     it provides users with a collection of fast servers that are up some
     of the time, which is more useful than a set of slow servers that are
-    always "available".
+    always "available". +
+ +
+    Note that (as also described in the Bandwidth section) Tor uses
+    powers of two, not powers of ten: 1 GByte is 1024*1024*1024, not
+    one billion. Be careful: some internet service providers might count
+    GBytes differently.
 
 [[AccountingRule]] **AccountingRule** **sum**|**max**|**in**|**out**::
     How we determine when our AccountingMax has been reached (when we
@@ -2217,7 +2345,8 @@ is non-zero):
     __filename__. The file format is the same as the standard Unix
     "**resolv.conf**" file (7). This option, like all other ServerDNS options,
     only affects name lookups that your server does on behalf of clients.
-    (Defaults to use the system DNS configuration.)
+    (Defaults to use the system DNS configuration or a localhost DNS service
+    in case no nameservers are found in a given configuration.)
 
 [[ServerDNSAllowBrokenConfig]] **ServerDNSAllowBrokenConfig** **0**|**1**::
     If this option is false, Tor exits immediately if there are problems
@@ -2257,7 +2386,8 @@ is non-zero):
     When this option is enabled and BridgeRelay is also enabled, and we have
     GeoIP data, Tor keeps a per-country count of how many client
     addresses have contacted it so that it can help the bridge authority guess
-    which countries have blocked access to it. (Default: 1)
+    which countries have blocked access to it. If ExtraInfoStatistics is
+    enabled, it will be published as part of extra-info document. (Default: 1)
 
 [[ServerDNSRandomizeCase]] **ServerDNSRandomizeCase** **0**|**1**::
     When this option is set, Tor sets the case of each character randomly in
@@ -2339,6 +2469,10 @@ is non-zero):
 [[ExtraInfoStatistics]] **ExtraInfoStatistics** **0**|**1**::
     When this option is enabled, Tor includes previously gathered statistics in
     its extra-info documents that it uploads to the directory authorities.
+    Disabling this option also removes bandwidth usage statistics, and
+    GeoIPFile and GeoIPv6File hashes from the extra-info file. Bridge
+    ServerTransportPlugin lines are always includes in the extra-info file,
+    because they are required by BridgeDB.
     (Default: 1)
 
 [[ExtendAllowPrivateAddresses]] **ExtendAllowPrivateAddresses** **0**|**1**::
@@ -2606,12 +2740,6 @@ on the public Tor network.
     multiple times: the values from multiple lines are spliced together. When
     this is set then **VersioningAuthoritativeDirectory** should be set too.
 
-[[RecommendedPackages]] **RecommendedPackages** __PACKAGENAME__ __VERSION__ __URL__ __DIGESTTYPE__**=**__DIGEST__ ::
-    Adds "package" line to the directory authority's vote.  This information
-    is used to vote on the correct URL and digest for the released versions
-    of different Tor-related packages, so that the consensus can certify
-    them.  This line may appear any number of times.
-
 [[RecommendedClientVersions]] **RecommendedClientVersions** __STRING__::
     STRING is a comma-separated list of Tor versions currently believed to be
     safe for clients to use. This information is included in version 2
@@ -2797,15 +2925,30 @@ on the public Tor network.
     before it will treat advertised bandwidths as wholly
     unreliable. (Default: 500)
 
+[[AuthDirRejectRequestsUnderLoad]] **AuthDirRejectRequestsUnderLoad** **0**|**1**::
+    If set, the directory authority will start rejecting directory requests
+    from non relay connections by sending a 503 error code if it is under
+    bandwidth pressure (reaching the configured limit if any). Relays will
+    always tried to be answered even if this is on. (Default: 1)
+
+
 HIDDEN SERVICE OPTIONS
 ----------------------
 
-The following options are used to configure a hidden service.
+The following options are used to configure a hidden service. Some options
+apply per service and some apply for the whole tor instance.
+
+The next section describes the per service options that can only be set
+**after** the **HiddenServiceDir** directive
+
+**PER SERVICE OPTIONS:**
 
 [[HiddenServiceDir]] **HiddenServiceDir** __DIRECTORY__::
     Store data files for a hidden service in DIRECTORY. Every hidden service
     must have a separate directory. You may use this option  multiple times to
     specify multiple services. If DIRECTORY does not exist, Tor will create it.
+    Please note that you cannot add new Onion Service to already running Tor
+    instance if **Sandbox** is enabled.
     (Note: in current versions of Tor, if DIRECTORY is a relative path,
     it will be relative to the current
     working directory of Tor instance, not to its DataDirectory.  Do not
@@ -2824,18 +2967,12 @@ The following options are used to configure a hidden service.
     connects to that VIRTPORT, one of the TARGETs from those lines will be
     chosen at random. Note that address-port pairs have to be comma-separated.
 
-[[PublishHidServDescriptors]] **PublishHidServDescriptors** **0**|**1**::
-    If set to 0, Tor will run any hidden services you configure, but it won't
-    advertise them to the rendezvous directory. This option is only useful if
-    you're using a Tor controller that handles hidserv publishing for you.
-    (Default: 1)
-
 [[HiddenServiceVersion]] **HiddenServiceVersion** **2**|**3**::
     A list of rendezvous service descriptor versions to publish for the hidden
     service. Currently, versions 2 and 3 are supported. (Default: 3)
 
 [[HiddenServiceAuthorizeClient]] **HiddenServiceAuthorizeClient** __auth-type__ __client-name__,__client-name__,__...__::
-    If configured, the hidden service is accessible for authorized clients
+    If configured, the v2 hidden service is accessible for authorized clients
     only. The auth-type can either be \'basic' for a general-purpose
     authorization protocol or \'stealth' for a less scalable protocol that also
     hides service activity from unauthorized clients. Only clients that are
@@ -2856,7 +2993,7 @@ The following options are used to configure a hidden service.
 
 [[HiddenServiceExportCircuitID]] **HiddenServiceExportCircuitID** __protocol__::
    The onion service will use the given protocol to expose the global circuit
-   identifier of each inbound client circuit via the selected protocol. The only
+   identifier of each inbound client circuit. The only
    protocol supported right now \'haproxy'. This option is only for v3
    services. (Default: none) +
  +
@@ -2873,12 +3010,12 @@ The following options are used to configure a hidden service.
  +
       global_circuit_id = (0xAA << 24) + (0xBB << 16) + (0xCC << 8) + 0xDD; +
  +
-   In the case above, where the last 32-bit is 0xffffffff, the global circuit
+   In the case above, where the last 32-bits are 0xffffffff, the global circuit
    identifier would be 4294967295. You can use this value together with Tor's
-   control port where it is possible to terminate a circuit given the global
-   circuit identifier. For more information about this see controls-spec.txt. +
+   control port to terminate particular circuits using their global
+   circuit identifiers. For more information about this see control-spec.txt. +
  +
-   The HAProxy version 1 proxy protocol is described in detail at
+   The HAProxy version 1 protocol is described in detail at
    https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
 
 [[HiddenServiceMaxStreams]] **HiddenServiceMaxStreams** __N__::
@@ -2908,6 +3045,38 @@ The following options are used to configure a hidden service.
     Number of introduction points the hidden service will have. You can't
     have more than 10 for v2 service and 20 for v3. (Default: 3)
 
+[[HiddenServiceEnableIntroDoSDefense]] **HiddenServiceEnableIntroDoSDefense** **0**|**1**::
+    Enable DoS defense at the intropoint level. When this is enabled, the
+    rate and burst parameter (see below) will be sent to the intro point which
+    will then use them to apply rate limiting for introduction request to this
+    service.
+  +
+    The introduction point honors the consensus parameters except if this is
+    specifically set by the service operator using this option. The service
+    never looks at the consensus parameters in order to enable or disable this
+    defense. (Default: 0)
+
+[[HiddenServiceEnableIntroDoSRatePerSec]] **HiddenServiceEnableIntroDoSRatePerSec** __NUM__::
+    The allowed client introduction rate per second at the introduction
+    point. If this option is 0, it is considered infinite and thus if
+    **HiddenServiceEnableIntroDoSDefense** is set, it then effectively
+    disables the defenses. (Default: 25)
+
+[[HiddenServiceEnableIntroDoSBurstPerSec]] **HiddenServiceEnableIntroDoSBurstPerSec** __NUM__::
+    The allowed client introduction burst per second at the introduction
+    point. If this option is 0, it is considered infinite and thus if
+    **HiddenServiceEnableIntroDoSDefense** is set, it then effectively
+    disables the defenses. (Default: 200)
+
+
+**PER INSTANCE OPTIONS:**
+
+[[PublishHidServDescriptors]] **PublishHidServDescriptors** **0**|**1**::
+    If set to 0, Tor will run any hidden services you configure, but it won't
+    advertise them to the rendezvous directory. This option is only useful if
+    you're using a Tor controller that handles hidserv publishing for you.
+    (Default: 1)
+
 [[HiddenServiceSingleHopMode]] **HiddenServiceSingleHopMode** **0**|**1**::
     **Experimental - Non Anonymous** Hidden Services on a tor instance in
     HiddenServiceSingleHopMode make one-hop (direct) circuits between the onion
@@ -2947,31 +3116,42 @@ Client Authorization
 
 (Version 3 only)
 
-To configure client authorization on the service side, the
-"<HiddenServiceDir>/authorized_clients/" directory needs to exist. Each file
-in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the
-file name is irrelevant) and its content format MUST be:
+Service side:
+
+  To configure client authorization on the service side, the
+  "<HiddenServiceDir>/authorized_clients/" directory needs to exist. Each file
+  in that directory should be suffixed with ".auth" (i.e. "alice.auth"; the
+  file name is irrelevant) and its content format MUST be:
+
+          <auth-type>:<key-type>:<base32-encoded-public-key>
 
-        <auth-type>:<key-type>:<base32-encoded-public-key>
+  The supported <auth-type> are: "descriptor". The supported <key-type> are:
+  "x25519". The <base32-encoded-public-key> is the base32 representation of
+  the raw key bytes only (32 bytes for x25519).
 
-The supported <auth-type> are: "descriptor". The supported <key-type> are:
-"x25519". The <base32-encoded-public-key> is the base32 representation of
-the raw key bytes only (32 bytes for x25519).
+  Each file MUST contain one line only. Any malformed file will be
+  ignored. Client authorization will only be enabled for the service if tor
+  successfully loads at least one authorization file.
 
-Each file MUST contain one line only. Any malformed file will be
-ignored. Client authorization will only be enabled for the service if tor
-successfully loads at least one authorization file.
+  Note that once you've configured client authorization, anyone else with the
+  address won't be able to access it from this point on. If no authorization is
+  configured, the service will be accessible to anyone with the onion address.
 
-Note that once you've configured client authorization, anyone else with the
-address won't be able to access it from this point on. If no authorization is
-configured, the service will be accessible to anyone with the onion address.
+  Revoking a client can be done by removing their ".auth" file, however the
+  revocation will be in effect only after the tor process gets restarted even if
+  a SIGHUP takes place.
 
-Revoking a client can be done by removing their ".auth" file, however the
-revocation will be in effect only after the tor process gets restarted even if
-a SIGHUP takes place.
+Client side:
 
-See the Appendix G in the rend-spec-v3.txt file of
-https://spec.torproject.org/[torspec] for more information.
+  To access a v3 onion service with client authorization as a client, make sure
+  you have ClientOnionAuthDir set in your torrc. Then, in the
+  <ClientOnionAuthDir> directory, create an .auth_private file for the onion
+  service corresponding to this key (i.e. 'bob_onion.auth_private').  The
+  contents of the <ClientOnionAuthDir>/<user>.auth_private file should look like:
+
+      <56-char-onion-addr-without-.onion-part>:descriptor:x25519:<x25519 private key in base32>
+
+For more information, please see https://2019.www.torproject.org/docs/tor-onion-service.html.en#ClientAuthorization .
 
 TESTING NETWORK OPTIONS
 -----------------------
@@ -3260,10 +3440,6 @@ __CacheDirectory__**/cached-microdescs** and **cached-microdescs.new**::
     router. The ".new" file is an append-only journal; when it gets too
     large, all entries are merged into a new cached-microdescs file.
 
-__CacheDirectory__**/cached-routers** and **cached-routers.new**::
-    Obsolete versions of cached-descriptors and cached-descriptors.new. When
-    Tor can't find the newer files, it looks here instead.
-
 __DataDirectory__**/state**::
     A set of persistent key-value mappings. These are documented in
     the file. These include:
diff --git a/scripts/coccinelle/ctrl-reply-cleanup.cocci b/scripts/coccinelle/ctrl-reply-cleanup.cocci
new file mode 100644
index 0000000000..f085cd4684
--- /dev/null
+++ b/scripts/coccinelle/ctrl-reply-cleanup.cocci
@@ -0,0 +1,43 @@
+// Script to clean up after ctrl-reply.cocci -- run as a separate step
+// because cleanup_write2 (even when disabled) somehow prevents the
+// match rule in ctrl-reply.cocci from matching.
+
+// If it doesn't have to be a printf, turn it into a write
+
+@ cleanup_write @
+expression E;
+constant code, s;
+@@
+-control_printf_endreply(E, code, s)
++control_write_endreply(E, code, s)
+
+// Use send_control_done() instead of explicitly writing it out
+@ cleanup_send_done @
+type T;
+identifier f != send_control_done;
+expression E;
+@@
+ T f(...) {
+<...
+-control_write_endreply(E, 250, "OK")
++send_control_done(E)
+ ...>
+ }
+
+// Clean up more printfs that could be writes
+//
+// For some reason, including this rule, even disabled, causes the
+// match rule in ctrl-reply.cocci to fail to match some code that has
+// %s in its format strings
+
+@ cleanup_write2 @
+expression E1, E2;
+constant code;
+@@
+(
+-control_printf_endreply(E1, code, "%s", E2)
++control_write_endreply(E1, code, E2)
+|
+-control_printf_midreply(E1, code, "%s", E2)
++control_write_midreply(E1, code, E2)
+)
diff --git a/scripts/coccinelle/ctrl-reply.cocci b/scripts/coccinelle/ctrl-reply.cocci
new file mode 100644
index 0000000000..d6e9aeedd7
--- /dev/null
+++ b/scripts/coccinelle/ctrl-reply.cocci
@@ -0,0 +1,87 @@
+// Script to edit control_*.c for refactored control reply output functions
+
+@ initialize:python @
+@@
+import re
+from coccilib.report import *
+
+# reply strings "NNN-foo", "NNN+foo", "NNN foo", etc.
+r = re.compile(r'^"(\d+)([ +-])(.*)\\r\\n"$')
+
+# Generate name of function to call based on which separator character
+# comes between the numeric code and the text
+def idname(sep, base):
+    if sep == '+':
+        return base + "datareply"
+    elif sep == '-':
+        return base + "midreply"
+    else:
+        return base + "endreply"
+
+# Generate the actual replacements used by the rules
+def gen(s, base, p):
+    pos = p[0]
+    print_report(pos, "%s %s" % (base, s))
+    m = r.match(s)
+    if m is None:
+        # String not correct format, so fail match
+        cocci.include_match(False)
+        print_report(pos, "BAD STRING %s" % s)
+        return
+
+    code, sep, s1 = m.groups()
+
+    if r'\r\n' in s1:
+        # Extra CRLF in string, so fail match
+        cocci.include_match(False)
+        print_report(pos, "extra CRLF in string %s" % s)
+        return
+
+    coccinelle.code = code
+    # Need a string that is a single C token, because Coccinelle only allows
+    # "identifiers" to be output from Python scripts?
+    coccinelle.body = '"%s"' % s1
+    coccinelle.id = idname(sep, base)
+    return
+
+@ match @
+identifier f;
+position p;
+expression E;
+constant s;
+@@
+(
+ connection_printf_to_buf@f@p(E, s, ...)
+|
+ connection_write_str_to_buf@f@p(s, E)
+)
+
+@ script:python sc1 @
+s << match.s;
+p << match.p;
+f << match.f;
+id;
+body;
+code;
+@@
+if f == 'connection_printf_to_buf':
+    gen(s, 'control_printf_', p)
+elif f == 'connection_write_str_to_buf':
+    gen(s, 'control_write_', p)
+else:
+    raise(ValueError("%s: %s" % (f, s)))
+
+@ replace @
+constant match.s;
+expression match.E;
+identifier match.f;
+identifier sc1.body, sc1.id, sc1.code;
+@@
+(
+-connection_write_str_to_buf@f(s, E)
++id(E, code, body)
+|
+-connection_printf_to_buf@f(E, s
++id(E, code, body
+ , ...)
+)
diff --git a/scripts/coccinelle/debugmm.cocci b/scripts/coccinelle/debugmm.cocci
new file mode 100644
index 0000000000..dbd308df33
--- /dev/null
+++ b/scripts/coccinelle/debugmm.cocci
@@ -0,0 +1,29 @@
+// Look for use of expressions with side-effects inside of debug logs.
+//
+// This script detects expressions like ++E, --E, E++, and E-- inside of
+// calls to log_debug().
+//
+// The log_debug() macro exits early if debug logging is not enabled,
+// potentially causing problems if its arguments have side-effects.
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  --E ...+>, ... );
+
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  ++E ...+>, ... );
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  E-- ...+>, ... );
+
+
+@@
+expression E;
+@@
+*log_debug(... , <+...  E++ ...+>, ... );
diff --git a/scripts/coccinelle/tor-coccinelle.h b/scripts/coccinelle/tor-coccinelle.h
new file mode 100644
index 0000000000..8f625dcee4
--- /dev/null
+++ b/scripts/coccinelle/tor-coccinelle.h
@@ -0,0 +1,3 @@
+#define MOCK_IMPL(a, b, c) a b c
+#define CHECK_PRINTF(a, b)
+#define STATIC static
diff --git a/scripts/codegen/fuzzing_include_am.py b/scripts/codegen/fuzzing_include_am.py
index 3c948d87cf..a944584453 100755
--- a/scripts/codegen/fuzzing_include_am.py
+++ b/scripts/codegen/fuzzing_include_am.py
@@ -13,6 +13,7 @@ FUZZERS = """
 	iptsv2
 	microdesc
 	socks
+	strops
 	vrs
 """
 
@@ -23,12 +24,12 @@ FUZZING_CPPFLAGS = \
 FUZZING_CFLAGS = \
 	$(AM_CFLAGS) $(TEST_CFLAGS)
 FUZZING_LDFLAG = \
-	@TOR_LDFLAGS_zlib@ @TOR_LDFLAGS_openssl@ @TOR_LDFLAGS_libevent@
+	@TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
 FUZZING_LIBS = \
 	$(TOR_INTERNAL_TESTING_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	@TOR_LIBEVENT_LIBS@ @TOR_OPENSSL_LIBS@ \
+	@TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
 	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
 	@TOR_SYSTEMD_LIBS@ \
 	@TOR_LZMA_LIBS@ \
diff --git a/scripts/codegen/run_trunnel.sh b/scripts/codegen/run_trunnel.sh
index 428804342a..645b3c2158 100755
--- a/scripts/codegen/run_trunnel.sh
+++ b/scripts/codegen/run_trunnel.sh
@@ -9,9 +9,7 @@ OPTIONS="--require-version=1.5.1"
 
 # Get all .trunnel files recursively from that directory so we can support
 # multiple sub-directories.
-for file in `find ./src/trunnel/ -name '*.trunnel'`; do
-  python -m trunnel ${OPTIONS} $file
-done
+find ./src/trunnel/ -name '*.trunnel' -exec python -m trunnel ${OPTIONS} {} \;
 
 python -m trunnel ${OPTIONS} --write-c-files --target-dir=./src/ext/trunnel/
 
diff --git a/scripts/git/git-merge-forward.sh b/scripts/git/git-merge-forward.sh
new file mode 100755
index 0000000000..bdd0da5b75
--- /dev/null
+++ b/scripts/git/git-merge-forward.sh
@@ -0,0 +1,428 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+
+function usage()
+{
+  echo "$SCRIPT_NAME [-h] [-n] [-t <test-branch-prefix> [-u]]"
+  echo
+  echo "  arguments:"
+  echo "   -h: show this help text"
+  echo "   -n: dry run mode"
+  echo "       (default: run commands)"
+  echo "   -t: test branch mode: create new branches from the commits checked"
+  echo "       out in each maint directory. Call these branches prefix_029,"
+  echo "       prefix_035, ... , prefix_master."
+  echo "       (default: merge forward maint-*, release-*, and master)"
+  echo "   -u: in test branch mode, if a prefix_* branch already exists,"
+  echo "       skip creating that branch. Use after a merge error, to"
+  echo "       restart the merge forward at the first unmerged branch."
+  echo "       (default: if a prefix_* branch already exists, fail and exit)"
+  echo
+  echo " env vars:"
+  echo "   required:"
+  echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
+  echo "       You must set this env var, we recommend \$HOME/git/"
+  echo "       (default: fail if this env var is not set;"
+  echo "       current: $GIT_PATH)"
+  echo
+  echo "   optional:"
+  echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
+  echo "       The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
+  echo "       (default: tor; current: $TOR_MASTER_NAME)"
+  echo "   TOR_WKT_NAME: the name of the directory containing the tor"
+  echo "       worktrees. The tor worktrees are:"
+  echo "       \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
+  echo "       (default: tor-wkt; current: $TOR_WKT_NAME)"
+  echo "   we recommend that you set these env vars in your ~/.profile"
+}
+
+#################
+# Configuration #
+#################
+
+# Don't change this configuration - set the env vars in your .profile
+
+# Where are all those git repositories?
+GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
+# The tor master git repository directory from which all the worktree have
+# been created.
+TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
+# The worktrees location (directory).
+TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
+
+##########################
+# Git branches to manage #
+##########################
+
+# The branches and worktrees need to be modified when there is a new branch,
+# and when an old branch is no longer supported.
+
+# Configuration of the branches that needs merging. The values are in order:
+#   (0) current maint/release branch name
+#   (1) previous maint/release name to merge into (0)
+#         (only used in merge forward mode)
+#   (2) Full path of the git worktree
+#   (3) current branch suffix
+#         (maint branches only, only used in test branch mode)
+#   (4) previous test branch suffix to merge into (3)
+#         (maint branches only, only used in test branch mode)
+#
+# Merge forward example:
+#   $ cd <PATH/TO/WORKTREE> (2)
+#   $ git checkout maint-0.3.5 (0)
+#   $ git pull
+#   $ git merge maint-0.3.4 (1)
+#
+# Test branch example:
+#   $ cd <PATH/TO/WORKTREE> (2)
+#   $ git checkout -b ticket99999_035 (3)
+#   $ git checkout maint-0.3.5 (0)
+#   $ git pull
+#   $ git checkout ticket99999_035
+#   $ git merge maint-0.3.5
+#   $ git merge ticket99999_034 (4)
+#
+# First set of arrays are the maint-* branch and then the release-* branch.
+# New arrays need to be in the WORKTREE= array else they aren't considered.
+#
+# Only used in test branch mode
+# There is no previous branch to merge forward, so the second and fifth items
+# must be blank ("")
+MAINT_029_TB=( "maint-0.2.9" "" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" \
+    "_029" "")
+# Used in maint/release merge and test branch modes
+MAINT_035=( "maint-0.3.5" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" \
+    "_035" "_029")
+MAINT_040=( "maint-0.4.0" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" \
+    "_040" "_035")
+MAINT_041=( "maint-0.4.1" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" \
+    "_041" "_040")
+MAINT_MASTER=( "master" "maint-0.4.1" "$GIT_PATH/$TOR_MASTER_NAME" \
+    "_master" "_041")
+
+RELEASE_029=( "release-0.2.9" "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" )
+RELEASE_035=( "release-0.3.5" "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" )
+RELEASE_040=( "release-0.4.0" "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" )
+RELEASE_041=( "release-0.4.1" "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" )
+
+# The master branch path has to be the main repository thus contains the
+# origin that will be used to fetch the updates. All the worktrees are created
+# from that repository.
+ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
+
+# SC2034 -- shellcheck thinks that these are unused.  We know better.
+ACTUALLY_THESE_ARE_USED=<<EOF
+${MAINT_029_TB[0]}
+${MAINT_035[0]}
+${MAINT_040[0]}
+${MAINT_041[0]}
+${MAINT_MASTER[0]}
+${RELEASE_029[0]}
+${RELEASE_035[0]}
+${RELEASE_040[0]}
+${RELEASE_041[0]}
+EOF
+
+#######################
+# Argument processing #
+#######################
+
+# Controlled by the -n option. The dry run option will just output the command
+# that would have been executed for each worktree.
+DRY_RUN=0
+
+# Controlled by the -t <test-branch-prefix> option. The test branch base
+# name option makes git-merge-forward.sh create new test branches:
+# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward.
+TEST_BRANCH_PREFIX=
+
+# Controlled by the -u option. The use existing option checks for existing
+# branches with the <test-branch-prefix>, and checks them out, rather than
+# creating a new branch.
+USE_EXISTING=0
+
+while getopts "hnt:u" opt; do
+  case "$opt" in
+    h) usage
+       exit 0
+       ;;
+    n) DRY_RUN=1
+       echo "    *** DRY RUN MODE ***"
+       ;;
+    t) TEST_BRANCH_PREFIX="$OPTARG"
+       echo "    *** CREATING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***"
+       ;;
+    u) USE_EXISTING=1
+       echo "    *** USE EXISTING TEST BRANCHES MODE ***"
+       ;;
+    *)
+       echo
+       usage
+       exit 1
+       ;;
+  esac
+done
+
+###########################
+# Git worktrees to manage #
+###########################
+
+if [ -z "$TEST_BRANCH_PREFIX" ]; then
+
+  # maint/release merge mode
+  #
+  # List of all worktrees to work on. All defined above. Ordering is important.
+  # Always the maint-* branch BEFORE then the release-*.
+  WORKTREE=(
+    RELEASE_029[@]
+
+    MAINT_035[@]
+    RELEASE_035[@]
+
+    MAINT_040[@]
+    RELEASE_040[@]
+
+    MAINT_041[@]
+    RELEASE_041[@]
+
+    MAINT_MASTER[@]
+  )
+
+else
+
+  # Test branch mode: merge to maint only, and create a new branch for 0.2.9
+  WORKTREE=(
+    MAINT_029_TB[@]
+
+    MAINT_035[@]
+
+    MAINT_040[@]
+
+    MAINT_041[@]
+
+    MAINT_MASTER[@]
+  )
+
+fi
+
+COUNT=${#WORKTREE[@]}
+
+#############
+# Constants #
+#############
+
+# Control characters
+CNRM=$'\x1b[0;0m'   # Clear color
+
+# Bright color
+BGRN=$'\x1b[1;32m'
+BBLU=$'\x1b[1;34m'
+BRED=$'\x1b[1;31m'
+BYEL=$'\x1b[1;33m'
+IWTH=$'\x1b[3;37m'
+
+# Strings for the pretty print.
+MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
+SUCCESS="${BGRN}success${CNRM}"
+FAILED="${BRED}failed${CNRM}"
+
+####################
+# Helper functions #
+####################
+
+# Validate the given returned value (error code), print success or failed. The
+# second argument is the error output in case of failure, it is printed out.
+# On failure, this function exits.
+function validate_ret
+{
+  if [ "$1" -eq 0 ]; then
+    printf "%s\\n" "$SUCCESS"
+  else
+    printf "%s\\n" "$FAILED"
+    printf "    %s" "$2"
+    exit 1
+  fi
+}
+
+# Switch to the given branch name.
+function switch_branch
+{
+  local cmd="git checkout '$1'"
+  printf "  %s Switching branch to %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Checkout a new branch with the given branch name.
+function new_branch
+{
+  local cmd="git checkout -b '$1'"
+  printf "  %s Creating new branch %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Switch to an existing branch, or checkout a new branch with the given
+# branch name.
+function switch_or_new_branch
+{
+  local cmd="git rev-parse --verify '$1'"
+  if [ $DRY_RUN -eq 0 ]; then
+    # Call switch_branch if there is a branch, or new_branch if there is not
+    msg=$( eval "$cmd" 2>&1 )
+    RET=$?
+    if [ $RET -eq 0 ]; then
+      # Branch: (commit id)
+      switch_branch "$1"
+    elif [ $RET -eq 128 ]; then
+      # Not a branch: "fatal: Needed a single revision"
+      new_branch "$1"
+    else
+      # Unexpected return value
+      validate_ret $RET "$msg"
+    fi
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:"
+    switch_branch "$1"
+    new_branch "$1"
+  fi
+}
+
+# Pull the given branch name.
+function pull_branch
+{
+  local cmd="git pull"
+  printf "  %s Pulling branch %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Merge the given branch name ($1) into the current branch ($2).
+function merge_branch
+{
+  local cmd="git merge --no-edit '$1'"
+  printf "  %s Merging branch %s into %s..." "$MARKER" "$1" "$2"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Pull the given branch name.
+function merge_branch_origin
+{
+  local cmd="git merge --ff-only 'origin/$1'"
+  printf "  %s Merging branch origin/%s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Go into the worktree repository.
+function goto_repo
+{
+  if [ ! -d "$1" ]; then
+    echo "  $1: Not found. Stopping."
+    exit 1
+  fi
+  cd "$1" || exit
+}
+
+# Fetch the origin. No arguments.
+function fetch_origin
+{
+  local cmd="git fetch origin"
+  printf "  %s Fetching origin..." "$MARKER"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+###############
+# Entry point #
+###############
+
+# First, fetch the origin.
+goto_repo "$ORIGIN_PATH"
+fetch_origin
+
+# Go over all configured worktree.
+for ((i=0; i<COUNT; i++)); do
+  current=${!WORKTREE[$i]:0:1}
+  previous=${!WORKTREE[$i]:1:1}
+  repo_path=${!WORKTREE[$i]:2:1}
+  # default to merge forward mode
+  test_current=
+  test_previous=
+  target_current="$current"
+  target_previous="$previous"
+  if [ "$TEST_BRANCH_PREFIX" ]; then
+    test_current_suffix=${!WORKTREE[$i]:3:1}
+    test_current=${TEST_BRANCH_PREFIX}${test_current_suffix}
+    # the current test branch, if present, or maint/release branch, if not
+    target_current="$test_current"
+    test_previous_suffix=${!WORKTREE[$i]:4:1}
+    if [ "$test_previous_suffix" ]; then
+      test_previous=${TEST_BRANCH_PREFIX}${test_previous_suffix}
+      # the previous test branch, if present, or maint/release branch, if not
+      target_previous="$test_previous"
+    fi
+  fi
+
+  printf "%s Handling branch \\n" "$MARKER" "${BYEL}$target_current${CNRM}"
+
+  # Go into the worktree to start merging.
+  goto_repo "$repo_path"
+  if [ "$test_current" ]; then
+    if [ $USE_EXISTING -eq 0 ]; then
+      # Create a test branch from the currently checked-out branch/commit
+      # Fail if it already exists
+      new_branch "$test_current"
+    else
+      # Switch if it exists, or create if it does not
+      switch_or_new_branch "$test_current"
+    fi
+  fi
+  # Checkout the current maint/release branch
+  switch_branch "$current"
+  # Update the current maint/release branch with an origin merge to get the
+  # latest updates
+  merge_branch_origin "$current"
+  if [ "$test_current" ]; then
+    # Checkout the test branch
+    switch_branch "$test_current"
+    # Merge the updated maint branch into the test branch
+    merge_branch "$current" "$test_current"
+  fi
+  # Merge the previous branch into the target branch
+  # Merge Forward Example:
+  #   merge maint-0.2.9 into maint-0.3.5.
+  # Test Branch Example:
+  #   merge bug99999_029 into bug99999_035.
+  # Skip the merge if the previous branch does not exist
+  # (there's nothing to merge forward into the oldest test branch)
+  if [ "$target_previous" ]; then
+    merge_branch "$target_previous" "$target_current"
+  fi
+done
diff --git a/scripts/git/git-pull-all.sh b/scripts/git/git-pull-all.sh
new file mode 100755
index 0000000000..dc16066388
--- /dev/null
+++ b/scripts/git/git-pull-all.sh
@@ -0,0 +1,260 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+
+function usage()
+{
+  echo "$SCRIPT_NAME [-h] [-n]"
+  echo
+  echo "  arguments:"
+  echo "   -h: show this help text"
+  echo "   -n: dry run mode"
+  echo "       (default: run commands)"
+  echo
+  echo " env vars:"
+  echo "   required:"
+  echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
+  echo "       You must set this env var, we recommend \$HOME/git/"
+  echo "       (default: fail if this env var is not set;"
+  echo "       current: $GIT_PATH)"
+  echo
+  echo "   optional:"
+  echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
+  echo "       The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
+  echo "       (default: tor; current: $TOR_MASTER_NAME)"
+  echo "   TOR_WKT_NAME: the name of the directory containing the tor"
+  echo "       worktrees. The tor worktrees are:"
+  echo "       \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
+  echo "       (default: tor-wkt; current: $TOR_WKT_NAME)"
+  echo "   we recommend that you set these env vars in your ~/.profile"
+}
+
+#################
+# Configuration #
+#################
+
+# Don't change this configuration - set the env vars in your .profile
+
+# Where are all those git repositories?
+GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
+# The tor master git repository directory from which all the worktree have
+# been created.
+TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
+# The worktrees location (directory).
+TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
+
+##########################
+# Git branches to manage #
+##########################
+
+# Configuration of the branches that need pulling. The values are in order:
+#   (1) Branch name to pull (update).
+#   (2) Full path of the git worktree.
+#
+# As an example:
+#   $ cd <PATH/TO/WORKTREE> (3)
+#   $ git checkout maint-0.3.5 (1)
+#   $ git pull
+#
+# First set of arrays are the maint-* branch and then the release-* branch.
+# New arrays need to be in the WORKTREE= array else they aren't considered.
+MAINT_029=( "maint-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/maint-0.2.9" )
+MAINT_035=( "maint-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/maint-0.3.5" )
+MAINT_040=( "maint-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.0" )
+MAINT_041=( "maint-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/maint-0.4.1" )
+MAINT_MASTER=( "master" "$GIT_PATH/$TOR_MASTER_NAME" )
+
+RELEASE_029=( "release-0.2.9" "$GIT_PATH/$TOR_WKT_NAME/release-0.2.9" )
+RELEASE_035=( "release-0.3.5" "$GIT_PATH/$TOR_WKT_NAME/release-0.3.5" )
+RELEASE_040=( "release-0.4.0" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.0" )
+RELEASE_041=( "release-0.4.1" "$GIT_PATH/$TOR_WKT_NAME/release-0.4.1" )
+
+# The master branch path has to be the main repository thus contains the
+# origin that will be used to fetch the updates. All the worktrees are created
+# from that repository.
+ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
+
+# SC2034 -- shellcheck thinks that these are unused.  We know better.
+ACTUALLY_THESE_ARE_USED=<<EOF
+${MAINT_029[0]}
+${MAINT_035[0]}
+${MAINT_040[0]}
+${MAINT_041[0]}
+${MAINT_MASTER[0]}
+${RELEASE_029[0]}
+${RELEASE_035[0]}
+${RELEASE_040[0]}
+${RELEASE_041[0]}
+EOF
+
+###########################
+# Git worktrees to manage #
+###########################
+
+# List of all worktrees to work on. All defined above. Ordering is important.
+# Always the maint-* branch first then the release-*.
+WORKTREE=(
+  MAINT_029[@]
+  RELEASE_029[@]
+
+  MAINT_035[@]
+  RELEASE_035[@]
+
+  MAINT_040[@]
+  RELEASE_040[@]
+
+  MAINT_041[@]
+  RELEASE_041[@]
+
+  MAINT_MASTER[@]
+)
+COUNT=${#WORKTREE[@]}
+
+#######################
+# Argument processing #
+#######################
+
+# Controlled by the -n option. The dry run option will just output the command
+# that would have been executed for each worktree.
+DRY_RUN=0
+
+while getopts "hn" opt; do
+  case "$opt" in
+    h) usage
+       exit 0
+       ;;
+    n) DRY_RUN=1
+       echo "    *** DRY DRUN MODE ***"
+       ;;
+    *)
+       echo
+       usage
+       exit 1
+       ;;
+  esac
+done
+
+#############
+# Constants #
+#############
+
+# Control characters
+CNRM=$'\x1b[0;0m'   # Clear color
+
+# Bright color
+BGRN=$'\x1b[1;32m'
+BBLU=$'\x1b[1;34m'
+BRED=$'\x1b[1;31m'
+BYEL=$'\x1b[1;33m'
+IWTH=$'\x1b[3;37m'
+
+# Strings for the pretty print.
+MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
+SUCCESS="${BGRN}ok${CNRM}"
+FAILED="${BRED}failed${CNRM}"
+
+####################
+# Helper functions #
+####################
+
+# Validate the given returned value (error code), print success or failed. The
+# second argument is the error output in case of failure, it is printed out.
+# On failure, this function exits.
+function validate_ret
+{
+  if [ "$1" -eq 0 ]; then
+    printf "%s\\n" "$SUCCESS"
+  else
+    printf "%s\\n" "$FAILED"
+    printf "    %s" "$2"
+    exit 1
+  fi
+}
+
+# Switch to the given branch name.
+function switch_branch
+{
+  local cmd="git checkout $1"
+  printf "  %s Switching branch to %s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Pull the given branch name.
+function merge_branch
+{
+  local cmd="git merge --ff-only origin/$1"
+  printf "  %s Merging branch origin/%s..." "$MARKER" "$1"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Go into the worktree repository.
+function goto_repo
+{
+  if [ ! -d "$1" ]; then
+    echo "  $1: Not found. Stopping."
+    exit 1
+  fi
+  cd "$1" || exit
+}
+
+# Fetch the origin. No arguments.
+function fetch_origin
+{
+  local cmd="git fetch origin"
+  printf "  %s Fetching origin..." "$MARKER"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+# Fetch tor-github pull requests. No arguments.
+function fetch_tor_github
+{
+  local cmd="git fetch tor-github"
+  printf "  %s Fetching tor-github..." "$MARKER"
+  if [ $DRY_RUN -eq 0 ]; then
+    msg=$( eval "$cmd" 2>&1 )
+    validate_ret $? "$msg"
+  else
+    printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
+  fi
+}
+
+###############
+# Entry point #
+###############
+
+# First, fetch tor-github.
+goto_repo "$ORIGIN_PATH"
+fetch_tor_github
+
+# Then, fetch the origin.
+fetch_origin
+
+# Go over all configured worktree.
+for ((i=0; i<COUNT; i++)); do
+  current=${!WORKTREE[$i]:0:1}
+  repo_path=${!WORKTREE[$i]:1:1}
+
+  printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$current${CNRM}"
+
+  # Go into the worktree to start merging.
+  goto_repo "$repo_path"
+  # Checkout the current branch
+  switch_branch "$current"
+  # Update the current branch by merging the origin to get the latest.
+  merge_branch "$current"
+done
diff --git a/scripts/git/git-push-all.sh b/scripts/git/git-push-all.sh
new file mode 100755
index 0000000000..7c43fe24d8
--- /dev/null
+++ b/scripts/git/git-push-all.sh
@@ -0,0 +1,292 @@
+#!/usr/bin/env bash
+
+SCRIPT_NAME=$(basename "$0")
+
+function usage()
+{
+  if [ "$TOR_PUSH_SAME" ]; then
+    CURRENT_PUSH_SAME="push"
+  else
+    CURRENT_PUSH_SAME="skip"
+  fi
+
+  echo "$SCRIPT_NAME [-h] [-r <remote-name> [-t <test-branch-prefix>]] [-s]"
+  # The next line looks misaligned, but it lines up in the output
+  echo "                [-- [-n] [--no-atomic] <git push options>]"
+  echo
+  echo "  arguments:"
+  echo "   -h: show this help text"
+  echo "   -n: dry run mode"
+  echo "       (default: run commands)"
+  echo "   -r: push to remote-name, rather than the default upstream remote."
+  echo "       (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)"
+  echo "   -t: test branch mode: push test branches to remote-name. Pushes"
+  echo "       branches prefix_029, prefix_035, ... , prefix_master."
+  echo "       (default: push maint-*, release-*, and master)"
+  echo "   -s: push branches whose tips match upstream maint, release, or"
+  echo "       master branches. The default is to skip these branches,"
+  echo "       because they do not contain any new code. Use -s to test for"
+  echo "       CI environment failures, using code that previously passed CI."
+  echo "       (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
+  echo "   --: pass further arguments to git push."
+  echo "       (default: git push --atomic, current: $GIT_PUSH)"
+  echo
+  echo " env vars:"
+  echo "   optional:"
+  echo "   TOR_GIT_PUSH_PATH: change to this directory before pushing."
+  echo "       (default: if \$TOR_FULL_GIT_PATH is set,"
+  echo "       use \$TOR_FULL_GIT_PATH/\$TOR_MASTER;"
+  echo "       Otherwise, use the current directory for pushes;"
+  echo "       current: $TOR_GIT_PUSH_PATH)"
+  echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
+  echo "       We recommend using \$HOME/git/."
+  echo "       (default: use the current directory for pushes;"
+  echo "       current: $TOR_FULL_GIT_PATH)"
+  echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
+  echo "       The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
+  echo "       (default: tor; current: $TOR_MASTER_NAME)"
+  echo
+  echo "   TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
+  echo "       Overridden by -r."
+  echo "       (default: upstream; current: $UPSTREAM_REMOTE)"
+  echo "   TOR_GIT_PUSH: the git push command and default arguments."
+  echo "       Overridden by <git push options> after --."
+  echo "       (default: git push --atomic; current: $GIT_PUSH)"
+  echo "   TOR_PUSH_SAME: push branches whose tips match upstream maint,"
+  echo "       release, or master branches. Inverted by -s."
+  echo "       (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
+  echo "   TOR_PUSH_DELAY: pushes the master and maint branches separately,"
+  echo "       so that CI runs in a sensible order."
+  echo "       (default: push all branches immediately; current: $PUSH_DELAY)"
+  echo "   we recommend that you set these env vars in your ~/.profile"
+}
+
+set -e
+
+#################
+# Configuration #
+#################
+
+# Don't change this configuration - set the env vars in your .profile
+#
+# The tor master git repository directory from which all the worktree have
+# been created.
+TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
+# Which directory do we push from?
+if [ "$TOR_FULL_GIT_PATH" ]; then
+  TOR_GIT_PUSH_PATH=${TOR_GIT_PUSH_PATH:-"$TOR_FULL_GIT_PATH/$TOR_MASTER_NAME"}
+fi
+# git push command and default arguments
+GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"}
+# The upstream remote which git.torproject.org/tor.git points to.
+DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
+# Push to a different upstream remote using -r <remote-name>
+UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE}
+# Add a delay between pushes, so CI runs on the most important branches first
+PUSH_DELAY=${TOR_PUSH_DELAY:-0}
+# Push (1) or skip (0) test branches that are the same as an upstream
+# maint/master branch. Push if you are testing that the CI environment still
+# works on old code, skip if you are testing new code in the branch.
+# Default: skip unchanged branches.
+# Inverted by the -s option.
+PUSH_SAME=${TOR_PUSH_SAME:-0}
+
+#######################
+# Argument processing #
+#######################
+
+# Controlled by the -t <test-branch-prefix> option. The test branch base
+# name option makes git-merge-forward.sh create new test branches:
+# <tbbn>_029, <tbbn>_035, ... , <tbbn>_master, and merge forward.
+TEST_BRANCH_PREFIX=
+
+while getopts ":hr:st:" opt; do
+  case "$opt" in
+    h) usage
+       exit 0
+       ;;
+    r) UPSTREAM_REMOTE="$OPTARG"
+       echo "    *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***"
+       shift
+       shift
+       OPTIND=$((OPTIND - 2))
+       ;;
+    s) PUSH_SAME=$((! PUSH_SAME))
+       if [ "$PUSH_SAME" -eq 0 ]; then
+         echo "    *** SKIPPING UNCHANGED TEST BRANCHES ***"
+       else
+         echo "    *** PUSHING UNCHANGED TEST BRANCHES ***"
+       fi
+       shift
+       OPTIND=$((OPTIND - 1))
+       ;;
+    t) TEST_BRANCH_PREFIX="$OPTARG"
+       echo "    *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***"
+       shift
+       shift
+       OPTIND=$((OPTIND - 2))
+       ;;
+    *)
+       # Assume we're done with script arguments,
+       # and git push will handle the option
+       break
+       ;;
+  esac
+done
+
+# getopts doesn't allow "-" as an option character,
+# so we have to handle -- manually
+if [ "$1" = "--" ]; then
+  shift
+fi
+
+if [ "$TEST_BRANCH_PREFIX" ]; then
+  if [ "$UPSTREAM_REMOTE" = "$DEFAULT_UPSTREAM_REMOTE" ]; then
+    echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \
+      "the default remote $DEFAULT_UPSTREAM_REMOTE is not allowed."
+    echo
+    usage
+    exit 1
+  fi
+fi
+
+if [ "$TOR_GIT_PUSH_PATH" ]; then
+  echo "Changing to $GIT_PUSH_PATH before pushing"
+  cd "$TOR_GIT_PUSH_PATH"
+else
+  echo "Pushing from the current directory"
+fi
+
+echo "Calling $GIT_PUSH" "$@" "<branches>"
+
+################################
+# Git upstream remote branches #
+################################
+
+DEFAULT_UPSTREAM_BRANCHES=
+if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then
+  DEFAULT_UPSTREAM_BRANCHES=$(echo \
+    "$DEFAULT_UPSTREAM_REMOTE"/master \
+    "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.1 \
+    "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.4.0 \
+    "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.3.5 \
+    "$DEFAULT_UPSTREAM_REMOTE"/{release,maint}-0.2.9 \
+    )
+fi
+
+UPSTREAM_BRANCHES=$(echo \
+  "$UPSTREAM_REMOTE"/master \
+  "$UPSTREAM_REMOTE"/{release,maint}-0.4.1 \
+  "$UPSTREAM_REMOTE"/{release,maint}-0.4.0 \
+  "$UPSTREAM_REMOTE"/{release,maint}-0.3.5 \
+  "$UPSTREAM_REMOTE"/{release,maint}-0.2.9 \
+  )
+
+########################
+# Git branches to push #
+########################
+
+PUSH_BRANCHES=$(echo \
+  master \
+  {release,maint}-0.4.1 \
+  {release,maint}-0.4.0 \
+  {release,maint}-0.3.5 \
+  {release,maint}-0.2.9 \
+  )
+
+if [ -z "$TEST_BRANCH_PREFIX" ]; then
+
+  # maint/release push mode
+  #
+  # List of branches to push. Ordering is not important.
+  PUSH_BRANCHES=$(echo \
+    master \
+    {release,maint}-0.4.1 \
+    {release,maint}-0.4.0 \
+    {release,maint}-0.3.5 \
+    {release,maint}-0.2.9 \
+    )
+else
+
+  # Test branch mode: merge to maint only, and create a new branch for 0.2.9
+  #
+  # List of branches to push. Ordering is not important.
+  PUSH_BRANCHES=" \
+    ${TEST_BRANCH_PREFIX}_master \
+    ${TEST_BRANCH_PREFIX}_041 \
+    ${TEST_BRANCH_PREFIX}_040 \
+    ${TEST_BRANCH_PREFIX}_035 \
+    ${TEST_BRANCH_PREFIX}_029 \
+    "
+fi
+
+###############
+# Entry point #
+###############
+
+# Skip the test branches that are the same as the upstream branches
+if [ "$PUSH_SAME" -eq 0 ] && [ "$TEST_BRANCH_PREFIX" ]; then
+  NEW_PUSH_BRANCHES=
+  for b in $PUSH_BRANCHES; do
+    PUSH_COMMIT=$(git rev-parse "$b")
+    SKIP_UPSTREAM=
+    for u in $DEFAULT_UPSTREAM_BRANCHES $UPSTREAM_BRANCHES; do
+      UPSTREAM_COMMIT=$(git rev-parse "$u")
+      if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then
+        SKIP_UPSTREAM="$u"
+      fi
+    done
+    if [ "$SKIP_UPSTREAM" ]; then
+      printf "Skipping unchanged: %s remote: %s\\n" \
+        "$b" "$SKIP_UPSTREAM"
+    else
+      if [ "$NEW_PUSH_BRANCHES" ]; then
+        NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}"
+      else
+        NEW_PUSH_BRANCHES="${b}"
+      fi
+    fi
+  done
+  PUSH_BRANCHES=${NEW_PUSH_BRANCHES}
+fi
+
+if [ "$PUSH_DELAY" -le 0 ]; then
+  echo "Pushing $PUSH_BRANCHES"
+  # We know that there are no spaces in any branch within $PUSH_BRANCHES, so
+  # it is safe to use it unquoted.  (This also applies to the other shellcheck
+  # exceptions below.)
+  #
+  # Push all the branches at the same time
+  # shellcheck disable=SC2086
+  $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES
+else
+  # Push the branches in optimal CI order, with a delay between each push
+  PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V)
+  MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep master)
+  if [ -z "$TEST_BRANCH_PREFIX" ]; then
+    MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint)
+    RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \
+      tr "\\n" " ")
+    printf \
+      "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n%s\\n" \
+      "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES" "$RELEASE_BRANCHES"
+  else
+    # Actually test branches based on maint branches
+    MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v master)
+    printf "Pushing with %ss delays, so CI runs in this order:\\n%s\\n%s\\n" \
+      "$PUSH_DELAY" "$MASTER_BRANCH" "$MAINT_BRANCHES"
+    # No release branches
+    RELEASE_BRANCHES=
+  fi
+  $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$MASTER_BRANCH"
+  sleep "$PUSH_DELAY"
+  # shellcheck disable=SC2086
+  for b in $MAINT_BRANCHES; do
+    $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b"
+    sleep "$PUSH_DELAY"
+  done
+  if [ "$RELEASE_BRANCHES" ]; then
+    # shellcheck disable=SC2086
+    $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES
+  fi
+fi
diff --git a/scripts/git/post-merge.git-hook b/scripts/git/post-merge.git-hook
new file mode 100755
index 0000000000..eae4f999e7
--- /dev/null
+++ b/scripts/git/post-merge.git-hook
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# This is post-merge git hook script to check for changes in:
+# * git hook scripts
+# * helper scripts for using git efficiently.
+# If any changes are detected, a diff of them is printed.
+#
+# To install this script, copy it to .git/hooks/post-merge in local copy of
+# tor git repo and make sure it has permission to execute.
+
+git_toplevel=$(git rev-parse --show-toplevel)
+
+check_for_diffs() {
+        installed="$git_toplevel/.git/hooks/$1"
+        latest="$git_toplevel/scripts/git/$1.git-hook"
+
+        if [ -e "$installed" ]
+        then
+               if ! cmp "$installed" "$latest" >/dev/null 2>&1
+               then
+                        echo "ATTENTION: $1 hook has changed:"
+                        echo "==============================="
+                        diff -u "$installed" "$latest"
+               fi
+        fi
+}
+
+check_for_script_update() {
+        fullpath="$1"
+
+        if ! git diff ORIG_HEAD HEAD --exit-code -- "$fullpath" >/dev/null
+        then
+                echo "ATTENTION: $1 has changed:"
+                git --no-pager diff ORIG_HEAD HEAD -- "$fullpath"
+        fi
+}
+
+cur_branch=$(git rev-parse --abbrev-ref HEAD)
+if [ "$cur_branch" != "master" ]; then
+  echo "post-merge: Not a master branch. Skipping."
+  exit 0
+fi
+
+check_for_diffs "pre-push"
+check_for_diffs "pre-commit"
+check_for_diffs "post-merge"
+
+for file in "$git_toplevel"/scripts/git/* ; do
+        check_for_script_update "$file"
+done
+
diff --git a/scripts/git/pre-commit.git-hook b/scripts/git/pre-commit.git-hook
new file mode 100755
index 0000000000..1c381ec60a
--- /dev/null
+++ b/scripts/git/pre-commit.git-hook
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# To install this script, copy it to .git/hooks/pre-commit in local copy of
+# tor git repo and make sure it has permission to execute.
+#
+# This is pre-commit git hook script that prevents commiting your changeset if
+# it fails our code formatting, changelog entry formatting, module include
+# rules, or best practices tracker.
+
+workdir=$(git rev-parse --show-toplevel)
+
+cd "$workdir" || exit 1
+
+set -e
+
+if [ -n "$(ls ./changes/)" ]; then
+    python scripts/maint/lintChanges.py ./changes/*
+fi
+
+if [ -d src/lib ]; then
+    # This is the layout in 0.3.5
+    perl scripts/maint/checkSpace.pl -C \
+         src/lib/*/*.[ch] \
+         src/core/*/*.[ch] \
+         src/feature/*/*.[ch] \
+         src/app/*/*.[ch] \
+         src/test/*.[ch] \
+         src/test/*/*.[ch] \
+         src/tools/*.[ch]
+elif [ -d src/common ]; then
+    # This was the layout before 0.3.5
+    perl scripts/maint/checkSpace.pl -C \
+         src/common/*/*.[ch] \
+         src/or/*/*.[ch] \
+         src/test/*.[ch] \
+         src/test/*/*.[ch] \
+         src/tools/*.[ch]
+fi
+
+if test -e scripts/maint/practracker/includes.py; then
+    python scripts/maint/practracker/includes.py
+fi
+
+# Only call practracker if ${PT_DIR}/.enable_practracker_in_hooks exists
+# We do this check so that we can enable practracker in hooks in master, and
+# disable it on maint branches
+PT_DIR=scripts/maint/practracker
+
+if [ -e "${PT_DIR}/practracker.py" ]; then
+    if [ -e "${PT_DIR}/.enable_practracker_in_hooks" ]; then
+        if ! python3 "${PT_DIR}/practracker.py" "$workdir"; then
+            exit 1
+        fi
+    fi
+fi
+
+if [ -e scripts/maint/checkShellScripts.sh ]; then
+    scripts/maint/checkShellScripts.sh
+fi
diff --git a/scripts/git/pre-push.git-hook b/scripts/git/pre-push.git-hook
new file mode 100755
index 0000000000..f4504c4215
--- /dev/null
+++ b/scripts/git/pre-push.git-hook
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+# git pre-push hook script to:
+# 0) Call the pre-commit hook, if it is available
+# 1) prevent "fixup!" and "squash!" commit from ending up in master, release-*
+#    or maint-*
+# 2) Disallow pushing branches other than master, release-*
+#    and maint-* to origin (e.g. gitweb.torproject.org)
+#
+# To install this script, copy it into .git/hooks/pre-push path in your
+# local copy of git repository. Make sure it has permission to execute.
+# Furthermore, make sure that TOR_UPSTREAM_REMOTE_NAME environment
+# variable is set to local name of git remote that corresponds to upstream
+# repository on e.g. git.torproject.org.
+#
+# The following sample script was used as starting point:
+# https://github.com/git/git/blob/master/templates/hooks--pre-push.sample
+
+echo "Running pre-push hook"
+
+z40=0000000000000000000000000000000000000000
+
+upstream_name=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
+
+# Are you adding a new check to the git hooks?
+#  - Common checks belong in the pre-commit hook
+#  - Push-only checks belong in the pre-push hook
+#
+# Call the pre-commit hook for the common checks, if it is executable.
+workdir=$(git rev-parse --show-toplevel)
+if [ -x "$workdir/.git/hooks/pre-commit" ]; then
+  if ! "$workdir"/.git/hooks/pre-commit; then
+    exit 1
+  fi
+fi
+
+remote="$1"
+
+remote_name=$(git remote --verbose | grep "$2" | awk '{print $1}' | head -n 1)
+
+if [[ "$remote_name" != "$upstream_name" ]]; then
+  echo "Not pushing to upstream - refraining from further checks"
+  exit 0
+fi
+
+ref_is_upstream_branch() {
+        if [ "$1" == "refs/heads/master" ] ||
+                [[ "$1" == refs/heads/release-* ]] ||
+                [[ "$1" == refs/heads/maint-* ]]
+        then
+                return 1
+        fi
+}
+
+# shellcheck disable=SC2034
+while read -r local_ref local_sha remote_ref remote_sha
+do
+	if [ "$local_sha" = $z40 ]
+	then
+		# Handle delete
+		:
+	else
+		if [ "$remote_sha" = $z40 ]
+		then
+			# New branch, examine all commits
+			range="$local_sha"
+		else
+			# Update to existing branch, examine new commits
+			range="$remote_sha..$local_sha"
+		fi
+
+                if (ref_is_upstream_branch "$local_ref" == 0 ||
+                        ref_is_upstream_branch "$remote_ref"  == 0) &&
+                        [ "$local_ref" != "$remote_ref" ]
+                then
+                        if [ "$remote" == "origin" ]
+                        then
+                                echo >&2 "Not pushing: $local_ref to $remote_ref"
+			        echo >&2 "If you really want to push this, use --no-verify."
+                                exit 1
+                        else
+                                continue
+                        fi
+                fi
+
+                # Check for fixup! commit
+                commit=$(git rev-list -n 1 --grep '^fixup!' "$range")
+		if [ -n "$commit" ]
+		then
+			echo >&2 "Found fixup! commit in $local_ref, not pushing"
+			echo >&2 "If you really want to push this, use --no-verify."
+			exit 1
+		fi
+
+                # Check for squash! commit
+                commit=$(git rev-list -n 1 --grep '^squash!' "$range")
+		if [ -n "$commit" ]
+		then
+			echo >&2 "Found squash! commit in $local_ref, not pushing"
+			echo >&2 "If you really want to push this, use --no-verify."
+			exit 1
+		fi
+	fi
+done
+
+exit 0
diff --git a/scripts/maint/add_c_file.py b/scripts/maint/add_c_file.py
new file mode 100755
index 0000000000..adf7ce79bb
--- /dev/null
+++ b/scripts/maint/add_c_file.py
@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+
+"""
+   Add a C file with matching header to the Tor codebase.  Creates
+   both files from templates, and adds them to the right include.am file.
+
+   Example usage:
+
+   % add_c_file.py ./src/feature/dirauth/ocelot.c
+"""
+
+import os
+import re
+import time
+
+def topdir_file(name):
+    """Strip opening "src" from a filename"""
+    if name.startswith("src/"):
+        name = name[4:]
+    return name
+
+def guard_macro(name):
+    """Return the guard macro that should be used for the header file 'name'.
+    """
+    td = topdir_file(name).replace(".", "_").replace("/", "_").upper()
+    return "TOR_{}".format(td)
+
+def makeext(name, new_extension):
+    """Replace the extension for the file called 'name' with 'new_extension'.
+    """
+    base = os.path.splitext(name)[0]
+    return base + "." + new_extension
+
+def instantiate_template(template, output_fname):
+    """
+    Fill in a template with string using the fields that should be used
+    for 'output_fname'.
+    """
+    names = {
+        # The relative location of the header file.
+        'header_path' : makeext(topdir_file(output_fname), "h"),
+        # The relative location of the C file file.
+        'c_file_path' : makeext(topdir_file(output_fname), "c"),
+        # The truncated name of the file.
+        'short_name' : os.path.basename(output_fname),
+        # The current year, for the copyright notice
+        'this_year' : time.localtime().tm_year,
+        # An appropriate guard macro, for the header.
+        'guard_macro' : guard_macro(output_fname),
+    }
+
+    return template.format(**names)
+
+HEADER_TEMPLATE = """\
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-{this_year}, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file {short_name}
+ * @brief Header for {c_file_path}
+ **/
+
+#ifndef {guard_macro}
+#define {guard_macro}
+
+#endif /* !defined({guard_macro}) */
+"""
+
+C_FILE_TEMPLATE = """\
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-{this_year}, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file {short_name}
+ * @brief DOCDOC
+ **/
+
+#include "orconfig.h"
+#include "{header_path}"
+"""
+
+class AutomakeChunk:
+    """
+    Represents part of an automake file.  If it is decorated with
+    an ADD_C_FILE comment, it has a "kind" based on what to add to it.
+    Otherwise, it only has a bunch of lines in it.
+    """
+    pat = re.compile(r'# ADD_C_FILE: INSERT (\S*) HERE', re.I)
+
+    def __init__(self):
+        self.lines = []
+        self.kind = ""
+
+    def addLine(self, line):
+        """
+        Insert a line into this chunk while parsing the automake file.
+        """
+        m = self.pat.match(line)
+        if m:
+            if self.lines:
+                raise ValueError("control line not preceded by a blank line")
+            self.kind = m.group(1)
+
+        self.lines.append(line)
+        if line.strip() == "":
+            return True
+
+        return False
+
+    def insertMember(self, member):
+        """
+        Add a new member to this chunk.  Try to insert it in alphabetical
+        order with matching indentation, but don't freak out too much if the
+        source isn't consistent.
+
+        Assumes that this chunk is of the form:
+           FOOBAR = \
+              X     \
+              Y     \
+              Z
+        """
+        prespace = "\t"
+        postspace = "\t\t"
+        for lineno, line in enumerate(self.lines):
+            m = re.match(r'(\s+)(\S+)(\s+)\\', line)
+            if not m:
+                continue
+            prespace, fname, postspace = m.groups()
+            if fname > member:
+                self.insert_before(lineno, member, prespace, postspace)
+                return
+        self.insert_at_end(member, prespace, postspace)
+
+    def insert_before(self, lineno, member, prespace, postspace):
+        self.lines.insert(lineno,
+                          "{}{}{}\\\n".format(prespace, member, postspace))
+
+    def insert_at_end(self, member, prespace, postspace):
+        lastline = self.lines[-1]
+        self.lines[-1] += '{}\\\n'.format(postspace)
+        self.lines.append("{}{}\n".format(prespace, member))
+
+    def dump(self, f):
+        """Write all the lines in this chunk to the file 'f'."""
+        for line in self.lines:
+            f.write(line)
+            if not line.endswith("\n"):
+                f.write("\n")
+
+class ParsedAutomake:
+    """A sort-of-parsed automake file, with identified chunks into which
+       headers and c files can be inserted.
+    """
+    def __init__(self):
+        self.chunks = []
+        self.by_type = {}
+
+    def addChunk(self, chunk):
+        """Add a newly parsed AutomakeChunk to this file."""
+        self.chunks.append(chunk)
+        self.by_type[chunk.kind.lower()] = chunk
+
+    def add_file(self, fname, kind):
+        """Insert a file of kind 'kind' to the appropriate section of this
+           file. Return True if we added it.
+        """
+        if kind.lower() in self.by_type:
+            self.by_type[kind.lower()].insertMember(fname)
+            return True
+        else:
+            return False
+
+    def dump(self, f):
+        """Write this file into a file 'f'."""
+        for chunk in self.chunks:
+            chunk.dump(f)
+
+def get_include_am_location(fname):
+    """Find the right include.am file for introducing a new file.  Return None
+       if we can't guess one.
+
+       Note that this function is imperfect because our include.am layout is
+       not (yet) consistent.
+    """
+    td = topdir_file(fname)
+    m = re.match(r'^lib/([a-z0-9_]*)/', td)
+    if m:
+        return "src/lib/{}/include.am".format(m.group(1))
+
+    if re.match(r'^(core|feature|app)/', td):
+        return "src/core/include.am"
+
+    if re.match(r'^test/', td):
+        return "src/test/include.am"
+
+    return None
+
+def run(fn):
+    """
+    Create a new C file and H file corresponding to the filename "fn", and
+    add them to include.am.
+    """
+
+    cf = makeext(fn, "c")
+    hf = makeext(fn, "h")
+
+    if os.path.exists(cf):
+        print("{} already exists".format(cf))
+        return 1
+    if os.path.exists(hf):
+        print("{} already exists".format(hf))
+        return 1
+
+    with open(cf, 'w') as f:
+        f.write(instantiate_template(C_FILE_TEMPLATE, cf))
+
+    with open(hf, 'w') as f:
+        f.write(instantiate_template(HEADER_TEMPLATE, hf))
+
+    iam = get_include_am_location(cf)
+    if iam is None or not os.path.exists(iam):
+        print("Made files successfully but couldn't identify include.am for {}"
+              .format(cf))
+        return 1
+
+    amfile = ParsedAutomake()
+    cur_chunk = AutomakeChunk()
+    with open(iam) as f:
+        for line in f:
+            if cur_chunk.addLine(line):
+                amfile.addChunk(cur_chunk)
+                cur_chunk = AutomakeChunk()
+        amfile.addChunk(cur_chunk)
+
+    amfile.add_file(cf, "sources")
+    amfile.add_file(hf, "headers")
+
+    with open(iam+".tmp", 'w') as f:
+        amfile.dump(f)
+
+    os.rename(iam+".tmp", iam)
+
+if __name__ == '__main__':
+    import sys
+    sys.exit(run(sys.argv[1]))
diff --git a/scripts/maint/annotate_ifdef_directives b/scripts/maint/annotate_ifdef_directives
deleted file mode 100755
index ca267a865e..0000000000
--- a/scripts/maint/annotate_ifdef_directives
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/python
-# Copyright (c) 2017-2019, The Tor Project, Inc.
-# See LICENSE for licensing information
-
-import re
-
-LINE_OBVIOUSNESS_LIMIT = 4
-
-class Problem(Exception):
-    pass
-
-def uncomment(s):
-    s = re.sub(r'//.*','',s)
-    s = re.sub(r'/\*.*','',s)
-    return s.strip()
-
-def translate(f_in, f_out):
-    whole_file = []
-    stack = []
-    cur_level = whole_file
-    lineno = 0
-    for line in f_in:
-        lineno += 1
-        m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)',
-                     line)
-        if not m:
-            f_out.write(line)
-            continue
-        command,rest = m.groups()
-        if command in ("if", "ifdef", "ifndef"):
-            # The #if directive pushes us one level lower on the stack.
-            if command == 'ifdef':
-                rest = "defined(%s)"%uncomment(rest)
-            elif command == 'ifndef':
-                rest = "!defined(%s)"%uncomment(rest)
-            elif rest.endswith("\\"):
-                rest = rest[:-1]+"..."
-
-            rest = uncomment(rest)
-
-            new_level = [ (command, rest, lineno) ]
-            stack.append(cur_level)
-            cur_level = new_level
-            f_out.write(line)
-        elif command in ("else", "elif"):
-            if len(cur_level) == 0 or cur_level[-1][0] == 'else':
-                raise Problem("Unexpected #%s on %d"% (command,lineno))
-            if (len(cur_level) == 1 and command == 'else' and
-                lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT):
-                f_out.write("#else /* !(%s) */\n"%cur_level[0][1])
-            else:
-                f_out.write(line)
-            cur_level.append((command, rest, lineno))
-        else:
-            assert command == 'endif'
-            if len(stack) == 0:
-                raise Problem("Unmatched #%s on %s"% (command,lineno))
-            if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT:
-                f_out.write(line)
-            elif len(cur_level) == 1 or (
-                    len(cur_level) == 2 and cur_level[1][0] == 'else'):
-                f_out.write("#endif /* %s */\n"%cur_level[0][1])
-            else:
-                f_out.write("#endif /* %s || ... */\n"%cur_level[0][1])
-            cur_level = stack.pop()
-    if len(stack) or cur_level != whole_file:
-        raise Problem("Missing #endif")
-
-import sys,os
-for fn in sys.argv[1:]:
-    with open(fn+"_OUT", 'w') as output_file:
-        translate(open(fn, 'r'), output_file)
-    os.rename(fn+"_OUT", fn)
-
diff --git a/scripts/maint/annotate_ifdef_directives.py b/scripts/maint/annotate_ifdef_directives.py
new file mode 100755
index 0000000000..102128bfa0
--- /dev/null
+++ b/scripts/maint/annotate_ifdef_directives.py
@@ -0,0 +1,317 @@
+#!/usr/bin/python
+# Copyright (c) 2017-2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+r"""
+This script iterates over a list of C files. For each file, it looks at the
+#if/#else C macros, and annotates them with comments explaining what they
+match.
+
+For example, it replaces this kind of input...
+
+>>> INPUT = '''
+... #ifdef HAVE_OCELOT
+...   C code here
+... #if MIMSY == BOROGROVE
+...   block 1
+...   block 1
+...   block 1
+...   block 1
+... #else
+...   block 2
+...   block 2
+...   block 2
+...   block 2
+... #endif
+... #endif
+... '''
+
+With this kind of output:
+>>> EXPECTED_OUTPUT = '''
+... #ifdef HAVE_OCELOT
+...   C code here
+... #if MIMSY == BOROGROVE
+...   block 1
+...   block 1
+...   block 1
+...   block 1
+... #else /* !(MIMSY == BOROGROVE) */
+...   block 2
+...   block 2
+...   block 2
+...   block 2
+... #endif /* MIMSY == BOROGROVE */
+... #endif /* defined(HAVE_OCELOT) */
+... '''
+
+Here's how to use it:
+>>> import sys
+>>> if sys.version_info.major < 3: from cStringIO import StringIO
+>>> if sys.version_info.major >= 3: from io import StringIO
+
+>>> OUTPUT = StringIO()
+>>> translate(StringIO(INPUT), OUTPUT)
+>>> assert OUTPUT.getvalue() == EXPECTED_OUTPUT
+
+Note that only #else and #endif lines are annotated.  Existing comments
+on those lines are removed.
+"""
+
+import re
+
+# Any block with fewer than this many lines does not need annotations.
+LINE_OBVIOUSNESS_LIMIT = 4
+
+# Maximum line width.  This includes a terminating newline character.
+#
+# (This is the maximum before encoding, so that if the the operating system
+# uses multiple characers to encode newline, that's still okay.)
+LINE_WIDTH=80
+
+class Problem(Exception):
+    pass
+
+def close_parens_needed(expr):
+    """Return the number of left-parentheses needed to make 'expr'
+       balanced.
+
+    >>> close_parens_needed("1+2")
+    0
+    >>> close_parens_needed("(1 + 2)")
+    0
+    >>> close_parens_needed("(1 + 2")
+    1
+    >>> close_parens_needed("(1 + (2 *")
+    2
+    >>> close_parens_needed("(1 + (2 * 3) + (4")
+    2
+    """
+    return expr.count("(") - expr.count(")")
+
+def truncate_expression(expr, new_width):
+    """Given a parenthesized C expression in 'expr', try to return a new
+       expression that is similar to 'expr', but no more than 'new_width'
+       characters long.
+
+       Try to return an expression with balanced parentheses.
+
+    >>> truncate_expression("1+2+3", 8)
+    '1+2+3'
+    >>> truncate_expression("1+2+3+4+5", 8)
+    '1+2+3...'
+    >>> truncate_expression("(1+2+3+4)", 8)
+    '(1+2...)'
+    >>> truncate_expression("(1+(2+3+4))", 8)
+    '(1+...)'
+    >>> truncate_expression("(((((((((", 8)
+    '((...))'
+    """
+    if len(expr) <= new_width:
+        # The expression is already short enough.
+        return expr
+
+    ellipsis = "..."
+
+    # Start this at the minimum that we might truncate.
+    n_to_remove = len(expr) + len(ellipsis) - new_width
+
+    # Try removing characters, one by one, until we get something where
+    # re-balancing the parentheses still fits within the limit.
+    while n_to_remove < len(expr):
+        truncated = expr[:-n_to_remove] + ellipsis
+        truncated += ")" * close_parens_needed(truncated)
+        if len(truncated) <= new_width:
+            return truncated
+        n_to_remove += 1
+
+    return ellipsis
+
+def commented_line(fmt, argument, maxwidth=LINE_WIDTH):
+    # (This is a raw docstring so that our doctests can use \.)
+    r"""
+    Return fmt%argument, for use as a commented line.  If the line would
+    be longer than maxwidth, truncate argument but try to keep its
+    parentheses balanced.
+
+    Requires that fmt%"..." will fit into maxwidth characters.
+
+    Requires that fmt ends with a newline.
+
+    >>> commented_line("/* %s */\n", "hello world", 32)
+    '/* hello world */\n'
+    >>> commented_line("/* %s */\n", "hello world", 15)
+    '/* hello... */\n'
+    >>> commented_line("#endif /* %s */\n", "((1+2) && defined(FOO))", 32)
+    '#endif /* ((1+2) && defi...) */\n'
+
+
+    The default line limit is 80 characters including the newline:
+
+    >>> long_argument = "long " * 100
+    >>> long_line = commented_line("#endif /* %s */\n", long_argument)
+    >>> len(long_line)
+    80
+
+    >>> long_line[:40]
+    '#endif /* long long long long long long '
+    >>> long_line[40:]
+    'long long long long long long lon... */\n'
+
+    If a line works out to being 80 characters naturally, it isn't truncated,
+    and no ellipsis is added.
+
+    >>> medium_argument = "a"*66
+    >>> medium_line = commented_line("#endif /* %s */\n", medium_argument)
+    >>> len(medium_line)
+    80
+    >>> "..." in medium_line
+    False
+    >>> medium_line[:40]
+    '#endif /* aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
+    >>> medium_line[40:]
+    'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa */\n'
+
+
+    """
+    assert fmt.endswith("\n")
+    result = fmt % argument
+    if len(result) <= maxwidth:
+        return result
+    else:
+        # How long can we let the argument be?  Try filling in the
+        # format with an empty argument to find out.
+        max_arg_width = maxwidth - len(fmt % "")
+        result = fmt % truncate_expression(argument, max_arg_width)
+        assert len(result) <= maxwidth
+        return result
+
+def negate(expr):
+    """Return a negated version of expr; try to avoid double-negation.
+
+    We usually wrap expressions in parentheses and add a "!".
+    >>> negate("A && B")
+    '!(A && B)'
+
+    But if we recognize the expression as negated, we can restore it.
+    >>> negate(negate("A && B"))
+    'A && B'
+
+    The same applies for defined(FOO).
+    >>> negate("defined(FOO)")
+    '!defined(FOO)'
+    >>> negate(negate("defined(FOO)"))
+    'defined(FOO)'
+
+    Internal parentheses don't confuse us:
+    >>> negate("!(FOO) && !(BAR)")
+    '!(!(FOO) && !(BAR))'
+
+    """
+    expr = expr.strip()
+    # See whether we match !(...), with no intervening close-parens.
+    m = re.match(r'^!\s*\(([^\)]*)\)$', expr)
+    if m:
+        return m.group(1)
+
+
+    # See whether we match !?defined(...), with no intervening close-parens.
+    m = re.match(r'^(!?)\s*(defined\([^\)]*\))$', expr)
+    if m:
+        if m.group(1) == "!":
+            prefix = ""
+        else:
+            prefix = "!"
+        return prefix + m.group(2)
+
+    return "!(%s)" % expr
+
+def uncomment(s):
+    """
+    Remove existing trailing comments from an #else or #endif line.
+    """
+    s = re.sub(r'//.*','',s)
+    s = re.sub(r'/\*.*','',s)
+    return s.strip()
+
+def translate(f_in, f_out):
+    """
+    Read a file from f_in, and write its annotated version to f_out.
+    """
+    # A stack listing our current if/else state.  Each member of the stack
+    # is a list of directives.  Each directive is a 3-tuple of
+    #    (command, rest, lineno)
+    # where "command" is one of if/ifdef/ifndef/else/elif, and where
+    # "rest" is an expression in a format suitable for use with #if, and where
+    # lineno is the line number where the directive occurred.
+    stack = []
+    # the stack element corresponding to the top level of the file.
+    whole_file = []
+    cur_level = whole_file
+    lineno = 0
+    for line in f_in:
+        lineno += 1
+        m = re.match(r'\s*#\s*(if|ifdef|ifndef|else|endif|elif)\b\s*(.*)',
+                     line)
+        if not m:
+            # no directive, so we can just write it out.
+            f_out.write(line)
+            continue
+        command,rest = m.groups()
+        if command in ("if", "ifdef", "ifndef"):
+            # The #if directive pushes us one level lower on the stack.
+            if command == 'ifdef':
+                rest = "defined(%s)"%uncomment(rest)
+            elif command == 'ifndef':
+                rest = "!defined(%s)"%uncomment(rest)
+            elif rest.endswith("\\"):
+                rest = rest[:-1]+"..."
+
+            rest = uncomment(rest)
+
+            new_level = [ (command, rest, lineno) ]
+            stack.append(cur_level)
+            cur_level = new_level
+            f_out.write(line)
+        elif command in ("else", "elif"):
+            # We stay at the same level on the stack.  If we have an #else,
+            # we comment it.
+            if len(cur_level) == 0 or cur_level[-1][0] == 'else':
+                raise Problem("Unexpected #%s on %d"% (command,lineno))
+            if (len(cur_level) == 1 and command == 'else' and
+                lineno > cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT):
+                f_out.write(commented_line("#else /* %s */\n",
+                                           negate(cur_level[0][1])))
+            else:
+                f_out.write(line)
+            cur_level.append((command, rest, lineno))
+        else:
+            # We pop one element on the stack, and comment an endif.
+            assert command == 'endif'
+            if len(stack) == 0:
+                raise Problem("Unmatched #%s on %s"% (command,lineno))
+            if lineno <= cur_level[0][2] + LINE_OBVIOUSNESS_LIMIT:
+                f_out.write(line)
+            elif len(cur_level) == 1 or (
+                    len(cur_level) == 2 and cur_level[1][0] == 'else'):
+                f_out.write(commented_line("#endif /* %s */\n",
+                                           cur_level[0][1]))
+            else:
+                f_out.write(commented_line("#endif /* %s || ... */\n",
+                                           cur_level[0][1]))
+            cur_level = stack.pop()
+    if len(stack) or cur_level != whole_file:
+        raise Problem("Missing #endif")
+
+if __name__ == '__main__':
+
+    import sys,os
+
+    if sys.argv[1] == "--self-test":
+        import doctest
+        doctest.testmod()
+        sys.exit(0)
+
+    for fn in sys.argv[1:]:
+        with open(fn+"_OUT", 'w') as output_file:
+            translate(open(fn, 'r'), output_file)
+        os.rename(fn+"_OUT", fn)
diff --git a/scripts/maint/checkIncludes.py b/scripts/maint/checkIncludes.py
index 46a3f39638..926b201b35 100755
--- a/scripts/maint/checkIncludes.py
+++ b/scripts/maint/checkIncludes.py
@@ -1,115 +1,14 @@
-#!/usr/bin/python3
+#!/usr/bin/python
 # Copyright 2018 The Tor Project, Inc.  See LICENSE file for licensing info.
 
-"""This script looks through all the directories for files matching *.c or
-   *.h, and checks their #include directives to make sure that only "permitted"
-   headers are included.
+# This file is no longer here; see practracker/includes.py for this
+# functionality.  This is a stub file that exists so that older git
+# hooks will know where to look.
 
-   Any #include directives with angle brackets (like #include <stdio.h>) are
-   ignored -- only directives with quotes (like #include "foo.h") are
-   considered.
+import sys, os
 
-   To decide what includes are permitted, this script looks at a .may_include
-   file in each directory.  This file contains empty lines, #-prefixed
-   comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h)
-   for files that are permitted.
-"""
+dirname = os.path.split(sys.argv[0])[0]
+new_location = os.path.join(dirname, "practracker", "includes.py")
+python = sys.executable
 
-
-from __future__ import print_function
-
-import fnmatch
-import os
-import re
-import sys
-
-# Global: Have there been any errors?
-trouble = False
-
-if sys.version_info[0] <= 2:
-    def open_file(fname):
-        return open(fname, 'r')
-else:
-    def open_file(fname):
-        return open(fname, 'r', encoding='utf-8')
-
-def err(msg):
-    """ Declare that an error has happened, and remember that there has
-        been an error. """
-    global trouble
-    trouble = True
-    print(msg, file=sys.stderr)
-
-def fname_is_c(fname):
-    """ Return true iff 'fname' is the name of a file that we should
-        search for possibly disallowed #include directives. """
-    return fname.endswith(".h") or fname.endswith(".c")
-
-INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"')
-RULES_FNAME = ".may_include"
-
-class Rules(object):
-    """ A 'Rules' object is the parsed version of a .may_include file. """
-    def __init__(self, dirpath):
-        self.dirpath = dirpath
-        self.patterns = []
-        self.usedPatterns = set()
-
-    def addPattern(self, pattern):
-        self.patterns.append(pattern)
-
-    def includeOk(self, path):
-        for pattern in self.patterns:
-            if fnmatch.fnmatchcase(path, pattern):
-                self.usedPatterns.add(pattern)
-                return True
-        return False
-
-    def applyToLines(self, lines, context=""):
-        lineno = 0
-        for line in lines:
-            lineno += 1
-            m = INCLUDE_PATTERN.match(line)
-            if m:
-                include = m.group(1)
-                if not self.includeOk(include):
-                    err("Forbidden include of {} on line {}{}".format(
-                        include, lineno, context))
-
-    def applyToFile(self, fname):
-        with open_file(fname) as f:
-            #print(fname)
-            self.applyToLines(iter(f), " of {}".format(fname))
-
-    def noteUnusedRules(self):
-        for p in self.patterns:
-            if p not in self.usedPatterns:
-                print("Pattern {} in {} was never used.".format(p, self.dirpath))
-
-def load_include_rules(fname):
-    """ Read a rules file from 'fname', and return it as a Rules object. """
-    result = Rules(os.path.split(fname)[0])
-    with open_file(fname) as f:
-        for line in f:
-            line = line.strip()
-            if line.startswith("#") or not line:
-                continue
-            result.addPattern(line)
-    return result
-
-list_unused = False
-
-for dirpath, dirnames, fnames in os.walk("src"):
-    if ".may_include" in fnames:
-        rules = load_include_rules(os.path.join(dirpath, RULES_FNAME))
-        for fname in fnames:
-            if fname_is_c(fname):
-                rules.applyToFile(os.path.join(dirpath,fname))
-        if list_unused:
-            rules.noteUnusedRules()
-
-if trouble:
-    err(
-"""To change which includes are allowed in a C file, edit the {}
-files in its enclosing directory.""".format(RULES_FNAME))
-    sys.exit(1)
+os.execl(python, python, new_location, *sys.argv[1:])
diff --git a/scripts/maint/checkShellScripts.sh b/scripts/maint/checkShellScripts.sh
new file mode 100755
index 0000000000..4c872c7ee0
--- /dev/null
+++ b/scripts/maint/checkShellScripts.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Tor Project, Inc.
+# See LICENSE for license information
+#
+# checkShellScripts.sh
+# --------------------
+# If shellcheck is installed, check all the shell scripts that we can fix.
+
+set -e
+
+# Only run this script if shellcheck is installed
+# command echoes the path to shellcheck, which is a useful diagnostic log
+if ! command -v shellcheck; then
+    printf "%s: Install shellcheck to check shell scripts.\\n" "$0"
+    exit 0
+fi
+
+# Some platforms don't have realpath
+if command -v realpath ; then
+    HERE=$(dirname "$(realpath "$0")")
+else
+    HERE=$(dirname "$0")
+    if [ ! -d "$HERE" ] || [ "$HERE" = "." ]; then
+        HERE=$(dirname "$PWD/$0")
+    fi
+fi
+TOPLEVEL=$(dirname "$(dirname "$HERE")")
+
+# Check we actually have a tor/src directory
+if [ ! -d "$TOPLEVEL/src" ]; then
+    printf "Error: Couldn't find src directory in expected location: %s\\n" \
+        "$TOPLEVEL/src"
+    exit 1
+fi
+
+# Check *.sh scripts, but ignore the ones that we can't fix
+find "$TOPLEVEL/contrib" "$TOPLEVEL/doc" "$TOPLEVEL/scripts" "$TOPLEVEL/src" \
+    -name "*.sh" \
+    -not -path "$TOPLEVEL/src/ext/*" \
+    -not -path "$TOPLEVEL/src/rust/registry/*" \
+    -exec shellcheck {} +
+
+# Check scripts that aren't named *.sh
+if [ -d "$TOPLEVEL/scripts/test" ]; then
+    shellcheck \
+        "$TOPLEVEL/scripts/test/cov-diff" \
+        "$TOPLEVEL/scripts/test/coverage"
+fi
+if [ -e \
+    "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert" \
+   ]; then
+    shellcheck \
+        "$TOPLEVEL/contrib/dirauth-tools/nagios-check-tor-authority-cert"
+fi
+if [ -e "$TOPLEVEL/contrib/client-tools/torify" ]; then
+    shellcheck "$TOPLEVEL/contrib/client-tools/torify"
+fi
+if [ -d "$TOPLEVEL/scripts/git" ]; then
+    shellcheck "$TOPLEVEL/scripts/git/"*.git-hook
+fi
diff --git a/scripts/maint/checkSpace.pl b/scripts/maint/checkSpace.pl
index 633b47e314..9c9b68ff9d 100755
--- a/scripts/maint/checkSpace.pl
+++ b/scripts/maint/checkSpace.pl
@@ -18,6 +18,8 @@ if ($ARGV[0] =~ /^-/) {
 
 our %basenames = ();
 
+our %guardnames = ();
+
 for my $fn (@ARGV) {
     open(F, "$fn");
     my $lastnil = 0;
@@ -31,6 +33,10 @@ for my $fn (@ARGV) {
     } else {
         $basenames{$basename} = $fn;
     }
+    my $isheader = ($fn =~ /\.h/);
+    my $seenguard = 0;
+    my $guardname = "<none>";
+
     while (<F>) {
         ## Warn about windows-style newlines.
         #    (We insist on lines that end with a single LF character, not
@@ -112,6 +118,23 @@ for my $fn (@ARGV) {
                     next;
                 }
             }
+
+            if ($isheader) {
+                if ($seenguard == 0) {
+                    if (/ifndef\s+(\S+)/) {
+                        ++$seenguard;
+                        $guardname = $1;
+                    }
+                } elsif ($seenguard == 1) {
+                    if (/^\#define (\S+)/) {
+                        ++$seenguard;
+                        if ($1 ne $guardname) {
+                            msg "GUARD:$fn:$.: Header guard macro mismatch.\n";
+                        }
+                    }
+                }
+            }
+
             if (m!/\*.*?\*/!) {
                 s!\s*/\*.*?\*/!!;
             } elsif (m!/\*!) {
@@ -154,7 +177,7 @@ for my $fn (@ARGV) {
                     $1 ne "elsif" and $1 ne "WINAPI" and $2 ne "WINAPI" and
                     $1 ne "void" and $1 ne "__attribute__" and $1 ne "op" and
                     $1 ne "size_t" and $1 ne "double" and $1 ne "uint64_t" and
-                    $1 ne "workqueue_reply_t") {
+                    $1 ne "workqueue_reply_t" and $1 ne "bool") {
                     msg "     fn ():$fn:$.\n";
                 }
             }
@@ -201,6 +224,15 @@ for my $fn (@ARGV) {
             }
         }
     }
+    if ($isheader && $C) {
+        if ($seenguard < 2) {
+            msg "$fn:No #ifndef/#define header guard pair found.\n";
+        } elsif ($guardnames{$guardname}) {
+            msg "$fn:Guard macro $guardname also used in $guardnames{$guardname}\n";
+        } else {
+            $guardnames{$guardname} = $fn;
+        }
+    }
     close(F);
 }
 
diff --git a/scripts/maint/fallback.whitelist b/scripts/maint/fallback.whitelist
deleted file mode 100644
index 79551948c6..0000000000
--- a/scripts/maint/fallback.whitelist
+++ /dev/null
@@ -1,997 +0,0 @@
-# updateFallbackDirs.py directory mirror whitelist
-#
-# Format:
-# IPv4:DirPort orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
-# or use:
-# scripts/maint/generateFallbackDirLine.py fingerprint ...
-#
-# All attributes must match for the directory mirror to be included.
-# If the fallback has an ipv6 key, the whitelist line must also have
-# it, and vice versa, otherwise they don't match.
-# (The blacklist overrides the whitelist.)
-
-# To replace this list with the hard-coded fallback list (for testing), use
-# a command similar to:
-#   cat src/app/config/fallback_dirs.inc | grep \" | grep -v weight | \
-#   tr -d '\n' | \
-#   sed 's/"" / /g' | sed 's/""/"/g' | tr \" '\n' | grep -v '^$' \
-#   > scripts/maint/fallback.whitelist
-#
-# When testing before a release, exclusions due to changed details will result
-# in a warning, unless the IPv4 address or port change happened recently.
-# Then it is only logged at info level, as part of the eligibility check.
-# Exclusions due to stability also are only shown at info level.
-#
-# Add the number of selected, slow, and excluded relays, and compare that to
-# the number of hard-coded relays. If it's less, use info-level logs to find
-# out why each of the missing relays was excluded.
-
-# If a relay operator wants their relay to be a FallbackDir,
-# enter the following information here:
-# <IPv4>:<DirPort> orport=<ORPort> id=<ID> [ ipv6=<IPv6>:<IPv6 ORPort> ]
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008362.html
-# https://trac.torproject.org/projects/tor/ticket/22321#comment:22
-78.47.18.110:443 orport=80 id=F8D27B163B9247B232A2EEE68DD8B698695C28DE ipv6=[2a01:4f8:120:4023::110]:80 # fluxe3
-131.188.40.188:1443 orport=80 id=EBE718E1A49EE229071702964F8DB1F318075FF8 ipv6=[2001:638:a000:4140::ffff:188]:80 # fluxe4
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008366.html
-5.39.88.19:9030 orport=9001 id=7CB8C31432A796731EA7B6BF4025548DFEB25E0C ipv6=[2001:41d0:8:9a13::1]:9050
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008370.html
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008517.html
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008555.html
-212.47.237.95:9030 orport=9001 id=3F5D8A879C58961BB45A3D26AC41B543B40236D6
-212.47.237.95:9130 orport=9101 id=6FB38EB22E57EF7ED5EF00238F6A48E553735D88
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008372.html
-# IPv6 tunnel available on request (is this a good idea?)
-108.53.208.157:80 orport=443 id=4F0DB7E687FC7C0AE55C8F243DA8B0EB27FBF1F2
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008373.html
-167.114.35.28:9030 orport=9001 id=E65D300F11E1DB12C534B0146BDAB6972F1A8A48
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008378.html
-144.76.14.145:110 orport=143 id=14419131033443AE6E21DA82B0D307F7CAE42BDB ipv6=[2a01:4f8:190:9490::dead]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008379.html
-# Email sent directly to teor, verified using relay contact info
-91.121.84.137:4951 orport=4051 id=6DE61A6F72C1E5418A66BFED80DFB63E4C77668F
-91.121.84.137:4952 orport=4052 id=9FBEB75E8BC142565F12CBBE078D63310236A334
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008381.html
-# Sent additional emails to teor with updated relays
-81.7.11.96:9030 orport=9001 id=8FA37B93397015B2BC5A525C908485260BE9F422 # Doedel22
-# 9F5068310818ED7C70B0BC4087AB55CB12CB4377 not found in current consensus
-178.254.19.101:80 orport=443 id=F9246DEF2B653807236DA134F2AEAB103D58ABFE # Freebird31
-178.254.19.101:9030 orport=9001 id=0C475BA4D3AA3C289B716F95954CAD616E50C4E5 # Freebird32
-81.7.14.253:9001 orport=443 id=1AE039EE0B11DB79E4B4B29CBA9F752864A0259E # Ichotolot60
-81.7.11.186:1080 orport=443 id=B86137AE9681701901C6720E55C16805B46BD8E3 # BeastieJoy60
-85.25.213.211:465 orport=80 id=CE47F0356D86CF0A1A2008D97623216D560FB0A8 # BeastieJoy61
-85.25.159.65:995 orport=80 id=52BFADA8BEAA01BA46C8F767F83C18E2FE50C1B9 # BeastieJoy63
-81.7.3.67:993 orport=443 id=A2E6BB5C391CD46B38C55B4329C35304540771F1 # BeastieJoy62
-81.7.14.31:9001 orport=443 id=7600680249A22080ECC6173FBBF64D6FCF330A61 # Ichotolot62
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008382.html
-51.255.33.237:9091 orport=9001 id=A360C21FA87FFA2046D92C17086A6B47E5C68109
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008383.html
-81.7.14.246:80 orport=443 id=CE75BF0972ADD52AF8807602374E495C815DB304 ipv6=[2a02:180:a:51::dead]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008384.html
-# Sent additional email to teor with fingerprint change
-149.202.98.161:80 orport=443 id=FC64CD763F8C1A319BFBBF62551684F4E1E42332 ipv6=[2001:41d0:8:4528::161]:443
-193.111.136.162:80 orport=443 id=C79552275DFCD486B942510EF663ED36ACA1A84B ipv6=[2001:4ba0:cafe:10d0::1]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008416.html
-185.100.84.212:80 orport=443 id=330CD3DB6AD266DC70CDB512B036957D03D9BC59 ipv6=[2a06:1700:0:7::1]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2015-December/008417.html
-178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443
-178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008513.html
-178.62.173.203:9030 orport=9001 id=DD85503F2D1F52EF9EAD621E942298F46CD2FC10 ipv6=[2a03:b0c0:0:1010::a4:b001]:9001
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008534.html
-5.9.110.236:9030 orport=9001 id=0756B7CD4DFC8182BE23143FAC0642F515182CEB ipv6=[2a01:4f8:162:51e2::2]:9001
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-January/008542.html
-178.62.199.226:80 orport=443 id=CBEFF7BA4A4062045133C053F2D70524D8BBE5BE ipv6=[2a03:b0c0:2:d0::b7:5001]:443
-
-# Email sent directly to teor, verified using relay contact info
-94.23.204.175:9030 orport=9001 id=5665A3904C89E22E971305EE8C1997BCA4123C69
-
-# Email sent directly to teor, verified using relay contact info
-171.25.193.77:80 orport=443 id=A10C4F666D27364036B562823E5830BC448E046A ipv6=[2001:67c:289c:3::77]:443
-171.25.193.78:80 orport=443 id=A478E421F83194C114F41E94F95999672AED51FE ipv6=[2001:67c:289c:3::78]:443
-171.25.193.20:80 orport=443 id=DD8BD7307017407FCC36F8D04A688F74A0774C02 ipv6=[2001:67c:289c::20]:443
-# same machine as DD8BD7307017407FCC36F8D04A688F74A0774C02
-171.25.193.25:80 orport=443 id=185663B7C12777F052B2C2D23D7A239D8DA88A0F ipv6=[2001:67c:289c::25]:443
-
-# Email sent directly to teor, verified using relay contact info
-212.47.229.2:9030 orport=9001 id=20462CBA5DA4C2D963567D17D0B7249718114A68 ipv6=[2001:bc8:4400:2100::f03]:9001
-93.115.97.242:9030 orport=9001 id=B5212DB685A2A0FCFBAE425738E478D12361710D
-46.28.109.231:9030 orport=9001 id=F70B7C5CD72D74C7F9F2DC84FA9D20D51BA13610 ipv6=[2a02:2b88:2:1::4205:1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-85.235.250.88:80 orport=443 id=72B2B12A3F60408BDBC98C6DF53988D3A0B3F0EE # TykRelay01
-185.96.88.29:80 orport=443 id=86C281AD135058238D7A337D546C902BE8505DDE # TykRelay051
-# This fallback opted-in in previous releases, then changed its details,
-# and so we blacklisted it. Now we want to whitelist changes.
-# Assume details update is permanent
-185.96.180.29:80 orport=443 id=F93D8F37E35C390BCAD9F9069E13085B745EC216 # TykRelay06
-
-# Email sent directly to teor, verified using relay contact info
-185.11.180.67:80 orport=9001 id=794D8EA8343A4E820320265D05D4FA83AB6D1778
-
-# Email sent directly to teor, verified using relay contact info
-178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443
-46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7
-178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443
-178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443
-178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443
-81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443
-209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443
-81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443
-46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1
-178.16.208.62:80 orport=443 id=5CF8AFA5E4B0BB88942A44A3F3AAE08C3BDFD60B ipv6=[2a00:1c20:4089:1234:a6a4:2926:d0af:dfee]:443"
-46.165.221.166:80 orport=443 id=EE5F897C752D46BCFF531641B853FC6BC78DD4A7
-178.16.208.60:80 orport=443 id=B44FBE5366AD98B46D829754FA4AC599BAE41A6A ipv6=[2a00:1c20:4089:1234:67bc:79f3:61c0:6e49]:443
-178.16.208.55:80 orport=443 id=C4AEA05CF380BAD2230F193E083B8869B4A29937 ipv6=[2a00:1c20:4089:1234:7b2c:11c5:5221:903e]:443
-178.16.208.61:80 orport=443 id=3B52392E2256C35CDCF7801FF898FC88CE6D431A ipv6=[2a00:1c20:4089:1234:2712:a3d0:666b:88a6]:443
-81.89.96.88:80 orport=443 id=55ED4BB49F6D3F36D8D9499BE43500E017A5EF82 ipv6=[2a02:180:1:1:14c5:b0b7:2d7d:5f3a]:443
-209.222.8.196:80 orport=443 id=C86D2F3DEFE287A0EEB28D4887AF14E35C172733 ipv6=[2001:19f0:1620:41c1:426c:5adf:2ed5:4e88]:443
-81.89.96.89:80 orport=443 id=28651F419F5A1CF74511BB500C58112192DD4943 ipv6=[2a02:180:1:1:2ced:24e:32ea:a03b]:443
-46.165.221.166:9030 orport=9001 id=8C7106C880FE8AA1319DD71B59623FCB8914C9F1
-178.16.208.56:80 orport=443 id=2CDCFED0142B28B002E89D305CBA2E26063FADE2 ipv6=[2a00:1c20:4089:1234:cd49:b58a:9ebe:67ec]:443
-178.16.208.58:80 orport=443 id=A4C98CEA3F34E05299417E9F885A642C88EF6029 ipv6=[2a00:1c20:4089:1234:cdae:1b3e:cc38:3d45]:443
-178.16.208.57:80 orport=443 id=92CFD9565B24646CAC2D172D3DB503D69E777B8A ipv6=[2a00:1c20:4089:1234:7825:2c5d:1ecd:c66f]:443
-178.16.208.59:80 orport=443 id=136F9299A5009A4E0E96494E723BDB556FB0A26B ipv6=[2a00:1c20:4089:1234:bff6:e1bb:1ce3:8dc6]:443
-
-# Email sent directly to teor, verified using relay contact info
-5.39.76.158:80 orport=443 id=C41F60F8B00E7FEF5CCC5BC6BB514CA1B8AAB651
-
-# Email sent directly to teor, verified using relay contact info
-109.163.234.2:80 orport=443 id=14F92FF956105932E9DEC5B82A7778A0B1BD9A52
-109.163.234.4:80 orport=443 id=4888770464F0E900EFEF1BA181EA873D13F7713C
-109.163.234.5:80 orport=443 id=5EB8D862E70981B8690DEDEF546789E26AB2BD24
-109.163.234.7:80 orport=443 id=23038A7F2845EBA2234ECD6651BD4A7762F51B18
-109.163.234.8:80 orport=443 id=0818DAE0E2DDF795AEDEAC60B15E71901084F281
-109.163.234.9:80 orport=443 id=ABF7FBF389C9A747938B639B20E80620B460B2A9
-62.102.148.67:80 orport=443 id=4A0C3E177AF684581EF780981AEAF51A98A6B5CF
-# Assume details update is permanent
-77.247.181.166:80 orport=443 id=77131D7E2EC1CA9B8D737502256DA9103599CE51 # CriticalMass
-77.247.181.164:80 orport=443 id=204DFD2A2C6A0DC1FA0EACB495218E0B661704FD # HaveHeart
-77.247.181.162:80 orport=443 id=7BFB908A3AA5B491DA4CA72CCBEE0E1F2A939B55 # sofia
-
-# https://twitter.com/biotimylated/status/718994247500718080
-212.47.252.149:9030 orport=9001 id=2CAC39BAA996791CEFAADC9D4754D65AF5EB77C0
-
-# Email sent directly to teor, verified using relay contact info
-46.165.230.5:80 orport=443 id=A0F06C2FADF88D3A39AA3072B406F09D7095AC9E
-
-# Email sent directly to teor, verified using relay contact info
-94.242.246.24:23 orport=8080 id=EC116BCB80565A408CE67F8EC3FE3B0B02C3A065 ipv6=[2a01:608:ffff:ff07::1:24]:9004
-176.126.252.11:443 orport=9001 id=B0279A521375F3CB2AE210BDBFC645FDD2E1973A ipv6=[2a02:59e0:0:7::11]:9003
-176.126.252.12:21 orport=8080 id=379FB450010D17078B3766C2273303C358C3A442 ipv6=[2a02:59e0:0:7::12]:81
-94.242.246.23:443 orport=9001 id=F65E0196C94DFFF48AFBF2F5F9E3E19AAE583FD0 ipv6=[2a01:608:ffff:ff07::1:23]:9003
-85.248.227.164:444 orport=9002 id=B84F248233FEA90CAD439F292556A3139F6E1B82 ipv6=[2a00:1298:8011:212::164]:9004
-85.248.227.163:443 orport=9001 id=C793AB88565DDD3C9E4C6F15CCB9D8C7EF964CE9 ipv6=[2a00:1298:8011:212::163]:9003
-
-# Email sent directly to teor, verified using relay contact info
-148.251.190.229:9030 orport=9010 id=BF0FB582E37F738CD33C3651125F2772705BB8E8 ipv6=[2a01:4f8:211:c68::2]:9010
-
-# Email sent directly to teor, verified using relay contact info
-5.79.68.161:81 orport=443 id=9030DCF419F6E2FBF84F63CBACBA0097B06F557E ipv6=[2001:1af8:4700:a012:1::1]:443
-5.79.68.161:9030 orport=9001 id=B7EC0C02D7D9F1E31B0C251A6B058880778A0CD1 ipv6=[2001:1af8:4700:a012:1::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-62.210.92.11:9030 orport=9001 id=0266B0660F3F20A7D1F3D8335931C95EF50F6C6B ipv6=[2001:bc8:338c::1]:9001
-62.210.92.11:9130 orport=9101 id=387B065A38E4DAA16D9D41C2964ECBC4B31D30FF ipv6=[2001:bc8:338c::1]:9101
-
-# Email sent directly to teor, verified using relay contact info
-188.165.194.195:9030 orport=9001 id=49E7AD01BB96F6FE3AB8C3B15BD2470B150354DF
-
-# Message sent directly to teor, verified using relay contact info
-95.215.44.110:80 orport=443 id=D56AA4A1AA71961F5279FB70A6DCF7AD7B993EB5
-95.215.44.122:80 orport=443 id=998D8FE06B867AA3F8D257A7D28FFF16964D53E2
-95.215.44.111:80 orport=443 id=A7C7FD510B20BC8BE8F2A1D911364E1A23FBD09F
-
-# Email sent directly to teor, verified using relay contact info
-86.59.119.88:80 orport=443 id=ACD889D86E02EDDAB1AFD81F598C0936238DC6D0
-86.59.119.83:80 orport=443 id=FC9AC8EA0160D88BCCFDE066940D7DD9FA45495B
-
-# Email sent directly to teor, verified using relay contact info
-193.11.164.243:9030 orport=9001 id=FFA72BD683BC2FCF988356E6BEC1E490F313FB07 ipv6=[2001:6b0:7:125::243]:9001
-109.105.109.162:52860 orport=60784 id=32EE911D968BE3E016ECA572BB1ED0A9EE43FC2F ipv6=[2001:948:7:2::163]:5001
-
-# Email sent directly to teor, verified using relay contact info
-146.0.32.144:9030 orport=9001 id=35E8B344F661F4F2E68B17648F35798B44672D7E
-
-# Email sent directly to teor, verified using relay contact info
-46.252.26.2:45212 orport=49991 id=E589316576A399C511A9781A73DA4545640B479D
-
-# Email sent directly to teor, verified using relay contact info
-89.187.142.208:80 orport=443 id=64186650FFE4469EBBE52B644AE543864D32F43C
-
-# Email sent directly to teor
-# Assume details update is permanent
-212.51.134.123:9030 orport=9001 id=50586E25BE067FD1F739998550EDDCB1A14CA5B2 # Jans
-
-# Email sent directly to teor, verified using relay contact info
-46.101.143.173:80 orport=443 id=F960DF50F0FD4075AC9B505C1D4FFC8384C490FB
-
-# Email sent directly to teor, verified using relay contact info
-193.171.202.146:9030 orport=9001 id=01A9258A46E97FF8B2CAC7910577862C14F2C524
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-197.231.221.211:9030 orport=443 id=BC630CBBB518BE7E9F4E09712AB0269E9DC7D626 # IPredator
-
-# Email sent directly to teor, verified using relay contact info
-185.61.138.18:8080 orport=4443 id=2541759BEC04D37811C2209A88E863320271EC9C
-
-# Email sent directly to teor, verified using relay contact info
-193.11.114.45:9031 orport=9002 id=80AAF8D5956A43C197104CEF2550CD42D165C6FB
-193.11.114.43:9030 orport=9001 id=12AD30E5D25AA67F519780E2111E611A455FDC89 ipv6=[2001:6b0:30:1000::99]:9050
-193.11.114.46:9032 orport=9003 id=B83DC1558F0D34353BB992EF93AFEAFDB226A73E
-
-# Email sent directly to teor, verified using relay contact info
-138.201.250.33:9012 orport=9011 id=2BA2C8E96B2590E1072AECE2BDB5C48921BF8510
-
-# Email sent directly to teor, verified using relay contact info
-37.221.162.226:9030 orport=9001 id=D64366987CB39F61AD21DBCF8142FA0577B92811
-
-# Email sent directly to teor, verified using relay contact info
-91.219.237.244:80 orport=443 id=92ECC9E0E2AF81BB954719B189AC362E254AD4A5
-
-# Email sent directly to teor, verified using relay contact info
-185.21.100.50:9030 orport=9001 id=58ED9C9C35E433EE58764D62892B4FFD518A3CD0 ipv6=[2a00:1158:2:cd00:0:74:6f:72]:443
-
-# Email sent directly to teor, verified using relay contact info
-193.35.52.53:9030 orport=9001 id=DAA39FC00B196B353C2A271459C305C429AF09E4
-
-# Email sent directly to teor, verified using relay contact info
-134.119.3.164:9030 orport=9001 id=D1B8AAA98C65F3DF7D8BB3AF881CAEB84A33D8EE
-
-# Email sent directly to teor, verified using relay contact info
-173.212.254.192:31336 orport=31337 id=99E246DB480B313A3012BC3363093CC26CD209C7
-
-# Email sent directly to teor, verified using relay contact info
-178.62.22.36:80 orport=443 id=A0766C0D3A667A3232C7D569DE94A28F9922FCB1 ipv6=[2a03:b0c0:1:d0::174:1]:9050
-188.166.23.127:80 orport=443 id=8672E8A01B4D3FA4C0BBE21C740D4506302EA487 ipv6=[2a03:b0c0:2:d0::27b:7001]:9050
-198.199.64.217:80 orport=443 id=B1D81825CFD7209BD1B4520B040EF5653C204A23 ipv6=[2604:a880:400:d0::1a9:b001]:9050
-159.203.32.149:80 orport=443 id=55C7554AFCEC1062DCBAC93E67B2E03C6F330EFC ipv6=[2604:a880:cad:d0::105:f001]:9050
-
-# Email sent directly to teor, verified using relay contact info
-5.196.31.80:9030 orport=9900 id=DFB2EB472643FAFCD5E73D2E37D51DB67203A695 ipv6=[2001:41d0:52:400::a65]:9900
-
-# Email sent directly to teor, verified using relay contact info
-188.138.112.60:1433 orport=1521 id=C414F28FD2BEC1553024299B31D4E726BEB8E788
-
-# Email sent directly to teor, verified using relay contact info
-213.61.66.118:9031 orport=9001 id=30648BC64CEDB3020F4A405E4AB2A6347FB8FA22
-213.61.66.117:9032 orport=9002 id=6E44A52E3D1FF7683FE5C399C3FB5E912DE1C6B4
-213.61.66.115:9034 orport=9004 id=480CCC94CEA04D2DEABC0D7373868E245D4C2AE2
-213.61.66.116:9033 orport=9003 id=A9DEB920B42B4EC1DE6249034039B06D61F38690
-
-# Email sent directly to teor, verified using relay contact info
-136.243.187.165:9030 orport=443 id=1AC65257D7BFDE7341046625470809693A8ED83E
-
-# Email sent directly to teor, verified using relay contact info
-212.47.230.49:9030 orport=9001 id=3D6D0771E54056AEFC28BB1DE816951F11826E97
-
-# Email sent directly to teor, verified using relay contact info
-192.99.55.69:80 orport=443 id=0682DE15222A4A4A0D67DBA72A8132161992C023
-192.99.59.140:80 orport=443 id=3C9148DA49F20654730FAC83FFF693A4D49D0244
-51.254.215.13:80 orport=443 id=73C30C8ABDD6D9346C822966DE73B9F82CB6178A
-51.254.215.129:80 orport=443 id=7B4491D05144B20AE8519AE784B94F0525A8BB79
-192.99.59.139:80 orport=443 id=82EC878ADA7C205146B9F5193A7310867FAA0D7B
-51.254.215.124:80 orport=443 id=98999EBE89B5FA9AA0C58421F0B46C3D0AF51CBA
-51.254.214.208:80 orport=443 id=C3F0D1417848EAFC41277A73DEB4A9F2AEC23DDF
-192.99.59.141:80 orport=443 id=F45426551795B9DA78BEDB05CD5F2EACED8132E4
-192.99.59.14:80 orport=443 id=161A1B29A37EBF096D2F8A9B1E176D6487FE42AE
-
-# Email sent directly to teor, verified using relay contact info
-151.80.42.103:9030 orport=9001 id=9007C1D8E4F03D506A4A011B907A9E8D04E3C605 ipv6=[2001:41d0:e:f67::114]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.39.92.199:80 orport=443 id=0BEA4A88D069753218EAAAD6D22EA87B9A1319D6 ipv6=[2001:41d0:8:b1c7::1]:443
-
-# Email sent directly to teor, verified using relay contact info
-176.31.159.231:80 orport=443 id=D5DBCC0B4F029F80C7B8D33F20CF7D97F0423BB1
-176.31.159.230:80 orport=443 id=631748AFB41104D77ADBB7E5CD4F8E8AE876E683
-195.154.79.128:80 orport=443 id=C697612CA5AED06B8D829FCC6065B9287212CB2F
-195.154.9.161:80 orport=443 id=B6295A9960F89BD0C743EEBC5670450EA6A34685
-46.148.18.74:8080 orport=443 id=6CACF0B5F03C779672F3C5C295F37C8D234CA3F7
-
-# Email sent directly to teor, verified using relay contact info
-37.187.102.108:80 orport=443 id=F4263275CF54A6836EE7BD527B1328836A6F06E1 ipv6=[2001:41d0:a:266c::1]:443 # EvilMoe
-212.47.241.21:80 orport=443 id=892F941915F6A0C6E0958E52E0A9685C190CF45C # EvilMoe
-
-# Email sent directly to teor, verified using relay contact info
-212.129.38.254:9030 orport=9001 id=FDF845FC159C0020E2BDDA120C30C5C5038F74B4
-
-# Email sent directly to teor
-37.157.195.87:8030 orport=443 id=12FD624EE73CEF37137C90D38B2406A66F68FAA2 # thanatosCZ
-5.189.169.190:8030 orport=8080 id=8D79F73DCD91FC4F5017422FAC70074D6DB8DD81 # thanatosDE
-
-# Email sent directly to teor, verified using relay contact info
-37.187.7.74:80 orport=443 id=AEA43CB1E47BE5F8051711B2BF01683DB1568E05 ipv6=[2001:41d0:a:74a::1]:443
-
-# Email sent directly to teor, verified using relay contact info
-185.66.250.141:9030 orport=9001 id=B1726B94885CE3AC3910CA8B60622B97B98E2529
-
-# Email sent directly to teor, verified using relay contact info
-185.104.120.7:9030 orport=443 id=445F1C853966624FB3CF1E12442570DC553CC2EC ipv6=[2a06:3000::120:7]:443
-185.104.120.2:9030 orport=21 id=518FF8708698E1DA09C823C36D35DF89A2CAD956
-185.104.120.4:9030 orport=9001 id=F92B3CB9BBE0CB22409843FB1AE4DBCD5EFAC835
-185.104.120.3:9030 orport=21 id=707C1B61AC72227B34487B56D04BAA3BA1179CE8 ipv6=[2a06:3000::120:3]:21
-
-# Email sent directly to teor, verified using relay contact info
-37.187.102.186:9030 orport=9001 id=489D94333DF66D57FFE34D9D59CC2D97E2CB0053 ipv6=[2001:41d0:a:26ba::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-198.96.155.3:8080 orport=5001 id=BCEDF6C193AA687AE471B8A22EBF6BC57C2D285E
-
-# Email sent directly to teor, verified using relay contact info
-212.83.154.33:8888 orport=443 id=3C79699D4FBC37DE1A212D5033B56DAE079AC0EF
-212.83.154.33:8080 orport=8443 id=322C6E3A973BC10FC36DE3037AD27BC89F14723B
-
-# Email sent directly to teor, verified using relay contact info
-51.255.41.65:9030 orport=9001 id=9231DF741915AA1630031A93026D88726877E93A
-
-# Email sent directly to teor, verified using relay contact info
-78.142.142.246:80 orport=443 id=5A5E03355C1908EBF424CAF1F3ED70782C0D2F74
-
-# Email sent directly to teor, verified using relay contact info
-195.154.97.91:80 orport=443 id=BD33C50D50DCA2A46AAED54CA319A1EFEBF5D714
-
-# Email sent directly to teor, verified using relay contact info
-62.210.129.246:80 orport=443 id=79E169B25E4C7CE99584F6ED06F379478F23E2B8
-
-# Email sent directly to teor, verified using relay contact info
-5.196.74.215:9030 orport=9001 id=5818055DFBAF0FA7F67E8125FD63E3E7F88E28F6
-
-# Email sent directly to teor, verified using relay contact info
-212.47.233.86:9030 orport=9001 id=B4CAFD9CBFB34EC5DAAC146920DC7DFAFE91EA20
-
-# Email sent directly to teor, verified using relay contact info
-85.214.206.219:9030 orport=9001 id=98F8D5F359949E41DE8DF3DBB1975A86E96A84A0
-
-# Email sent directly to teor, verified using relay contact info
-46.166.170.4:80 orport=443 id=19F42DB047B72C7507F939F5AEA5CD1FA4656205
-46.166.170.5:80 orport=443 id=DA705AD4591E7B4708FA2CAC3D53E81962F3E6F6
-
-# Email sent directly to teor, verified using relay contact info
-5.189.157.56:80 orport=443 id=77F6D6A6B6EAFB8F5DADDC07A918BBF378ED6725
-
-# Email sent directly to teor, verified using relay contact info
-46.28.110.244:80 orport=443 id=9F7D6E6420183C2B76D3CE99624EBC98A21A967E
-185.13.39.197:80 orport=443 id=001524DD403D729F08F7E5D77813EF12756CFA8D
-95.130.12.119:80 orport=443 id=587E0A9552E4274B251F29B5B2673D38442EE4BF
-
-# Email sent directly to teor, verified using relay contact info
-212.129.62.232:80 orport=443 id=B143D439B72D239A419F8DCE07B8A8EB1B486FA7
-
-# Email sent directly to teor, verified using relay contact info
-91.219.237.229:80 orport=443 id=1ECD73B936CB6E6B3CD647CC204F108D9DF2C9F7
-
-# Email sent directly to teor, verified using relay contact info
-46.101.151.222:80 orport=443 id=1DBAED235E3957DE1ABD25B4206BE71406FB61F8
-178.62.60.37:80 orport=443 id=175921396C7C426309AB03775A9930B6F611F794
-
-# Email sent directly to teor, verified using relay contact info
-178.62.197.82:80 orport=443 id=0D3EBA17E1C78F1E9900BABDB23861D46FCAF163
-
-# Email sent directly to teor, verified using relay contact info
-82.223.21.74:9030 orport=9001 id=7A32C9519D80CA458FC8B034A28F5F6815649A98 ipv6=[2001:470:53e0::cafe]:9050
-
-# Email sent directly to teor, verified using relay contact info
-146.185.177.103:80 orport=9030 id=9EC5E097663862DF861A18C32B37C5F82284B27D
-
-# Email sent directly to teor, verified using relay contact info
-37.187.22.87:9030 orport=9001 id=36B9E7AC1E36B62A9D6F330ABEB6012BA7F0D400 ipv6=[2001:41d0:a:1657::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-37.59.46.159:9030 orport=9001 id=CBD0D1BD110EC52963082D839AC6A89D0AE243E7
-
-# Email sent directly to teor, verified using relay contact info
-212.47.250.243:9030 orport=9001 id=5B33EDBAEA92F446768B3753549F3B813836D477
-# Confirm with operator before adding these
-#163.172.133.36:9030 orport=9001 id=D8C2BD36F01FA86F4401848A0928C4CB7E5FDFF9
-#158.69.216.70:9030 orport=9001 id=0ACE25A978D4422C742D6BC6345896719BF6A7EB
-
-# Email sent directly to teor, verified using relay contact info
-5.199.142.236:9030 orport=9001 id=F4C0EDAA0BF0F7EC138746F8FEF1CE26C7860265
-
-# Email sent directly to teor
-188.166.133.133:9030 orport=9001 id=774555642FDC1E1D4FDF2E0C31B7CA9501C5C9C7 ipv6=[2a03:b0c0:2:d0::26c0:1]:9001 # dropsy
-
-# Email sent directly to teor, verified using relay contact info
-46.8.249.10:80 orport=443 id=31670150090A7C3513CB7914B9610E786391A95D
-
-# Email sent directly to teor, verified using relay contact info
-144.76.163.93:9030 orport=9001 id=22F08CF09764C4E8982640D77F71ED72FF26A9AC
-
-# Email sent directly to teor, verified using relay contact info
-46.4.24.161:9030 orport=9001 id=DB4C76A3AD7E234DA0F00D6F1405D8AFDF4D8DED
-46.4.24.161:9031 orport=9002 id=7460F3D12EBE861E4EE073F6233047AACFE46AB4
-46.38.51.132:9030 orport=9001 id=810DEFA7E90B6C6C383C063028EC397A71D7214A
-163.172.194.53:9030 orport=9001 id=8C00FA7369A7A308F6A137600F0FA07990D9D451 ipv6=[2001:bc8:225f:142:6c69:7461:7669:73]:9001
-
-# Email sent directly to teor, verified using relay contact info
-176.10.107.180:9030 orport=9001 id=3D7E274A87D9A89AF064C13D1EE4CA1F184F2600
-
-# Email sent directly to teor, verified using relay contact info
-46.28.207.19:80 orport=443 id=5B92FA5C8A49D46D235735504C72DBB3472BA321
-46.28.207.141:80 orport=443 id=F69BED36177ED727706512BA6A97755025EEA0FB
-46.28.205.170:80 orport=443 id=AF322D83A4D2048B22F7F1AF5F38AFF4D09D0B76
-95.183.48.12:80 orport=443 id=7187CED1A3871F837D0E60AC98F374AC541CB0DA
-
-# Email sent directly to teor, verified using relay contact info
-93.180.156.84:9030 orport=9001 id=8844D87E9B038BE3270938F05AF797E1D3C74C0F
-
-# Email sent directly to teor, verified using relay contact info
-37.187.115.157:9030 orport=9001 id=D5039E1EBFD96D9A3F9846BF99EC9F75EDDE902A
-
-# Email sent directly to teor, verified using relay contact info
-5.34.183.205:80 orport=443 id=DDD7871C1B7FA32CB55061E08869A236E61BDDF8
-
-# Email sent directly to teor, verified using relay contact info
-51.254.246.203:9030 orport=9001 id=47B596B81C9E6277B98623A84B7629798A16E8D5
-
-# Email sent directly to teor, verified using relay contact info
-5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
-
-# Email sent directly to teor, verified using relay contact info
-# Updated details from atlas based on ticket #20010
-163.172.176.167:80 orport=443 id=230A8B2A8BA861210D9B4BA97745AEC217A94207
-163.172.149.155:80 orport=443 id=0B85617241252517E8ECF2CFC7F4C1A32DCD153F
-163.172.149.122:80 orport=443 id=A9406A006D6E7B5DA30F2C6D4E42A338B5E340B2
-
-# Email sent directly to teor, verified using relay contact info
-204.11.50.131:9030 orport=9001 id=185F2A57B0C4620582602761097D17DB81654F70
-
-# Email sent directly to teor, verified using relay contact info
-151.236.222.217:44607 orport=9001 id=94D58704C2589C130C9C39ED148BD8EA468DBA54
-
-# Email sent directly to teor, verified using relay contact info
-185.35.202.221:9030 orport=9001 id=C13B91384CDD52A871E3ECECE4EF74A7AC7DCB08 ipv6=[2a02:ed06::221]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.9.151.241:9030 orport=4223 id=9BF04559224F0F1C3C953D641F1744AF0192543A ipv6=[2a01:4f8:190:34f0::2]:4223
-
-# Email sent directly to teor, verified using relay contact info
-89.40.71.149:8081 orport=8080 id=EC639EDAA5121B47DBDF3D6B01A22E48A8CB6CC7
-
-# Email sent directly to teor, verified using relay contact info
-92.222.20.130:80 orport=443 id=0639612FF149AA19DF3BCEA147E5B8FED6F3C87C
-
-# Email sent directly to teor, verified using relay contact info
-80.112.155.100:9030 orport=9001 id=53B000310984CD86AF47E5F3CD0BFF184E34B383 ipv6=[2001:470:7b02::38]:9001
-
-# Email sent directly to teor, verified using relay contact info
-83.212.99.68:80 orport=443 id=DDBB2A38252ADDA53E4492DDF982CA6CC6E10EC0 ipv6=[2001:648:2ffc:1225:a800:bff:fe3d:67b5]:443
-
-# Email sent directly to teor, verified using relay contact info
-95.130.11.147:9030 orport=443 id=6B697F3FF04C26123466A5C0E5D1F8D91925967A
-
-# Email sent directly to teor, verified using relay contact info
-128.199.55.207:9030 orport=9001 id=BCEF908195805E03E92CCFE669C48738E556B9C5 ipv6=[2a03:b0c0:2:d0::158:3001]:9001
-
-# Email sent directly to teor, verified using relay contact info
-178.32.216.146:9030 orport=9001 id=17898F9A2EBC7D69DAF87C00A1BD2FABF3C9E1D2
-
-# Email sent directly to teor, verified using relay contact info
-212.83.40.238:9030 orport=9001 id=F409FA7902FD89270E8DE0D7977EA23BC38E5887
-
-# Email sent directly to teor, verified using relay contact info
-204.8.156.142:80 orport=443 id=94C4B7B8C50C86A92B6A20107539EE2678CF9A28
-
-# Email sent directly to teor, verified using relay contact info
-80.240.139.111:80 orport=443 id=DD3BE7382C221F31723C7B294310EF9282B9111B
-
-# Email sent directly to teor, verified using relay contact info
-185.97.32.18:9030 orport=9001 id=04250C3835019B26AA6764E85D836088BE441088
-
-# Email sent directly to teor
-149.56.45.200:9030 orport=9001 id=FE296180018833AF03A8EACD5894A614623D3F76 ipv6=[2607:5300:201:3000::17d3]:9002 # PiotrTorpotkinOne
-
-# Email sent directly to teor, verified using relay contact info
-81.2.209.10:443 orport=80 id=B6904ADD4C0D10CDA7179E051962350A69A63243 ipv6=[2001:15e8:201:1::d10a]:80
-
-# Email sent directly to teor, verified using relay contact info
-# IPv6 address unreliable
-195.154.164.243:80 orport=443 id=AC66FFA4AB35A59EBBF5BF4C70008BF24D8A7A5C #ipv6=[2001:bc8:399f:f000::1]:993
-138.201.26.2:80 orport=443 id=6D3A3ED5671E4E3F58D4951438B10AE552A5FA0F
-81.7.16.182:80 orport=443 id=51E1CF613FD6F9F11FE24743C91D6F9981807D82 ipv6=[2a02:180:1:1::517:10b6]:993
-134.119.36.135:80 orport=443 id=763C9556602BD6207771A7A3D958091D44C43228 ipv6=[2a00:1158:3::2a8]:993
-46.228.199.19:80 orport=443 id=E26AFC5F718E21AC502899B20C653AEFF688B0D2 ipv6=[2001:4ba0:cafe:4a::1]:993
-37.200.98.5:80 orport=443 id=231C2B9C8C31C295C472D031E06964834B745996 ipv6=[2a00:1158:3::11a]:993
-46.23.70.195:80 orport=443 id=C9933B3725239B6FAB5227BA33B30BE7B48BB485
-185.15.244.124:80 orport=443 id=935BABE2564F82016C19AEF63C0C40B5753BA3D2 ipv6=[2001:4ba0:cafe:e35::1]:993
-195.154.116.232:80 orport=443 id=B35C5739C8C5AB72094EB2B05738FD1F8EEF6EBD ipv6=[2001:bc8:399f:200::1]:993
-195.154.121.198:80 orport=443 id=0C77421C890D16B6D201283A2244F43DF5BC89DD ipv6=[2001:bc8:399f:100::1]:993
-37.187.20.59:80 orport=443 id=91D23D8A539B83D2FB56AA67ECD4D75CC093AC55 ipv6=[2001:41d0:a:143b::1]:993
-217.12.208.117:80 orport=443 id=E6E18151300F90C235D3809F90B31330737CEB43 ipv6=[2a00:1ca8:a7::1bb]:993
-81.7.10.251:80 orport=443 id=8073670F8F852971298F8AF2C5B23AE012645901 ipv6=[2a02:180:1:1::517:afb]:993
-46.36.39.50:80 orport=443 id=ED4B0DBA79AEF5521564FA0231455DCFDDE73BB6 ipv6=[2a02:25b0:aaaa:aaaa:8d49:b692:4852:0]:995
-91.194.90.103:80 orport=443 id=75C4495F4D80522CA6F6A3FB349F1B009563F4B7 ipv6=[2a02:c205:3000:5449::1]:993
-163.172.25.118:80 orport=22 id=0CF8F3E6590F45D50B70F2F7DA6605ECA6CD408F
-188.138.88.42:80 orport=443 id=70C55A114C0EF3DC5784A4FAEE64388434A3398F
-81.7.13.84:80 orport=443 id=0C1E7DD9ED0676C788933F68A9985ED853CA5812 ipv6=[2a02:180:1:1::5b8f:538c]:993
-213.246.56.95:80 orport=443 id=27E6E8E19C46751E7312420723C6162FF3356A4C ipv6=[2a00:c70:1:213:246:56:95:1]:993
-94.198.100.18:80 orport=443 id=BAACCB29197DB833F107E410E2BFAE5009EE7583
-217.12.203.46:80 orport=443 id=6A29FD8C00D573E6C1D47852345B0E5275BA3307
-212.117.180.107:80 orport=443 id=0B454C7EBA58657B91133A587C1BDAEDC6E23142
-217.12.199.190:80 orport=443 id=A37C47B03FF31CA6937D3D68366B157997FE7BCD ipv6=[2a02:27a8:0:2::486]:993
-216.230.230.247:80 orport=443 id=4C7BF55B1BFF47993DFF995A2926C89C81E4F04A
-69.30.215.42:80 orport=443 id=510176C07005D47B23E6796F02C93241A29AA0E9 ipv6=[2604:4300:a:2e::2]:993
-89.46.100.162:80 orport=443 id=6B7191639E179965FD694612C9B2C8FB4267B27D
-107.181.174.22:80 orport=443 id=5A551BF2E46BF26CC50A983F7435CB749C752553 ipv6=[2607:f7a0:3:4::4e]:993
-
-# Email sent directly to teor, verified using relay contact info
-212.238.208.48:9030 orport=9001 id=F406219CDD339026D160E53FCA0EF6857C70F109 ipv6=[2001:984:a8fb:1:ba27:ebff:feac:c109]:9001
-
-# Email sent directly to teor
-176.158.236.102:9030 orport=9001 id=DC163DDEF4B6F0C6BC226F9F6656A5A30C5C5686 # Underworld
-
-# Email sent directly to teor, verified using relay contact info
-91.229.20.27:9030 orport=9001 id=9A0D54D3A6D2E0767596BF1515E6162A75B3293F
-
-# Email sent directly to teor, verified using relay contact info
-80.127.137.19:80 orport=443 id=6EF897645B79B6CB35E853B32506375014DE3621 ipv6=[2001:981:47c1:1::6]:443
-
-# Email sent directly to teor
-163.172.138.22:80 orport=443 id=16102E458460349EE45C0901DAA6C30094A9BBEA ipv6=[2001:bc8:4400:2100::1:3]:443 # mkultra
-
-# Email sent directly to teor, verified using relay contact info
-97.74.237.196:9030 orport=9001 id=2F0F32AB1E5B943CA7D062C03F18960C86E70D94
-
-# Email sent directly to teor, verified using relay contact info
-192.187.124.98:9030 orport=9001 id=FD1871854BFC06D7B02F10742073069F0528B5CC
-
-# Email sent directly to teor, verified using relay contact info
-178.62.98.160:9030 orport=9001 id=8B92044763E880996A988831B15B2B0E5AD1544A
-
-# Email sent directly to teor, verified using relay contact info
-163.172.217.50:9030 orport=9001 id=02ECD99ECD596013A8134D46531560816ECC4BE6
-
-# Email sent directly to teor, verified using relay contact info
-185.100.86.100:80 orport=443 id=0E8C0C8315B66DB5F703804B3889A1DD66C67CE0
-185.100.84.82:80 orport=443 id=7D05A38E39FC5D29AFE6BE487B9B4DC9E635D09E
-
-# Email sent directly to teor, verified using relay contact info
-164.132.77.175:9030 orport=9001 id=3B33F6FCA645AD4E91428A3AF7DC736AD9FB727B
-78.24.75.53:9030 orport=9001 id=DEB73705B2929AE9BE87091607388939332EF123
-
-# Email sent directly to teor, verified using relay contact info
-46.101.237.246:9030 orport=9001 id=75F1992FD3F403E9C082A5815EB5D12934CDF46C ipv6=[2a03:b0c0:3:d0::208:5001]:9050
-178.62.86.96:9030 orport=9001 id=439D0447772CB107B886F7782DBC201FA26B92D1 ipv6=[2a03:b0c0:1:d0::3cf:7001]:9050
-
-# Email sent directly to teor, verified using relay contact info
-# Very low bandwidth, stale consensues, excluded to cut down on warnings
-#91.233.106.121:80 orport=443 id=896364B7996F5DFBA0E15D1A2E06D0B98B555DD6
-
-# Email sent directly to teor, verified using relay contact info
-167.114.113.48:9030 orport=403 id=2EC0C66EA700C44670444280AABAB1EC78B722A0
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-213.141.138.174:9030 orport=9001 id=BD552C165E2ED2887D3F1CCE9CFF155DDA2D86E6 # Schakalium
-
-# Email sent directly to teor, verified using relay contact info
-95.128.43.164:80 orport=443 id=616081EC829593AF4232550DE6FFAA1D75B37A90 ipv6=[2a02:ec0:209:10::4]:443
-
-# Email sent directly to teor, verified using relay contact info
-166.82.21.200:9030 orport=9029 id=D5C33F3E203728EDF8361EA868B2939CCC43FAFB
-
-# Email sent directly to teor, verified using relay contact info
-91.121.54.8:9030 orport=9001 id=CBEE0F3303C8C50462A12107CA2AE061831931BC
-
-# Email sent directly to teor, verified using relay contact info
-178.217.184.32:8080 orport=443 id=8B7F47AE1A5D954A3E58ACDE0865D09DBA5B738D
-
-# Email sent directly to teor, verified using relay contact info
-85.10.201.47:9030 orport=9001 id=D8B7A3A6542AA54D0946B9DC0257C53B6C376679 ipv6=[2a01:4f8:a0:43eb::beef]:9001
-
-# Email sent directly to teor, verified using relay contact info
-120.29.217.46:80 orport=443 id=5E853C94AB1F655E9C908924370A0A6707508C62
-
-# Email sent directly to teor, verified using relay contact info
-37.153.1.10:9030 orport=9001 id=9772EFB535397C942C3AB8804FB35CFFAD012438
-
-# Email sent directly to teor, verified using relay contact info
-92.222.4.102:9030 orport=9001 id=1A6B8B8272632D8AD38442027F822A367128405C
-
-# Email sent directly to teor, verified using relay contact info
-31.31.78.49:80 orport=443 id=46791D156C9B6C255C2665D4D8393EC7DBAA7798
-
-# Email sent directly to teor
-192.160.102.169:80 orport=9001 id=C0192FF43E777250084175F4E59AC1BA2290CE38 ipv6=[2620:132:300c:c01d::9]:9002 # manipogo
-192.160.102.166:80 orport=9001 id=547DA56F6B88B6C596B3E3086803CDA4F0EF8F21 ipv6=[2620:132:300c:c01d::6]:9002 # chaucer
-192.160.102.170:80 orport=9001 id=557ACEC850F54EEE65839F83CACE2B0825BE811E ipv6=[2620:132:300c:c01d::a]:9002 # ogopogo
-192.160.102.164:80 orport=9001 id=823AA81E277F366505545522CEDC2F529CE4DC3F ipv6=[2620:132:300c:c01d::4]:9002 # snowfall
-192.160.102.165:80 orport=9001 id=C90CA3B7FE01A146B8268D56977DC4A2C024B9EA ipv6=[2620:132:300c:c01d::5]:9002 # cowcat
-192.160.102.168:80 orport=9001 id=F6A358DD367B3282D6EF5824C9D45E1A19C7E815 ipv6=[2620:132:300c:c01d::8]:9002 # prawksi
-
-# Email sent directly to teor, verified using relay contact info
-136.243.214.137:80 orport=443 id=B291D30517D23299AD7CEE3E60DFE60D0E3A4664
-
-# Email sent directly to teor, verified using relay contact info
-192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
-192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
-
-# Email sent directly to teor, verified using relay contact info
-192.87.28.28:9030 orport=9001 id=ED2338CAC2711B3E331392E1ED2831219B794024
-# same machine as ED2338CAC2711B3E331392E1ED2831219B794024
-192.87.28.82:9030 orport=9001 id=844AE9CAD04325E955E2BE1521563B79FE7094B7
-
-# https://twitter.com/kosjoli/status/719507270904758272
-85.10.202.87:9030 orport=9001 id=971AFB23C168DCD8EDA17473C1C452B359DE3A5A
-176.9.5.116:9030 orport=9001 id=A1EB8D8F1EE28DB98BBB1EAA3B4BEDD303BAB911
-46.4.111.124:9030 orport=9001 id=D9065F9E57899B3D272AA212317AF61A9B14D204
-
-# Email sent directly to teor, verified using relay contact info
-185.100.85.61:80 orport=443 id=025B66CEBC070FCB0519D206CF0CF4965C20C96E
-
-# Email sent directly to teor, verified using relay contact info
-108.166.168.158:80 orport=443 id=CDAB3AE06A8C9C6BF817B3B0F1877A4B91465699
-
-# Email sent directly to teor, verified using relay contact info
-91.219.236.222:80 orport=443 id=20704E7DD51501DC303FA51B738D7B7E61397CF6
-
-# Email sent directly to teor, verified using relay contact info
-185.14.185.240:9030 orport=443 id=D62FB817B0288085FAC38A6DC8B36DCD85B70260
-192.34.63.137:9030 orport=443 id=ABCB4965F1FEE193602B50A365425105C889D3F8
-128.199.197.16:9030 orport=443 id=DEE5298B3BA18CDE651421CD2DCB34A4A69F224D
-
-# Email sent directly to teor, verified using relay contact info
-185.13.38.75:9030 orport=9001 id=D2A1703758A0FBBA026988B92C2F88BAB59F9361
-
-# Email sent directly to teor, verified using relay contact info
-128.204.39.106:9030 orport=9001 id=6F0F3C09AF9580F7606B34A7678238B3AF7A57B7
-
-# Email sent directly to teor, verified using relay contact info
-198.50.191.95:80 orport=443 id=39F096961ED2576975C866D450373A9913AFDC92
-
-# Email sent directly to teor, verified using relay contact info
-167.114.66.61:9696 orport=443 id=DE6CD5F09DF26076F26321B0BDFBE78ACD935C65 ipv6=[2607:5300:100::78d]:443
-
-# Email sent directly to teor, verified using relay contact info
-66.111.2.20:9030 orport=9001 id=9A68B85A02318F4E7E87F2828039FBD5D75B0142
-66.111.2.16:9030 orport=9001 id=3F092986E9B87D3FDA09B71FA3A602378285C77A
-
-# Email sent directly to teor, verified using relay contact info
-92.222.38.67:80 orport=443 id=DED6892FF89DBD737BA689698A171B2392EB3E82
-
-# Email sent directly to teor, verified using relay contact info
-212.47.228.115:9030 orport=443 id=BCA017ACDA48330D02BB70716639ED565493E36E
-
-# Email sent directly to teor, verified using relay contact info
-185.100.84.175:80 orport=443 id=39B59AF4FE54FAD8C5085FA9C15FDF23087250DB
-
-# Email sent directly to teor, verified using relay contact info
-166.70.207.2:9030 orport=9001 id=E3DB2E354B883B59E8DC56B3E7A353DDFD457812
-
-# Emails sent directly to teor, verified using relay contact info
-69.162.139.9:9030 orport=9001 id=4791FC0692EAB60DF2BCCAFF940B95B74E7654F6 ipv6=[2607:f128:40:1212::45a2:8b09]:9001
-
-# Email sent directly to teor, verified using relay contact info
-213.239.217.18:1338 orport=1337 id=C37BC191AC389179674578C3E6944E925FE186C2 ipv6=[2a01:4f8:a0:746a:101:1:1:1]:1337
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-188.40.128.246:9030 orport=9001 id=AD19490C7DBB26D3A68EFC824F67E69B0A96E601 ipv6=[2a01:4f8:221:1ac1:dead:beef:7005:9001]:9001 # sputnik
-
-# Email sent directly to teor, verified using relay contact info
-88.198.253.13:9030 orport=9001 id=DF924196D69AAE3C00C115A9CCDF7BB62A175310 ipv6=[2a01:4f8:11a:b1f::2]:9001
-
-# Email sent directly to teor, verified using relay contact info
-185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
-46.36.36.127:9030 orport=9001 id=C80DF89B21FF932DEC0D7821F679B6C79E1449C3
-
-# Email sent directly to teor, verified using relay contact info
-176.10.104.240:80 orport=443 id=0111BA9B604669E636FFD5B503F382A4B7AD6E80
-176.10.104.240:8080 orport=8443 id=AD86CD1A49573D52A7B6F4A35750F161AAD89C88
-176.10.104.243:80 orport=443 id=88487BDD980BF6E72092EE690E8C51C0AA4A538C
-176.10.104.243:8080 orport=8443 id=95DA61AEF23A6C851028C1AA88AD8593F659E60F
-
-# Email sent directly to teor, verified using relay contact info
-107.170.101.39:9030 orport=443 id=30973217E70AF00EBE51797FF6D9AA720A902EAA
-
-# Email sent directly to teor
-193.70.112.165:80 orport=443 id=F10BDE279AE71515DDCCCC61DC19AC8765F8A3CC # ParkBenchInd001
-
-# Email sent directly to teor
-185.220.101.6:10006 orport=20006 id=C08DE49658E5B3CFC6F2A952B453C4B608C9A16A # niftyvolcanorabbit
-185.220.101.13:10013 orport=20013 id=71AB4726D830FAE776D74AEF790CF04D8E0151B4 # niftycottontail
-185.220.101.5:10005 orport=20005 id=1084200B44021D308EA4253F256794671B1D099A # niftyhedgehog
-185.220.101.9:10009 orport=20009 id=14877C6384A9E793F422C8D1DDA447CACA4F7C4B # niftywoodmouse
-185.220.101.8:10008 orport=20008 id=24E91955D969AEA1D80413C64FE106FAE7FD2EA9 # niftymouse
-185.220.101.1:10001 orport=20001 id=28F4F392F8F19E3FBDE09616D9DB8143A1E2DDD3 # niftycottonmouse
-185.220.101.21:10021 orport=20021 id=348B89013EDDD99E4755951D1EC284D9FED71226 # niftysquirrel
-185.220.101.10:10010 orport=20010 id=4031460683AE9E0512D3620C2758D98758AC6C93 # niftyeuropeanrabbit
-185.220.101.34:10034 orport=20034 id=47C42E2094EE482E7C9B586B10BABFB67557030B # niftyquokka
-185.220.101.18:10018 orport=20018 id=5D5006E4992F2F97DF4F8B926C3688870EB52BD8 # niftyplagiodontia
-185.220.101.28:10028 orport=20028 id=609E598FB6A00BCF7872906B602B705B64541C50 # niftychipmunk
-185.220.101.20:10020 orport=20020 id=619349D82424C601CAEB94161A4CF778993DAEE7 # niftytucotuco
-185.220.101.17:10017 orport=20017 id=644DECC5A1879C0FE23DE927DD7049F58BBDF349 # niftyhutia
-185.220.101.0:10000 orport=20000 id=6E94866ED8CA098BACDFD36D4E8E2B459B8A734E # niftybeaver
-185.220.101.30:10030 orport=20030 id=71CFDEB4D9E00CCC3E31EC4E8A29E109BBC1FB36 # niftypedetidae
-185.220.101.29:10029 orport=20029 id=7DC52AE6667A30536BA2383CD102CFC24F20AD71 # niftyllipika
-185.220.101.41:10041 orport=20041 id=7E281CD2C315C4F7A84BC7C8721C3BC974DDBFA3 # niftyporcupine
-185.220.101.25:10025 orport=20025 id=8EE0534532EA31AA5172B1892F53B2F25C76EB02 # niftyjerboa
-185.220.101.33:10033 orport=20033 id=906DCB390F2BA987AE258D745E60BAAABAD31DE8 # niftyquokka
-185.220.101.26:10026 orport=20026 id=92A6085EABAADD928B6F8E871540A1A41CBC08BA # niftypedetes
-185.220.101.40:10040 orport=20040 id=9A857254F379194D1CD76F4A79A20D2051BEDA3F # niftynutria
-185.220.101.42:10042 orport=20042 id=9B816A5B3EB20B8E4E9B9D1FBA299BD3F40F0320 # niftypygmyjerboa
-185.220.101.2:10002 orport=20002 id=B740BCECC4A9569232CDD45C0E1330BA0D030D33 # niftybunny
-185.220.101.32:10032 orport=20032 id=B771AA877687F88E6F1CA5354756DF6C8A7B6B24 # niftypika
-185.220.101.12:10012 orport=20012 id=BC82F2190DE2E97DE65F49B4A95572374BDC0789 # niftycapybara
-185.220.101.22:10022 orport=20022 id=CA37CD46799449D83B6B98B8C22C649906307888 # niftyjackrabbit
-185.220.101.4:10004 orport=20004 id=CDA2EA326E2272C57ACB26773D7252C211795B78 # niftygerbil
-185.220.101.14:10014 orport=20014 id=E7EBA5D8A4E09684D11A1DF24F75362817333768 # niftyhare
-185.220.101.16:10016 orport=20016 id=EC1997D51892E4607C68E800549A1E7E4694005A # niftyguineapig
-185.220.101.24:10024 orport=20024 id=FDA70EC93DB01E3CB418CB6943B0C68464B18B4C # niftyrat
-
-# Email sent directly to teor, verified using relay contact info
-64.113.32.29:9030 orport=9001 id=30C19B81981F450C402306E2E7CFB6C3F79CB6B2
-
-# Emails sent directly to teor, verified using relay contact info
-51.254.101.242:9002 orport=9001 id=4CC9CC9195EC38645B699A33307058624F660CCF
-
-# Emails sent directly to teor, verified using relay contact info
-85.214.62.48:80 orport=443 id=6A7551EEE18F78A9813096E82BF84F740D32B911
-
-# Email sent directly to teor, verified using relay contact info
-173.255.245.116:9030 orport=9001 id=91E4015E1F82DAF0121D62267E54A1F661AB6DC7
-
-# Email sent directly to teor, verified using relay contact info
-62.216.5.120:9030 orport=9001 id=D032D4D617140D6B828FC7C4334860E45E414FBE
-
-# Email sent directly to teor, verified using relay contact info
-51.254.136.195:80 orport=443 id=7BB70F8585DFC27E75D692970C0EEB0F22983A63
-
-# Email sent directly to teor, verified using relay contact info
-163.172.13.165:9030 orport=9001 id=33DA0CAB7C27812EFF2E22C9705630A54D101FEB ipv6=[2001:bc8:38cb:201::8]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.196.88.122:9030 orport=9001 id=0C2C599AFCB26F5CFC2C7592435924C1D63D9484 ipv6=[2001:41d0:a:fb7a::1]:9001
-
-# Email sent directly to teor, verified using relay contact info
-5.9.158.75:80 orport=443 id=1AF72E8906E6C49481A791A6F8F84F8DFEBBB2BA ipv6=[2a01:4f8:190:514a::2]:443
-
-# Email sent directly to teor, verified using relay contact info
-46.101.169.151:9030 orport=9001 id=D760C5B436E42F93D77EF2D969157EEA14F9B39C ipv6=[2a03:b0c0:3:d0::74f:a001]:9001
-
-# Email sent directly to teor, verified using relay contact info
-199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3
-199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6
-199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29
-199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813
-199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338
-199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060
-199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C
-199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616
-199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971
-199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241
-199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0
-199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD
-199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887
-199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B
-199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A
-199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F
-199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2
-199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0
-199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51
-199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-December/011114.html
-86.105.212.130:9030 orport=443 id=9C900A7F6F5DD034CFFD192DAEC9CCAA813DB022
-
-# Email sent directly to teor, verified using relay contact info
-178.33.183.251:80 orport=443 id=DD823AFB415380A802DCAEB9461AE637604107FB ipv6=[2001:41d0:2:a683::251]:443
-
-# Email sent directly to teor, verified using relay contact info
-31.185.104.19:80 orport=443 id=9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
-# same machine as 9EAD5B2D3DBD96DBC80DCE423B0C345E920A758D
-31.185.104.20:80 orport=443 id=ADB2C26629643DBB9F8FE0096E7D16F9414B4F8D
-31.185.104.21:80 orport=443 id=C2AAB088555850FC434E68943F551072042B85F1
-31.185.104.22:80 orport=443 id=5BA3A52760A0EABF7E7C3ED3048A77328FF0F148
-
-# Email sent directly to teor, verified using relay contact info
-185.34.60.114:80 orport=443 id=7F7A695DF6F2B8640A70B6ADD01105BC2EBC5135
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013939.html
-94.142.242.84:80 orport=443 id=AA0D167E03E298F9A8CD50F448B81FBD7FA80D56 ipv6=[2a02:898:24:84::1]:443 # rejozenger
-
-# Email sent directly to teor, verified using relay contact info
-185.129.62.62:9030 orport=9001 id=ACDD9E85A05B127BA010466C13C8C47212E8A38F ipv6=[2a06:d380:0:3700::62]:9001
-
-# Email sent directly to teor, verified using relay contact info
-# The e84 part of the IPv6 address does not have a leading 0 in the consensus
-81.30.158.213:9030 orport=9001 id=789EA6C9AE9ADDD8760903171CFA9AC5741B0C70 ipv6=[2001:4ba0:cafe:e84::1]:9001
-
-# https://lists.torproject.org/pipermail/tor-relays/2016-December/011209.html
-5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
-
-# Email sent directly to teor, verified using relay contact info
-45.62.255.25:80 orport=443 id=3473ED788D9E63361D1572B7E82EC54338953D2A
-
-# Email sent directly to teor, verified using relay contact info
-217.79.179.177:9030 orport=9001 id=3E53D3979DB07EFD736661C934A1DED14127B684 ipv6=[2001:4ba0:fff9:131:6c4f::90d3]:9001
-
-# Email sent directly to teor, verified using relay contact info
-212.47.244.38:8080 orport=443 id=E81EF60A73B3809F8964F73766B01BAA0A171E20
-163.172.157.213:8080 orport=443 id=4623A9EC53BFD83155929E56D6F7B55B5E718C24
-163.172.139.104:8080 orport=443 id=68F175CCABE727AA2D2309BCD8789499CEE36ED7
-
-# Email sent directly to teor, verified using relay contact info
-163.172.223.200:80 orport=443 id=998BF3ED7F70E33D1C307247B9626D9E7573C438
-195.154.122.54:80 orport=443 id=64E99CB34C595A02A3165484BD1215E7389322C6
-
-# Email sent directly to teor, verified using relay contact info
-185.100.86.128:9030 orport=9001 id=9B31F1F1C1554F9FFB3455911F82E818EF7C7883
-185.100.85.101:9030 orport=9001 id=4061C553CA88021B8302F0814365070AAE617270
-31.171.155.108:9030 orport=9001 id=D3E5EDDBE5159388704D6785BE51930AAFACEC6F
-
-# Email sent directly to teor, verified using relay contact info
-89.163.247.43:9030 orport=9001 id=BC7ACFAC04854C77167C7D66B7E471314ED8C410 ipv6=[2001:4ba0:fff7:25::5]:9001
-
-# Email sent directly to teor, verified using relay contact info
-95.85.8.226:80 orport=443 id=1211AC1BBB8A1AF7CBA86BCE8689AA3146B86423
-
-# Email sent directly to teor, verified using relay contact info
-85.214.151.72:9030 orport=9001 id=722D365140C8C52DBB3C9FF6986E3CEFFE2BA812
-
-# email sent directly to teor
-72.52.75.27:9030 orport=9001 id=8567AD0A6369ED08527A8A8533A5162AC00F7678 # piecoopdotnet
-
-# Email sent directly to teor, verified using relay contact info
-5.9.146.203:80 orport=443 id=1F45542A24A61BF9408F1C05E0DCE4E29F2CBA11
-5.9.159.14:9030 orport=9001 id=0F100F60C7A63BED90216052324D29B08CFCF797
-
-# Email sent directly to teor, verified using relay contact info
-# Assume details update is permanent
-5.9.147.226:9030 orport=9001 id=B0553175AADB0501E5A61FC61CEA3970BE130FF2 ipv6=[2a01:4f8:190:30e1::2]:9001 # zwiubel
-
-# https://trac.torproject.org/projects/tor/ticket/22527#comment:1
-199.184.246.250:80 orport=443 id=1F6ABD086F40B890A33C93CC4606EE68B31C9556 ipv6=[2620:124:1009:1::171]:443
-
-# https://trac.torproject.org/projects/tor/ticket/24695
-163.172.53.84:143 orport=21 id=1C90D3AEADFF3BCD079810632C8B85637924A58E ipv6=[2001:bc8:24f8::]:21 # Multivac
-
-# Email sent directly to teor
-54.36.237.163:80 orport=443 id=DB2682153AC0CCAECD2BD1E9EBE99C6815807A1E # GermanCraft2
-
-# Email sent directly to teor
-62.138.7.171:9030 orport=9001 id=9844B981A80B3E4B50897098E2D65167E6AEF127 # 0x3d004
-62.138.7.171:8030 orport=8001 id=9285B22F7953D7874604EEE2B470609AD81C74E9 # 0x3d005
-91.121.23.100:9030 orport=9001 id=3711E80B5B04494C971FB0459D4209AB7F2EA799 # 0x3d002
-91.121.23.100:8030 orport=8001 id=CFBBA0D858F02E40B1432A65F6D13C9BDFE7A46B # 0x3d001
-51.15.13.245:9030 orport=9001 id=CED527EAC230E7B56E5B363F839671829C3BA01B # 0x3d006
-51.15.13.245:8030 orport=8001 id=8EBB8D1CF48FE2AB95C451DA8F10DB6235F40F8A # 0x3d007
-
-# Email sent directly to teor
-104.192.5.248:9030 orport=9001 id=BF735F669481EE1CCC348F0731551C933D1E2278 # Freeway11
-
-# Email sent directly to teor
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013961.html
-178.17.174.14:9030 orport=9001 id=B06F093A3D4DFAD3E923F4F28A74901BD4F74EB1 # TorExitMoldova
-178.17.170.156:9030 orport=9001 id=41C59606AFE1D1AA6EC6EF6719690B856F0B6587 # TorExitMoldova2
-
-# Email sent directly to teor
-163.172.221.44:59030 orport=59001 id=164604F5C86FC8CC9C0288BD9C02311958427597 # altego
-
-# Email sent directly to teor
-46.38.237.221:9030 orport=9001 id=D30E9D4D639068611D6D96861C95C2099140B805 # mine
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013911.html
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013912.html
-199.249.223.62:80 orport=443 id=0077BCBA7244DB3E6A5ED2746E86170066684887 # Quintex13
-199.249.224.45:80 orport=443 id=041646640AB306EA74B001966E86169B04CC88D2 # QuintexAirVPN26
-199.249.223.67:80 orport=443 id=155D6F57425F16C0624D77777641E4EB1B47C6F0 # Quintex18
-199.249.223.45:80 orport=443 id=1AE949967F82BBE7534A3D6BA77A7EBE1CED4369 # Quintex36
-199.249.223.63:80 orport=443 id=1DB25DF59DAA01B5BE3D3CEB8AFED115940EBE8B # Quintex14
-199.249.224.63:80 orport=443 id=1E5136DDC52FAE1219208F0A6BADB0BA62587EE6 # Quintex43
-199.249.224.46:80 orport=443 id=2ED4D25766973713EB8C56A290BF07E06B85BF12 # QuintexAirVPN27
-199.249.223.42:80 orport=443 id=3687FEC7E73F61AC66F7AE251E7DEE6BBD8C0252 # Quintex33
-199.249.223.49:80 orport=443 id=36D68478366CB8627866757EBCE7FB3C17FC1CB8 # Quintex40
-199.249.224.49:80 orport=443 id=3CA0D15567024D2E0B557DC0CF3E962B37999A79 # QuintexAirVPN30
-199.249.223.61:80 orport=443 id=40E7D6CE5085E4CDDA31D51A29D1457EB53F12AD # Quintex12
-199.249.223.76:80 orport=443 id=43209F6D50C657A56FE79AF01CA69F9EF19BD338 # QuintexAirVPN5
-199.249.224.41:80 orport=443 id=54A4820B46E65509BF3E2B892E66930A41759DE9 # QuintexAirVPN22
-199.249.223.73:80 orport=443 id=5649CB2158DA94FB747415F26628BEC07FA57616 # QuintexAirVPN8
-199.249.223.74:80 orport=443 id=5F4CD12099AF20FAF9ADFDCEC65316A376D0201C # QuintexAirVPN7
-199.249.223.75:80 orport=443 id=60D3667F56AEC5C69CF7E8F557DB21DDF6C36060 # QuintexAirVPN6
-199.249.223.46:80 orport=443 id=66E19E8C4773086F669A1E06A3F8C23B6C079129 # Quintex37
-199.249.224.65:80 orport=443 id=764BF8A03868F84C8F323C1A676AA254B80DC3BF # Quintex45
-199.249.223.48:80 orport=443 id=7A3DD280EA4CD4DD16EF8C67B93D9BDE184D1A81 # Quintex39
-199.249.224.68:80 orport=443 id=7E6E9A6FDDB8DC7C92F0CFCC3CBE76C29F061799 # Quintex48
-199.249.223.69:80 orport=443 id=7FA8E7E44F1392A4E40FFC3B69DB3B00091B7FD3 # Quintex20
-199.249.223.44:80 orport=443 id=8B80169BEF71450FC4069A190853523B7AEA45E1 # Quintex35
-199.249.224.60:80 orport=443 id=9314BD9503B9014261A65C221D77E57389DBCCC1 # Quintex50
-199.249.224.40:80 orport=443 id=9C1E7D92115D431385B8CAEA6A7C15FB89CE236B # QuintexAirVPN21
-199.249.223.65:80 orport=443 id=9D21F034C3BFF4E7737D08CF775DC1745706801F # Quintex16
-199.249.224.67:80 orport=443 id=9E2D7C6981269404AA1970B53891701A20424EF8 # Quintex47
-199.249.223.64:80 orport=443 id=9F2856F6D2B89AD4EF6D5723FAB167DB5A53519A # Quintex15
-199.249.224.48:80 orport=443 id=A0DB820FEC87C0405F7BF05DEE5E4ADED2BB9904 # QuintexAirVPN29
-199.249.224.64:80 orport=443 id=A4A393FEF48640961AACE92D041934B55348CEF9 # Quintex44
-199.249.223.72:80 orport=443 id=B028707969D8ED84E6DEA597A884F78AAD471971 # QuintexAirVPN9
-199.249.223.40:80 orport=443 id=B0CD9F9B5B60651ADC5919C0F1EAA87DBA1D9249 # Quintex31
-199.249.224.61:80 orport=443 id=B2197C23A4FF5D1C49EE45BA7688BA8BCCD89A0B # Quintex41
-199.249.223.71:80 orport=443 id=B6320E44A230302C7BF9319E67597A9B87882241 # QuintexAirVPN10
-199.249.223.60:80 orport=443 id=B7047FBDE9C53C39011CA84E5CB2A8E3543066D0 # Quintex11
-199.249.224.66:80 orport=443 id=C78AFFEEE320EA0F860961763E613FD2FAC855F5 # Quintex46
-199.249.224.44:80 orport=443 id=CB7C0D841FE376EF43F7845FF201B0290C0A239E # QuintexAirVPN25
-199.249.223.47:80 orport=443 id=CC14C97F1D23EE97766828FC8ED8582E21E11665 # Quintex38
-199.249.223.77:80 orport=443 id=CC4A3AE960E3617F49BF9887B79186C14CBA6813 # QuintexAirVPN4
-199.249.223.41:80 orport=443 id=D25210CE07C49F2A4F2BC7A506EB0F5EA7F5E2C2 # Quintex32
-199.249.223.79:80 orport=443 id=D33292FEDE24DD40F2385283E55C87F85C0943B6 # QuintexAirVPN2
-199.249.224.47:80 orport=443 id=D6FF2697CEA5C0C7DA84797C2E71163814FC2466 # QuintexAirVPN28
-199.249.223.68:80 orport=443 id=DF20497E487A979995D851A5BCEC313DF7E5BC51 # Quintex19
-199.249.223.43:80 orport=443 id=E480D577F58E782A5BC4FA6F49A6650E9389302F # Quintex34
-199.249.224.69:80 orport=443 id=EABC2DD0D47B5DB11F2D37EB3C60C2A4D91C10F2 # Quintex49
-199.249.223.78:80 orport=443 id=EC15DB62D9101481F364DE52EB8313C838BDDC29 # QuintexAirVPN3
-199.249.224.42:80 orport=443 id=F21DE9C7DE31601D9716781E17E24380887883D1 # QuintexAirVPN23
-199.249.223.81:80 orport=443 id=F7447E99EB5CBD4D5EB913EE0E35AC642B5C1EF3 # QuintexAirVPN1
-199.249.224.43:80 orport=443 id=FDD700C791CC6BB0AC1C2099A82CBC367AD4B764 # QuintexAirVPN24
-199.249.224.62:80 orport=443 id=FE00A3A835680E67FBBC895A724E2657BB253E97 # Quintex42
-199.249.223.66:80 orport=443 id=C5A53BCC174EF8FD0DCB223E4AA929FA557DEDB2 # Quintex17
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013914.html
-5.196.23.64:9030 orport=9001 id=775B0FAFDE71AADC23FFC8782B7BEB1D5A92733E # Aerodynamik01
-217.182.75.181:9030 orport=9001 id=EFEACD781604EB80FBC025EDEDEA2D523AEAAA2F # Aerodynamik02
-193.70.43.76:9030 orport=9001 id=484A10BA2B8D48A5F0216674C8DD50EF27BC32F3 # Aerodynamik03
-149.56.141.138:9030 orport=9001 id=1938EBACBB1A7BFA888D9623C90061130E63BB3F # Aerodynamik04
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013917.html
-104.200.20.46:80 orport=9001 id=78E2BE744A53631B4AAB781468E94C52AB73968B # bynumlawtor
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013929.html
-139.99.130.178:80 orport=443 id=867B95CACD64653FEEC4D2CEFC5C49B4620307A7 # coffswifi2
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013946.html
-172.98.193.43:80 orport=443 id=5E56738E7F97AA81DEEF59AF28494293DFBFCCDF # Backplane
-
-# Email sent directly to teor
-62.210.254.132:80 orport=443 id=8456DFA94161CDD99E480C2A2992C366C6564410 # turingmachine
-
-# Email sent directly to teor
-80.127.117.180:80 orport=443 id=328E54981C6DDD7D89B89E418724A4A7881E3192 ipv6=[2001:985:e77:10::4]:443 # sjc01
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/013960.html
-51.15.205.214:9030 orport=9001 id=8B6556601612F1E2AFCE2A12FFFAF8482A76DD1F ipv6=[2001:bc8:4400:2500::5:b07]:9001 # titania1
-51.15.205.214:9031 orport=9002 id=5E363D72488276160D062DDD2DFA25CFEBAF5EA9 ipv6=[2001:bc8:4400:2500::5:b07]:9002 # titania2
-
-# Email sent directly to teor
-185.129.249.124:9030 orport=9001 id=1FA8F638298645BE58AC905276680889CB795A94 # treadstone
-
-# https://lists.torproject.org/pipermail/tor-relays/2017-December/014000.html
-24.117.231.229:34175 orport=45117 id=CE24412AD69444954B4015E293AE53DDDAFEA3D6 # Anosognosia
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014012.html
-128.31.0.13:80 orport=443 id=A53C46F5B157DD83366D45A8E99A244934A14C46 # csailmitexit
-
-# Email sent directly to teor
-82.247.103.117:110 orport=995 id=C9B3C1661A9577BA24C1C2C6123918921A495509 # Casper01
-109.238.2.79:110 orport=995 id=7520892E3DD133D0B0464D01A158B54B8E2A8B75 # Casper02
-51.15.179.153:110 orport=995 id=BB60F5BA113A0B8B44B7B37DE3567FE561E92F78 # Casper04
-
-# Email sent directly to teor
-80.127.107.179:80 orport=443 id=BC6B2E2F62ACC5EDECBABE64DA1E48F84DD98B78 ipv6=[2001:981:4a22:c::6]:443 # TVISION02
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014020.html
-37.120.174.249:80 orport=443 id=11DF0017A43AF1F08825CD5D973297F81AB00FF3 ipv6=[2a03:4000:6:724c:df98:15f9:b34d:443]:443 # gGDHjdcC6zAlM8k08lX
-
-# These fallbacks opted-in in previous releases, then changed their details,
-# and so we blacklisted them. Now we want to whitelist changes.
-# Assume details update is permanent
-85.230.184.93:9030 orport=443 id=855BC2DABE24C861CD887DB9B2E950424B49FC34 # Logforme
-176.31.180.157:143 orport=22 id=E781F4EC69671B3F1864AE2753E0890351506329 ipv6=[2001:41d0:8:eb9d::1]:22 # armbrust
-
-# https://lists.torproject.org/pipermail/tor-relays/2018-January/014024.html
-82.161.212.209:9030 orport=9001 id=4E8CE6F5651E7342C1E7E5ED031E82078134FB0D ipv6=[2001:980:d7ed:1:ff:b0ff:fe00:d0b]:9001 # ymkeo
diff --git a/scripts/maint/format_changelog.py b/scripts/maint/format_changelog.py
index 8dce4b6e51..08b2155fa3 100755
--- a/scripts/maint/format_changelog.py
+++ b/scripts/maint/format_changelog.py
@@ -401,7 +401,7 @@ class ChangeLog(object):
         self.dumpEndOfChangelog()
 
 # Let's turn bugs to html.
-BUG_PAT = re.compile('(bug|ticket|feature)\s+(\d{4,5})', re.I)
+BUG_PAT = re.compile('(bug|ticket|issue|feature)\s+(\d{4,5})', re.I)
 def bug_html(m):
     return "%s <a href='https://bugs.torproject.org/%s'>%s</a>" % (m.group(1), m.group(2), m.group(2))
 
diff --git a/scripts/maint/generateFallbackDirLine.py b/scripts/maint/generateFallbackDirLine.py
deleted file mode 100755
index b856c938bf..0000000000
--- a/scripts/maint/generateFallbackDirLine.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-# Generate a fallback directory whitelist/blacklist line for every fingerprint
-# passed as an argument.
-#
-# Usage:
-# generateFallbackDirLine.py fingerprint ...
-
-import sys
-import urllib2
-
-import stem.descriptor.remote
-import stem.util.tor_tools
-
-if len(sys.argv) <= 1:
-  print('Usage: %s fingerprint ...' % sys.argv[0])
-  sys.exit(1)
-
-for fingerprint in sys.argv[1:]:
-  if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
-    print("'%s' isn't a valid relay fingerprint" % fingerprint)
-    sys.exit(1)
-
-  try:
-    desc = stem.descriptor.remote.get_server_descriptors(fingerprint).run()[0]
-  except urllib2.HTTPError as exc:
-    if exc.code == 404:
-      print('# %s not found in recent descriptors' % fingerprint)
-      continue
-    else:
-      raise
-
-  if not desc.dir_port:
-    print("# %s needs a DirPort" % fingerprint)
-  else:
-    ipv6_addresses = [(address, port) for address, port, is_ipv6 in desc.or_addresses if is_ipv6]
-    ipv6_field = ' ipv6=[%s]:%s' % ipv6_addresses[0] if ipv6_addresses else ''
-    print('%s:%s orport=%s id=%s%s # %s' % (desc.address, desc.dir_port, desc.or_port, fingerprint, ipv6_field, desc.nickname))
diff --git a/scripts/maint/lintChanges.py b/scripts/maint/lintChanges.py
index d5b8fcae5c..82c118f07e 100755
--- a/scripts/maint/lintChanges.py
+++ b/scripts/maint/lintChanges.py
@@ -35,6 +35,36 @@ NEEDS_SUBCATEGORIES = set([
     "Major features",
     ])
 
+def split_tor_version(version):
+    '''
+    Return the initial numeric components of the Tor version as a list of ints.
+    For versions earlier than 0.1.0, returns MAJOR, MINOR, and MICRO.
+    For versions 0.1.0 and later, returns MAJOR, MINOR, MICRO, and PATCHLEVEL if present.
+
+    If the version is malformed, returns None.
+    '''
+    version_match = re.match('([0-9]+)\.([0-9]+)\.([0-9]+)(\.([0-9]+))?', version)
+    if version_match is None:
+        return None
+
+    version_groups = version_match.groups()
+    if version_groups is None:
+        return None
+    if len(version_groups) < 3:
+        return None
+
+    if len(version_groups) != 5:
+        return None
+    version_components = version_groups[0:3]
+    version_components += version_groups[4:5]
+
+    try:
+        version_list = [int(v) for v in version_components if v is not None]
+    except ValueError:
+        return None
+
+    return version_list
+
 def lintfile(fname):
     have_warned = []
 
@@ -87,6 +117,32 @@ def lintfile(fname):
             warn("Bugfix does not say 'Fixes bug X; bugfix on Y'")
         elif re.search('tor-([0-9]+)', contents):
             warn("Do not prefix versions with 'tor-'. ('0.1.2', not 'tor-0.1.2'.)")
+        else:
+            bugfix_match = re.search('bugfix on ([0-9]+\.[0-9]+\.[0-9]+)', contents)
+            if bugfix_match is None:
+                warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)")
+            elif bugfix_match.group(0) is None:
+                warn("Versions must have at least 3 digits. ('0.1.2', '0.3.4.8', or '0.3.5.1-alpha'.)")
+            else:
+                bugfix_match = re.search('bugfix on ([0-9a-z][-.0-9a-z]+[0-9a-z])', contents)
+                bugfix_group = bugfix_match.groups() if bugfix_match is not None else None
+                bugfix_version = bugfix_group[0] if bugfix_group is not None else None
+                package_version = os.environ.get('PACKAGE_VERSION', None)
+                if bugfix_version is None:
+                    # This should be unreachable, unless the patterns are out of sync
+                    warn("Malformed bugfix version.")
+                elif package_version is not None:
+                    # If $PACKAGE_VERSION isn't set, skip this check
+                    bugfix_split = split_tor_version(bugfix_version)
+                    package_split = split_tor_version(package_version)
+                    if bugfix_split is None:
+                        # This should be unreachable, unless the patterns are out of sync
+                        warn("Malformed bugfix version: '{}'.".format(bugfix_version))
+                    elif package_split is None:
+                        # This should be unreachable, unless the patterns are out of sync, or the package versioning scheme has changed
+                        warn("Malformed $PACKAGE_VERSION: '{}'.".format(package_version))
+                    elif bugfix_split > package_split:
+                        warn("Bugfixes must be made on earlier versions (or this version). (Bugfix on version: '{}', current tor package version: '{}'.)".format(bugfix_version, package_version))
 
     return have_warned != []
 
diff --git a/scripts/maint/lookupFallbackDirContact.py b/scripts/maint/lookupFallbackDirContact.py
deleted file mode 100755
index 14c53d1282..0000000000
--- a/scripts/maint/lookupFallbackDirContact.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-
-# Lookup fallback directory contact lines for every fingerprint passed as an
-# argument.
-#
-# Usage:
-# lookupFallbackDirContact.py fingerprint ...
-
-import sys
-
-import stem.descriptor.remote as remote
-
-if len(sys.argv) <= 1:
-  print "Usage: {} fingerprint ...".format(sys.argv[0])
-  sys.exit(-1)
-
-# we need descriptors, because the consensus does not have contact infos
-descriptor_list = remote.get_server_descriptors(fingerprints=sys.argv[1:]).run()
-
-descriptor_list_fingerprints = []
-for d in descriptor_list:
-  assert d.fingerprint in sys.argv[1:]
-  descriptor_list_fingerprints.append(d.fingerprint)
-  print "{} {}".format(d.fingerprint, d.contact)
-
-for fingerprint in sys.argv[1:]:
-  if fingerprint not in descriptor_list_fingerprints:
-    print "{} not found in current descriptors".format(fingerprint)
diff --git a/scripts/maint/practracker/README b/scripts/maint/practracker/README
new file mode 100644
index 0000000000..d978b39806
--- /dev/null
+++ b/scripts/maint/practracker/README
@@ -0,0 +1,21 @@
+Practracker is a simple python tool that keeps track of places where
+our code is ugly, and tries to warn us about new ones or ones that
+get worse.
+
+Right now, practracker looks for the following kinds of
+best-practices violations:
+
+  .c files greater than 3000 lines long
+  .h files greater than 500 lines long
+  .c files with more than 50 includes
+  .h files with more than 15 includes
+
+  All files that include a local header not listed in a .may_include
+  file in the same directory, when that .may_include file has an
+  "!advisory" marker.
+
+The list of current violations is tracked in exceptions.txt; slight
+deviations of the current exceptions cause warnings, whereas large
+ones cause practracker to fail.
+
+For usage information, run "practracker.py --help".
diff --git a/scripts/maint/practracker/exceptions.txt b/scripts/maint/practracker/exceptions.txt
new file mode 100644
index 0000000000..7b15b37f8c
--- /dev/null
+++ b/scripts/maint/practracker/exceptions.txt
@@ -0,0 +1,335 @@
+# Welcome to the exceptions file for Tor's best-practices tracker!
+#
+# Each line of this file represents a single violation of Tor's best
+# practices -- typically, a violation that we had before practracker.py
+# first existed.
+#
+# There are three kinds of problems that we recognize right now:
+#   function-size -- a function of more than 100 lines.
+#   file-size -- a file of more than 3000 lines.
+#   include-count -- a file with more than 50 #includes.
+#
+# Each line below represents a single exception that practracker should
+# _ignore_. Each line has four parts:
+#  1. The word "problem".
+#  2. The kind of problem.
+#  3. The location of the problem: either a filename, or a
+#     filename:functionname pair.
+#  4. The magnitude of the problem to ignore.
+#
+# So for example, consider this line:
+#    problem file-size /src/core/or/connection_or.c 3200
+#
+# It tells practracker to allow the mentioned file to be up to 3200 lines
+# long, even though ordinarily it would warn about any file with more than
+# 3000 lines.
+#
+# You can either edit this file by hand, or regenerate it completely by
+# running `make practracker-regen`.
+#
+# Remember: It is better to fix the problem than to add a new exception!
+
+problem file-size /src/app/config/config.c 8518
+problem include-count /src/app/config/config.c 89
+problem function-size /src/app/config/config.c:options_act_reversible() 296
+problem function-size /src/app/config/config.c:options_act() 589
+problem function-size /src/app/config/config.c:resolve_my_address() 190
+problem function-size /src/app/config/config.c:options_validate() 1209
+problem function-size /src/app/config/config.c:options_init_from_torrc() 207
+problem function-size /src/app/config/config.c:options_init_from_string() 171
+problem function-size /src/app/config/config.c:options_init_logs() 145
+problem function-size /src/app/config/config.c:parse_bridge_line() 104
+problem function-size /src/app/config/config.c:parse_transport_line() 189
+problem function-size /src/app/config/config.c:parse_dir_authority_line() 150
+problem function-size /src/app/config/config.c:parse_dir_fallback_line() 101
+problem function-size /src/app/config/config.c:parse_port_config() 446
+problem function-size /src/app/config/config.c:parse_ports() 168
+problem file-size /src/app/config/or_options_st.h 1112
+problem include-count /src/app/main/main.c 68
+problem function-size /src/app/main/main.c:dumpstats() 102
+problem function-size /src/app/main/main.c:tor_init() 137
+problem function-size /src/app/main/main.c:sandbox_init_filter() 291
+problem function-size /src/app/main/main.c:run_tor_main_loop() 105
+problem function-size /src/app/main/ntmain.c:nt_service_install() 126
+problem dependency-violation /src/core/crypto/hs_ntor.c 1
+problem dependency-violation /src/core/crypto/onion_crypto.c 5
+problem dependency-violation /src/core/crypto/onion_fast.c 1
+problem dependency-violation /src/core/crypto/onion_tap.c 3
+problem dependency-violation /src/core/crypto/relay_crypto.c 9
+problem file-size /src/core/mainloop/connection.c 5569
+problem include-count /src/core/mainloop/connection.c 62
+problem function-size /src/core/mainloop/connection.c:connection_free_minimal() 185
+problem function-size /src/core/mainloop/connection.c:connection_listener_new() 324
+problem function-size /src/core/mainloop/connection.c:connection_handle_listener_read() 161
+problem function-size /src/core/mainloop/connection.c:connection_proxy_connect() 148
+problem function-size /src/core/mainloop/connection.c:connection_read_proxy_handshake() 153
+problem function-size /src/core/mainloop/connection.c:retry_listener_ports() 112
+problem function-size /src/core/mainloop/connection.c:connection_handle_read_impl() 111
+problem function-size /src/core/mainloop/connection.c:connection_buf_read_from_socket() 180
+problem function-size /src/core/mainloop/connection.c:connection_handle_write_impl() 241
+problem function-size /src/core/mainloop/connection.c:assert_connection_ok() 143
+problem dependency-violation /src/core/mainloop/connection.c 44
+problem dependency-violation /src/core/mainloop/cpuworker.c 12
+problem include-count /src/core/mainloop/mainloop.c 63
+problem function-size /src/core/mainloop/mainloop.c:conn_close_if_marked() 108
+problem function-size /src/core/mainloop/mainloop.c:run_connection_housekeeping() 123
+problem dependency-violation /src/core/mainloop/mainloop.c 49
+problem dependency-violation /src/core/mainloop/mainloop_pubsub.c 1
+problem dependency-violation /src/core/mainloop/mainloop_sys.c 1
+problem dependency-violation /src/core/mainloop/netstatus.c 4
+problem dependency-violation /src/core/mainloop/periodic.c 2
+problem dependency-violation /src/core/or/address_set.c 1
+problem file-size /src/core/or/channel.c 3487
+problem dependency-violation /src/core/or/channel.c 9
+problem file-size /src/core/or/channel.h 780
+problem dependency-violation /src/core/or/channelpadding.c 6
+problem function-size /src/core/or/channeltls.c:channel_tls_handle_var_cell() 160
+problem function-size /src/core/or/channeltls.c:channel_tls_process_versions_cell() 170
+problem function-size /src/core/or/channeltls.c:channel_tls_process_netinfo_cell() 214
+problem function-size /src/core/or/channeltls.c:channel_tls_process_certs_cell() 246
+problem function-size /src/core/or/channeltls.c:channel_tls_process_authenticate_cell() 202
+problem dependency-violation /src/core/or/channeltls.c 10
+problem include-count /src/core/or/circuitbuild.c 54
+problem function-size /src/core/or/circuitbuild.c:get_unique_circ_id_by_chan() 128
+problem function-size /src/core/or/circuitbuild.c:circuit_extend() 147
+problem function-size /src/core/or/circuitbuild.c:choose_good_exit_server_general() 206
+problem dependency-violation /src/core/or/circuitbuild.c 25
+problem include-count /src/core/or/circuitlist.c 55
+problem function-size /src/core/or/circuitlist.c:HT_PROTOTYPE() 109
+problem function-size /src/core/or/circuitlist.c:circuit_free_() 143
+problem function-size /src/core/or/circuitlist.c:circuit_find_to_cannibalize() 101
+problem function-size /src/core/or/circuitlist.c:circuit_about_to_free() 120
+problem function-size /src/core/or/circuitlist.c:circuits_handle_oom() 117
+problem dependency-violation /src/core/or/circuitlist.c 19
+problem function-size /src/core/or/circuitmux.c:circuitmux_set_policy() 109
+problem function-size /src/core/or/circuitmux.c:circuitmux_attach_circuit() 113
+problem dependency-violation /src/core/or/circuitmux_ewma.c 2
+problem file-size /src/core/or/circuitpadding.c 3096
+problem function-size /src/core/or/circuitpadding.c:circpad_machine_schedule_padding() 113
+problem dependency-violation /src/core/or/circuitpadding.c 6
+problem file-size /src/core/or/circuitpadding.h 813
+problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_relay_hide_intro_circuits() 103
+problem function-size /src/core/or/circuitpadding_machines.c:circpad_machine_client_hide_rend_circuits() 112
+problem dependency-violation /src/core/or/circuitpadding_machines.c 1
+problem function-size /src/core/or/circuitstats.c:circuit_build_times_parse_state() 123
+problem dependency-violation /src/core/or/circuitstats.c 11
+problem file-size /src/core/or/circuituse.c 3162
+problem function-size /src/core/or/circuituse.c:circuit_is_acceptable() 128
+problem function-size /src/core/or/circuituse.c:circuit_expire_building() 394
+problem function-size /src/core/or/circuituse.c:circuit_log_ancient_one_hop_circuits() 126
+problem function-size /src/core/or/circuituse.c:circuit_build_failed() 149
+problem function-size /src/core/or/circuituse.c:circuit_launch_by_extend_info() 108
+problem function-size /src/core/or/circuituse.c:circuit_get_open_circ_or_launch() 352
+problem function-size /src/core/or/circuituse.c:connection_ap_handshake_attach_circuit() 244
+problem dependency-violation /src/core/or/circuituse.c 23
+problem function-size /src/core/or/command.c:command_process_create_cell() 156
+problem function-size /src/core/or/command.c:command_process_relay_cell() 132
+problem dependency-violation /src/core/or/command.c 8
+problem file-size /src/core/or/connection_edge.c 4596
+problem include-count /src/core/or/connection_edge.c 65
+problem function-size /src/core/or/connection_edge.c:connection_ap_expire_beginning() 117
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite() 191
+problem function-size /src/core/or/connection_edge.c:connection_ap_handle_onion() 185
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_rewrite_and_attach() 421
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_send_begin() 111
+problem function-size /src/core/or/connection_edge.c:connection_ap_handshake_socks_resolved() 101
+problem function-size /src/core/or/connection_edge.c:connection_exit_begin_conn() 185
+problem function-size /src/core/or/connection_edge.c:connection_exit_connect() 102
+problem dependency-violation /src/core/or/connection_edge.c 27
+problem file-size /src/core/or/connection_or.c 3122
+problem include-count /src/core/or/connection_or.c 51
+problem function-size /src/core/or/connection_or.c:connection_or_group_set_badness_() 105
+problem function-size /src/core/or/connection_or.c:connection_or_client_learned_peer_id() 142
+problem function-size /src/core/or/connection_or.c:connection_or_compute_authenticate_cell_body() 231
+problem dependency-violation /src/core/or/connection_or.c 20
+problem dependency-violation /src/core/or/dos.c 5
+problem dependency-violation /src/core/or/onion.c 2
+problem file-size /src/core/or/or.h 1107
+problem include-count /src/core/or/or.h 49
+problem dependency-violation /src/core/or/or_periodic.c 1
+problem file-size /src/core/or/policies.c 3249
+problem function-size /src/core/or/policies.c:policy_summarize() 107
+problem dependency-violation /src/core/or/policies.c 14
+problem function-size /src/core/or/protover.c:protover_all_supported() 117
+problem dependency-violation /src/core/or/reasons.c 2
+problem file-size /src/core/or/relay.c 3264
+problem function-size /src/core/or/relay.c:circuit_receive_relay_cell() 127
+problem function-size /src/core/or/relay.c:relay_send_command_from_edge_() 109
+problem function-size /src/core/or/relay.c:connection_ap_process_end_not_open() 192
+problem function-size /src/core/or/relay.c:connection_edge_process_relay_cell_not_open() 137
+problem function-size /src/core/or/relay.c:handle_relay_cell_command() 369
+problem function-size /src/core/or/relay.c:connection_edge_package_raw_inbuf() 128
+problem function-size /src/core/or/relay.c:circuit_resume_edge_reading_helper() 146
+problem dependency-violation /src/core/or/relay.c 16
+problem dependency-violation /src/core/or/scheduler.c 1
+problem function-size /src/core/or/scheduler_kist.c:kist_scheduler_run() 171
+problem dependency-violation /src/core/or/scheduler_kist.c 2
+problem function-size /src/core/or/scheduler_vanilla.c:vanilla_scheduler_run() 109
+problem dependency-violation /src/core/or/scheduler_vanilla.c 1
+problem dependency-violation /src/core/or/sendme.c 2
+problem dependency-violation /src/core/or/status.c 12
+problem function-size /src/core/or/versions.c:tor_version_parse() 104
+problem dependency-violation /src/core/proto/proto_cell.c 3
+problem dependency-violation /src/core/proto/proto_control0.c 1
+problem dependency-violation /src/core/proto/proto_ext_or.c 2
+problem dependency-violation /src/core/proto/proto_http.c 1
+problem function-size /src/core/proto/proto_socks.c:parse_socks_client() 110
+problem dependency-violation /src/core/proto/proto_socks.c 8
+problem function-size /src/feature/client/addressmap.c:addressmap_rewrite() 109
+problem function-size /src/feature/client/bridges.c:rewrite_node_address_for_bridge() 126
+problem function-size /src/feature/client/circpathbias.c:pathbias_measure_close_rate() 108
+problem function-size /src/feature/client/dnsserv.c:evdns_server_callback() 153
+problem file-size /src/feature/client/entrynodes.c 3824
+problem function-size /src/feature/client/entrynodes.c:entry_guards_upgrade_waiting_circuits() 155
+problem function-size /src/feature/client/entrynodes.c:entry_guard_parse_from_state() 246
+problem file-size /src/feature/client/entrynodes.h 639
+problem function-size /src/feature/client/transports.c:handle_proxy_line() 108
+problem function-size /src/feature/client/transports.c:parse_method_line_helper() 110
+problem function-size /src/feature/client/transports.c:create_managed_proxy_environment() 109
+problem function-size /src/feature/control/control.c:connection_control_process_inbuf() 113
+problem function-size /src/feature/control/control_auth.c:handle_control_authenticate() 186
+problem function-size /src/feature/control/control_cmd.c:handle_control_extendcircuit() 150
+problem function-size /src/feature/control/control_cmd.c:handle_control_add_onion() 256
+problem function-size /src/feature/control/control_cmd.c:add_onion_helper_keyarg() 116
+problem function-size /src/feature/control/control_events.c:control_event_stream_status() 118
+problem include-count /src/feature/control/control_getinfo.c 54
+problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_misc() 108
+problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_dir() 302
+problem function-size /src/feature/control/control_getinfo.c:getinfo_helper_events() 234
+problem function-size /src/feature/dirauth/bwauth.c:dirserv_read_measured_bandwidths() 121
+problem file-size /src/feature/dirauth/dirvote.c 4700
+problem include-count /src/feature/dirauth/dirvote.c 53
+problem function-size /src/feature/dirauth/dirvote.c:format_networkstatus_vote() 231
+problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_bw_weights_v10() 233
+problem function-size /src/feature/dirauth/dirvote.c:networkstatus_compute_consensus() 956
+problem function-size /src/feature/dirauth/dirvote.c:networkstatus_add_detached_signatures() 119
+problem function-size /src/feature/dirauth/dirvote.c:dirvote_add_vote() 162
+problem function-size /src/feature/dirauth/dirvote.c:dirvote_compute_consensuses() 164
+problem function-size /src/feature/dirauth/dirvote.c:dirserv_generate_networkstatus_vote_obj() 283
+problem function-size /src/feature/dirauth/dsigs_parse.c:networkstatus_parse_detached_signatures() 196
+problem function-size /src/feature/dirauth/guardfraction.c:dirserv_read_guardfraction_file_from_str() 109
+problem function-size /src/feature/dirauth/process_descs.c:dirserv_add_descriptor() 125
+problem function-size /src/feature/dirauth/shared_random.c:should_keep_commit() 109
+problem function-size /src/feature/dirauth/voteflags.c:dirserv_compute_performance_thresholds() 172
+problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_cleanup() 115
+problem function-size /src/feature/dircache/consdiffmgr.c:consdiffmgr_rescan_flavor_() 111
+problem function-size /src/feature/dircache/consdiffmgr.c:consensus_diff_worker_threadfn() 132
+problem function-size /src/feature/dircache/dircache.c:handle_get_current_consensus() 165
+problem function-size /src/feature/dircache/dircache.c:directory_handle_command_post() 119
+problem file-size /src/feature/dirclient/dirclient.c 3215
+problem include-count /src/feature/dirclient/dirclient.c 51
+problem function-size /src/feature/dirclient/dirclient.c:directory_get_from_dirserver() 126
+problem function-size /src/feature/dirclient/dirclient.c:directory_initiate_request() 201
+problem function-size /src/feature/dirclient/dirclient.c:directory_send_command() 239
+problem function-size /src/feature/dirclient/dirclient.c:dir_client_decompress_response_body() 111
+problem function-size /src/feature/dirclient/dirclient.c:connection_dir_client_reached_eof() 189
+problem function-size /src/feature/dirclient/dirclient.c:handle_response_fetch_consensus() 104
+problem function-size /src/feature/dircommon/consdiff.c:gen_ed_diff() 203
+problem function-size /src/feature/dircommon/consdiff.c:apply_ed_diff() 158
+problem function-size /src/feature/dirparse/authcert_parse.c:authority_cert_parse_from_string() 181
+problem function-size /src/feature/dirparse/ns_parse.c:routerstatus_parse_entry_from_string() 280
+problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_verify_bw_weights() 389
+problem function-size /src/feature/dirparse/ns_parse.c:networkstatus_parse_vote_from_string() 635
+problem function-size /src/feature/dirparse/parsecommon.c:tokenize_string() 101
+problem function-size /src/feature/dirparse/parsecommon.c:get_next_token() 158
+problem function-size /src/feature/dirparse/routerparse.c:router_parse_entry_from_string() 554
+problem function-size /src/feature/dirparse/routerparse.c:extrainfo_parse_entry_from_string() 208
+problem function-size /src/feature/hibernate/hibernate.c:accounting_parse_options() 109
+problem function-size /src/feature/hs/hs_cell.c:hs_cell_build_establish_intro() 115
+problem function-size /src/feature/hs/hs_cell.c:hs_cell_parse_introduce2() 152
+problem function-size /src/feature/hs/hs_client.c:send_introduce1() 103
+problem function-size /src/feature/hs/hs_client.c:hs_config_client_authorization() 107
+problem function-size /src/feature/hs/hs_common.c:hs_get_responsible_hsdirs() 102
+problem function-size /src/feature/hs/hs_config.c:config_service_v3() 107
+problem function-size /src/feature/hs/hs_config.c:config_generic_service() 138
+problem function-size /src/feature/hs/hs_descriptor.c:desc_encode_v3() 101
+problem function-size /src/feature/hs/hs_descriptor.c:decrypt_desc_layer() 111
+problem function-size /src/feature/hs/hs_descriptor.c:decode_introduction_point() 122
+problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_superencrypted_v3() 107
+problem function-size /src/feature/hs/hs_descriptor.c:desc_decode_encrypted_v3() 107
+problem file-size /src/feature/hs/hs_service.c 4116
+problem function-size /src/feature/keymgt/loadkey.c:ed_key_init_from_file() 326
+problem function-size /src/feature/nodelist/authcert.c:trusted_dirs_load_certs_from_string() 123
+problem function-size /src/feature/nodelist/authcert.c:authority_certs_fetch_missing() 295
+problem function-size /src/feature/nodelist/fmt_routerstatus.c:routerstatus_format_entry() 162
+problem function-size /src/feature/nodelist/microdesc.c:microdesc_cache_rebuild() 134
+problem include-count /src/feature/nodelist/networkstatus.c 63
+problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_check_consensus_signature() 175
+problem function-size /src/feature/nodelist/networkstatus.c:networkstatus_set_current_consensus() 289
+problem function-size /src/feature/nodelist/node_select.c:router_pick_directory_server_impl() 122
+problem function-size /src/feature/nodelist/node_select.c:compute_weighted_bandwidths() 203
+problem function-size /src/feature/nodelist/node_select.c:router_pick_trusteddirserver_impl() 112
+problem function-size /src/feature/nodelist/nodelist.c:compute_frac_paths_available() 190
+problem file-size /src/feature/nodelist/routerlist.c 3241
+problem function-size /src/feature/nodelist/routerlist.c:router_rebuild_store() 148
+problem function-size /src/feature/nodelist/routerlist.c:router_add_to_routerlist() 168
+problem function-size /src/feature/nodelist/routerlist.c:routerlist_remove_old_routers() 121
+problem function-size /src/feature/nodelist/routerlist.c:update_consensus_router_descriptor_downloads() 135
+problem function-size /src/feature/nodelist/routerlist.c:update_extrainfo_downloads() 103
+problem function-size /src/feature/relay/dns.c:dns_resolve_impl() 131
+problem function-size /src/feature/relay/dns.c:configure_nameservers() 161
+problem function-size /src/feature/relay/dns.c:evdns_callback() 108
+problem file-size /src/feature/relay/router.c 3522
+problem include-count /src/feature/relay/router.c 56
+problem function-size /src/feature/relay/router.c:init_keys() 252
+problem function-size /src/feature/relay/router.c:get_my_declared_family() 114
+problem function-size /src/feature/relay/router.c:router_build_fresh_unsigned_routerinfo() 136
+problem function-size /src/feature/relay/router.c:router_dump_router_to_string() 367
+problem function-size /src/feature/relay/routerkeys.c:load_ed_keys() 294
+problem function-size /src/feature/rend/rendcache.c:rend_cache_store_v2_desc_as_client() 190
+problem function-size /src/feature/rend/rendclient.c:rend_client_send_introduction() 219
+problem function-size /src/feature/rend/rendcommon.c:rend_encode_v2_descriptors() 221
+problem function-size /src/feature/rend/rendmid.c:rend_mid_establish_intro_legacy() 104
+problem function-size /src/feature/rend/rendparse.c:rend_parse_v2_service_descriptor() 181
+problem function-size /src/feature/rend/rendparse.c:rend_parse_introduction_points() 129
+problem file-size /src/feature/rend/rendservice.c 4511
+problem function-size /src/feature/rend/rendservice.c:rend_service_prune_list_impl_() 107
+problem function-size /src/feature/rend/rendservice.c:rend_config_service() 162
+problem function-size /src/feature/rend/rendservice.c:rend_service_load_auth_keys() 178
+problem function-size /src/feature/rend/rendservice.c:rend_service_receive_introduction() 330
+problem function-size /src/feature/rend/rendservice.c:rend_service_parse_intro_for_v3() 111
+problem function-size /src/feature/rend/rendservice.c:rend_service_decrypt_intro() 112
+problem function-size /src/feature/rend/rendservice.c:rend_service_intro_has_opened() 126
+problem function-size /src/feature/rend/rendservice.c:rend_service_rendezvous_has_opened() 117
+problem function-size /src/feature/rend/rendservice.c:directory_post_to_hs_dir() 106
+problem function-size /src/feature/rend/rendservice.c:upload_service_descriptor() 111
+problem function-size /src/feature/rend/rendservice.c:rend_consider_services_intro_points() 170
+problem function-size /src/feature/stats/rephist.c:rep_hist_load_mtbf_data() 185
+problem function-size /src/feature/stats/rephist.c:rep_hist_format_exit_stats() 148
+problem function-size /src/lib/compress/compress.c:tor_compress_impl() 127
+problem function-size /src/lib/compress/compress_zstd.c:tor_zstd_compress_process() 123
+problem function-size /src/lib/container/smartlist.c:smartlist_bsearch_idx() 107
+problem function-size /src/lib/crypt_ops/crypto_rand.c:crypto_strongest_rand_syscall() 102
+problem function-size /src/lib/encoding/binascii.c:base64_encode() 106
+problem function-size /src/lib/encoding/confline.c:parse_config_line_from_str_verbose() 117
+problem function-size /src/lib/encoding/cstring.c:unescape_string() 108
+problem function-size /src/lib/fs/dir.c:check_private_dir() 230
+problem function-size /src/lib/math/prob_distr.c:sample_uniform_interval() 145
+problem function-size /src/lib/net/address.c:tor_addr_parse_mask_ports() 194
+problem function-size /src/lib/net/address.c:tor_addr_compare_masked() 110
+problem function-size /src/lib/net/inaddr.c:tor_inet_pton() 107
+problem function-size /src/lib/net/socketpair.c:tor_ersatz_socketpair() 102
+problem function-size /src/lib/osinfo/uname.c:get_uname() 116
+problem function-size /src/lib/process/process_unix.c:process_unix_exec() 220
+problem function-size /src/lib/process/process_win32.c:process_win32_exec() 151
+problem function-size /src/lib/process/process_win32.c:process_win32_create_pipe() 109
+problem function-size /src/lib/process/restrict.c:set_max_file_descriptors() 102
+problem function-size /src/lib/process/setuid.c:switch_id() 156
+problem function-size /src/lib/sandbox/sandbox.c:prot_strings() 104
+problem function-size /src/lib/string/scanf.c:tor_vsscanf() 112
+problem function-size /src/lib/tls/tortls_nss.c:tor_tls_context_new() 152
+problem function-size /src/lib/tls/tortls_openssl.c:tor_tls_context_new() 170
+problem function-size /src/lib/tls/x509_nss.c:tor_tls_create_certificate_internal() 121
+problem function-size /src/tools/tor-gencert.c:parse_commandline() 111
+problem function-size /src/tools/tor-resolve.c:build_socks5_resolve_request() 102
+problem function-size /src/tools/tor-resolve.c:do_resolve() 171
+problem function-size /src/tools/tor-resolve.c:main() 112
+
+problem dependency-violation /scripts/maint/practracker/testdata/a.c 3
+problem dependency-violation /scripts/maint/practracker/testdata/header.h 3
+problem dependency-violation /src/core/crypto/hs_ntor.h 1
+problem dependency-violation /src/core/or/cell_queue_st.h 1
+problem dependency-violation /src/core/or/channel.h 1
+problem dependency-violation /src/core/or/circuitlist.h 1
+problem dependency-violation /src/core/or/connection_edge.h 1
+problem dependency-violation /src/core/or/or.h 1
diff --git a/scripts/maint/practracker/includes.py b/scripts/maint/practracker/includes.py
new file mode 100755
index 0000000000..397439b4ef
--- /dev/null
+++ b/scripts/maint/practracker/includes.py
@@ -0,0 +1,285 @@
+#!/usr/bin/python
+# Copyright 2018 The Tor Project, Inc.  See LICENSE file for licensing info.
+
+"""This script looks through all the directories for files matching *.c or
+   *.h, and checks their #include directives to make sure that only "permitted"
+   headers are included.
+
+   Any #include directives with angle brackets (like #include <stdio.h>) are
+   ignored -- only directives with quotes (like #include "foo.h") are
+   considered.
+
+   To decide what includes are permitted, this script looks at a .may_include
+   file in each directory.  This file contains empty lines, #-prefixed
+   comments, filenames (like "lib/foo/bar.h") and file globs (like lib/*/*.h)
+   for files that are permitted.
+"""
+
+
+from __future__ import print_function
+
+import fnmatch
+import os
+import re
+import sys
+
+if sys.version_info[0] <= 2:
+    def open_file(fname):
+        return open(fname, 'r')
+else:
+    def open_file(fname):
+        return open(fname, 'r', encoding='utf-8')
+
+def warn(msg):
+    print(msg, file=sys.stderr)
+
+def fname_is_c(fname):
+    """ Return true iff 'fname' is the name of a file that we should
+        search for possibly disallowed #include directives. """
+    return fname.endswith(".h") or fname.endswith(".c")
+
+INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"')
+RULES_FNAME = ".may_include"
+
+ALLOWED_PATTERNS = [
+    re.compile(r'^.*\*\.(h|inc)$'),
+    re.compile(r'^.*/.*\.h$'),
+    re.compile(r'^ext/.*\.c$'),
+    re.compile(r'^orconfig.h$'),
+    re.compile(r'^micro-revision.i$'),
+]
+
+def pattern_is_normal(s):
+    for p in ALLOWED_PATTERNS:
+        if p.match(s):
+            return True
+    return False
+
+class Error(object):
+    def __init__(self, location, msg, is_advisory=False):
+        self.location = location
+        self.msg = msg
+        self.is_advisory = is_advisory
+
+    def __str__(self):
+        return "{} at {}".format(self.msg, self.location)
+
+class Rules(object):
+    """ A 'Rules' object is the parsed version of a .may_include file. """
+    def __init__(self, dirpath):
+        self.dirpath = dirpath
+        if dirpath.startswith("src/"):
+            self.incpath = dirpath[4:]
+        else:
+            self.incpath = dirpath
+        self.patterns = []
+        self.usedPatterns = set()
+        self.is_advisory = False
+
+    def addPattern(self, pattern):
+        if pattern == "!advisory":
+            self.is_advisory = True
+            return
+        if not pattern_is_normal(pattern):
+            warn("Unusual pattern {} in {}".format(pattern, self.dirpath))
+        self.patterns.append(pattern)
+
+    def includeOk(self, path):
+        for pattern in self.patterns:
+            if fnmatch.fnmatchcase(path, pattern):
+                self.usedPatterns.add(pattern)
+                return True
+        return False
+
+    def applyToLines(self, lines, loc_prefix=""):
+        lineno = 0
+        for line in lines:
+            lineno += 1
+            m = INCLUDE_PATTERN.match(line)
+            if m:
+                include = m.group(1)
+                if not self.includeOk(include):
+                    yield Error("{}{}".format(loc_prefix,str(lineno)),
+                                "Forbidden include of {}".format(include),
+                                is_advisory=self.is_advisory)
+
+    def applyToFile(self, fname, f):
+        for error in self.applyToLines(iter(f), "{}:".format(fname)):
+            yield error
+
+    def noteUnusedRules(self):
+        for p in self.patterns:
+            if p not in self.usedPatterns:
+                warn("Pattern {} in {} was never used.".format(p, self.dirpath))
+
+    def getAllowedDirectories(self):
+        allowed = []
+        for p in self.patterns:
+            m = re.match(r'^(.*)/\*\.(h|inc)$', p)
+            if m:
+                allowed.append(m.group(1))
+                continue
+            m = re.match(r'^(.*)/[^/]*$', p)
+            if m:
+                allowed.append(m.group(1))
+                continue
+
+        return allowed
+
+include_rules_cache = {}
+
+def load_include_rules(fname):
+    """ Read a rules file from 'fname', and return it as a Rules object.
+        Return 'None' if fname does not exist.
+    """
+    if fname in include_rules_cache:
+        return include_rules_cache[fname]
+    if not os.path.exists(fname):
+        include_rules_cache[fname] = None
+        return None
+    result = Rules(os.path.split(fname)[0])
+    with open_file(fname) as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith("#") or not line:
+                continue
+            result.addPattern(line)
+    include_rules_cache[fname] = result
+    return result
+
+def get_all_include_rules():
+    """Return a list of all the Rules objects we have loaded so far,
+       sorted by their directory names."""
+    return [ rules for (fname,rules) in
+             sorted(include_rules_cache.items())
+             if rules is not None ]
+
+def remove_self_edges(graph):
+    """Takes a directed graph in as an adjacency mapping (a mapping from
+       node to a list of the nodes to which it connects).
+
+       Remove all edges from a node to itself."""
+
+    for k in list(graph):
+        graph[k] = [ d for d in graph[k] if d != k ]
+
+def toposort(graph, limit=100):
+    """Takes a directed graph in as an adjacency mapping (a mapping from
+       node to a list of the nodes to which it connects).  Tries to
+       perform a topological sort on the graph, arranging the nodes into
+       "levels", such that every member of each level is only reachable
+       by members of later levels.
+
+       Returns a list of the members of each level.
+
+       Modifies the input graph, removing every member that could be
+       sorted.  If the graph does not become empty, then it contains a
+       cycle.
+
+       "limit" is the max depth of the graph after which we give up trying
+       to sort it and conclude we have a cycle.
+    """
+    all_levels = []
+
+    n = 0
+    while graph:
+        n += 0
+        cur_level = []
+        all_levels.append(cur_level)
+        for k in list(graph):
+            graph[k] = [ d for d in graph[k] if d in graph ]
+            if graph[k] == []:
+                cur_level.append(k)
+        for k in cur_level:
+            del graph[k]
+        n += 1
+        if n > limit:
+            break
+
+    return all_levels
+
+def consider_include_rules(fname, f):
+    dirpath = os.path.split(fname)[0]
+    rules_fname = os.path.join(dirpath, RULES_FNAME)
+    rules = load_include_rules(os.path.join(dirpath, RULES_FNAME))
+    if rules is None:
+        return
+
+    for err in rules.applyToFile(fname, f):
+        yield err
+
+    list_unused = False
+    log_sorted_levels = False
+
+def walk_c_files(topdir="src"):
+    """Run through all .c and .h files under topdir, looking for
+       include-rule violations. Yield those violations."""
+
+    for dirpath, dirnames, fnames in os.walk(topdir):
+        for fname in fnames:
+            if fname_is_c(fname):
+                fullpath = os.path.join(dirpath,fname)
+                with open(fullpath) as f:
+                    for err in consider_include_rules(fullpath, f):
+                        yield err
+
+def run_check_includes(topdir, list_unused=False, log_sorted_levels=False,
+                       list_advisories=False):
+    trouble = False
+
+    for err in walk_c_files(topdir):
+        if err.is_advisory and not list_advisories:
+            continue
+        print(err, file=sys.stderr)
+        if not err.is_advisory:
+            trouble = True
+
+    if trouble:
+        err(
+    """To change which includes are allowed in a C file, edit the {}
+    files in its enclosing directory.""".format(RULES_FNAME))
+        sys.exit(1)
+
+    if list_unused:
+        for rules in get_all_include_rules():
+            rules.noteUnusedRules()
+
+    uses_dirs = { }
+    for rules in get_all_include_rules():
+        uses_dirs[rules.incpath] = rules.getAllowedDirectories()
+
+    remove_self_edges(uses_dirs)
+    all_levels = toposort(uses_dirs)
+
+    if log_sorted_levels:
+        for (n, cur_level) in enumerate(all_levels):
+            if cur_level:
+                print(n, cur_level)
+
+    if uses_dirs:
+        print("There are circular .may_include dependencies in here somewhere:",
+              uses_dirs)
+        sys.exit(1)
+
+def main(argv):
+    import argparse
+
+    progname = argv[0]
+    parser = argparse.ArgumentParser(prog=progname)
+    parser.add_argument("--toposort", action="store_true",
+                        help="Print a topologically sorted list of modules")
+    parser.add_argument("--list-unused", action="store_true",
+                        help="List unused lines in .may_include files.")
+    parser.add_argument("--list-advisories", action="store_true",
+                        help="List advisories as well as forbidden includes")
+    parser.add_argument("topdir", default="src", nargs="?",
+                        help="Top-level directory for the tor source")
+    args = parser.parse_args(argv[1:])
+
+    run_check_includes(topdir=args.topdir,
+                       log_sorted_levels=args.toposort,
+                       list_unused=args.list_unused,
+                       list_advisories=args.list_advisories)
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/scripts/maint/practracker/metrics.py b/scripts/maint/practracker/metrics.py
new file mode 100644
index 0000000000..4c62bc2425
--- /dev/null
+++ b/scripts/maint/practracker/metrics.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+# Implementation of various source code metrics.
+# These are currently ad-hoc string operations and regexps.
+# We might want to use a proper static analysis library in the future, if we want to get more advanced metrics.
+
+import re
+
+def get_file_len(f):
+    """Get file length of file"""
+    i = -1
+    for i, l in enumerate(f):
+        pass
+    return i + 1
+
+def get_include_count(f):
+    """Get number of #include statements in the file"""
+    include_count = 0
+    for line in f:
+        if re.match(r'\s*#\s*include', line):
+            include_count += 1
+    return include_count
+
+def get_function_lines(f):
+    """
+    Return iterator which iterates over functions and returns (function name, function lines)
+    """
+
+    # Skip lines that look like they are defining functions with these
+    # names: they aren't real function definitions.
+    REGEXP_CONFUSE_TERMS = {"MOCK_IMPL", "MOCK_DECL", "HANDLE_DECL",
+                            "ENABLE_GCC_WARNINGS", "ENABLE_GCC_WARNING",
+                            "DUMMY_TYPECHECK_INSTANCE",
+                            "DISABLE_GCC_WARNING", "DISABLE_GCC_WARNINGS"}
+
+    in_function = False
+    found_openbrace = False
+    for lineno, line in enumerate(f):
+        if not in_function:
+            # find the start of a function
+            m = re.match(r'^([a-zA-Z_][a-zA-Z_0-9]*),?\(', line)
+            if m:
+                func_name = m.group(1)
+                if func_name in REGEXP_CONFUSE_TERMS:
+                    continue
+                func_start = lineno
+                in_function = True
+        elif not found_openbrace and line.startswith("{"):
+            found_openbrace = True
+            func_start = lineno
+        else:
+            # Find the end of a function
+            if line.startswith("}"):
+                n_lines = lineno - func_start + 1
+                in_function = False
+                found_openbrace = False
+                yield (func_name, n_lines)
diff --git a/scripts/maint/practracker/practracker.py b/scripts/maint/practracker/practracker.py
new file mode 100755
index 0000000000..71741265f6
--- /dev/null
+++ b/scripts/maint/practracker/practracker.py
@@ -0,0 +1,303 @@
+#!/usr/bin/python
+
+"""
+Best-practices tracker for Tor source code.
+
+Go through the various .c files and collect metrics about them. If the metrics
+violate some of our best practices and they are not found in the optional
+exceptions file, then log a problem about them.
+
+We currently do metrics about file size, function size and number of includes,
+for C source files and headers.
+
+practracker.py should be run with its second argument pointing to the Tor
+top-level source directory like this:
+  $ python3 ./scripts/maint/practracker/practracker.py .
+
+To regenerate the exceptions file so that it allows all current
+problems in the Tor source, use the --regen flag:
+  $ python3 --regen ./scripts/maint/practracker/practracker.py .
+"""
+
+from __future__ import print_function
+
+import os, sys
+
+import metrics
+import util
+import problem
+import includes
+
+# The filename of the exceptions file (it should be placed in the practracker directory)
+EXCEPTIONS_FNAME = "./exceptions.txt"
+
+# Recommended file size
+MAX_FILE_SIZE = 3000 # lines
+# Recommended function size
+MAX_FUNCTION_SIZE = 100 # lines
+# Recommended number of #includes
+MAX_INCLUDE_COUNT = 50
+# Recommended file size for headers
+MAX_H_FILE_SIZE = 500
+# Recommended include count for headers
+MAX_H_INCLUDE_COUNT = 15
+# Recommended number of dependency violations
+MAX_DEP_VIOLATIONS = 0
+
+# Map from problem type to functions that adjust for tolerance
+TOLERANCE_FNS = {
+    'include-count': lambda n: int(n*1.1),
+    'function-size': lambda n: int(n*1.1),
+    'file-size': lambda n: int(n*1.02),
+    'dependency-violation': lambda n: (n+2)
+}
+
+#######################################################
+
+# The Tor source code topdir
+TOR_TOPDIR = None
+
+#######################################################
+
+if sys.version_info[0] <= 2:
+    def open_file(fname):
+        return open(fname, 'r')
+else:
+    def open_file(fname):
+        return open(fname, 'r', encoding='utf-8')
+
+def consider_file_size(fname, f):
+    """Consider the size of 'f' and yield an FileSizeItem for it.
+    """
+    file_size = metrics.get_file_len(f)
+    yield problem.FileSizeItem(fname, file_size)
+
+def consider_includes(fname, f):
+    """Consider the #include count in for 'f' and yield an IncludeCountItem
+        for it.
+    """
+    include_count = metrics.get_include_count(f)
+
+    yield problem.IncludeCountItem(fname, include_count)
+
+def consider_function_size(fname, f):
+    """yield a FunctionSizeItem for every function in f.
+    """
+
+    for name, lines in metrics.get_function_lines(f):
+        canonical_function_name = "%s:%s()" % (fname, name)
+        yield problem.FunctionSizeItem(canonical_function_name, lines)
+
+def consider_include_violations(fname, real_fname, f):
+    n = 0
+    for item in includes.consider_include_rules(real_fname, f):
+        n += 1
+    if n:
+        yield problem.DependencyViolationItem(fname, n)
+
+
+#######################################################
+
+def consider_all_metrics(files_list):
+    """Consider metrics for all files, and yield a sequence of problem.Item
+       object for those issues."""
+    for fname in files_list:
+        with open_file(fname) as f:
+            for item in consider_metrics_for_file(fname, f):
+                yield item
+
+def consider_metrics_for_file(fname, f):
+    """
+       Yield a sequence of problem.Item objects for all of the metrics in
+       'f'.
+    """
+    real_fname = fname
+    # Strip the useless part of the path
+    if fname.startswith(TOR_TOPDIR):
+        fname = fname[len(TOR_TOPDIR):]
+
+    # Get file length
+    for item in consider_file_size(fname, f):
+        yield item
+
+    # Consider number of #includes
+    f.seek(0)
+    for item in consider_includes(fname, f):
+        yield item
+
+    # Get function length
+    f.seek(0)
+    for item in consider_function_size(fname, f):
+        yield item
+
+    # Check for "upward" includes
+    f.seek(0)
+    for item in consider_include_violations(fname, real_fname, f):
+        yield item
+
+HEADER="""\
+# Welcome to the exceptions file for Tor's best-practices tracker!
+#
+# Each line of this file represents a single violation of Tor's best
+# practices -- typically, a violation that we had before practracker.py
+# first existed.
+#
+# There are three kinds of problems that we recognize right now:
+#   function-size -- a function of more than {MAX_FUNCTION_SIZE} lines.
+#   file-size -- a .c file of more than {MAX_FILE_SIZE} lines, or a .h
+#      file with more than {MAX_H_FILE_SIZE} lines.
+#   include-count -- a .c file with more than {MAX_INCLUDE_COUNT} #includes,
+       or a .h file with more than {MAX_H_INCLUDE_COUNT} #includes.
+#   dependency-violation -- a file includes a header that it should
+#      not, according to an advisory .may_include file.
+#
+# Each line below represents a single exception that practracker should
+# _ignore_. Each line has four parts:
+#  1. The word "problem".
+#  2. The kind of problem.
+#  3. The location of the problem: either a filename, or a
+#     filename:functionname pair.
+#  4. The magnitude of the problem to ignore.
+#
+# So for example, consider this line:
+#    problem file-size /src/core/or/connection_or.c 3200
+#
+# It tells practracker to allow the mentioned file to be up to 3200 lines
+# long, even though ordinarily it would warn about any file with more than
+# {MAX_FILE_SIZE} lines.
+#
+# You can either edit this file by hand, or regenerate it completely by
+# running `make practracker-regen`.
+#
+# Remember: It is better to fix the problem than to add a new exception!
+
+""".format(**globals())
+
+def main(argv):
+    import argparse
+
+    progname = argv[0]
+    parser = argparse.ArgumentParser(prog=progname)
+    parser.add_argument("--regen", action="store_true",
+                        help="Regenerate the exceptions file")
+    parser.add_argument("--list-overbroad", action="store_true",
+                        help="List over-broad exceptions")
+    parser.add_argument("--exceptions",
+                        help="Override the location for the exceptions file")
+    parser.add_argument("--strict", action="store_true",
+                        help="Make all warnings into errors")
+    parser.add_argument("--terse", action="store_true",
+                        help="Do not emit helpful instructions.")
+    parser.add_argument("--max-h-file-size", default=MAX_H_FILE_SIZE,
+                        help="Maximum lines per .h file")
+    parser.add_argument("--max-h-include-count", default=MAX_H_INCLUDE_COUNT,
+                        help="Maximum includes per .h file")
+    parser.add_argument("--max-file-size", default=MAX_FILE_SIZE,
+                        help="Maximum lines per .c file")
+    parser.add_argument("--max-include-count", default=MAX_INCLUDE_COUNT,
+                        help="Maximum includes per .c file")
+    parser.add_argument("--max-function-size", default=MAX_FUNCTION_SIZE,
+                        help="Maximum lines per function")
+    parser.add_argument("--max-dependency-violations", default=MAX_DEP_VIOLATIONS,
+                        help="Maximum number of dependency violations to allow")
+    parser.add_argument("--include-dir", action="append",
+                        default=["src"],
+                        help="A directory (under topdir) to search for source")
+    parser.add_argument("topdir", default=".", nargs="?",
+                        help="Top-level directory for the tor source")
+    args = parser.parse_args(argv[1:])
+
+    global TOR_TOPDIR
+    TOR_TOPDIR = args.topdir
+    if args.exceptions:
+        exceptions_file = args.exceptions
+    else:
+        exceptions_file = os.path.join(TOR_TOPDIR, "scripts/maint/practracker", EXCEPTIONS_FNAME)
+
+    # 0) Configure our thresholds of "what is a problem actually"
+    filt = problem.ProblemFilter()
+    filt.addThreshold(problem.FileSizeItem("*.c", int(args.max_file_size)))
+    filt.addThreshold(problem.IncludeCountItem("*.c", int(args.max_include_count)))
+    filt.addThreshold(problem.FileSizeItem("*.h", int(args.max_h_file_size)))
+    filt.addThreshold(problem.IncludeCountItem("*.h", int(args.max_h_include_count)))
+    filt.addThreshold(problem.FunctionSizeItem("*.c", int(args.max_function_size)))
+    filt.addThreshold(problem.DependencyViolationItem("*.c", int(args.max_dependency_violations)))
+    filt.addThreshold(problem.DependencyViolationItem("*.h", int(args.max_dependency_violations)))
+
+    if args.list_overbroad and args.regen:
+        print("Cannot use --regen with --list-overbroad",
+              file=sys.stderr)
+        sys.exit(1)
+
+    # 1) Get all the .c files we care about
+    files_list = util.get_tor_c_files(TOR_TOPDIR, args.include_dir)
+
+    # 2) Initialize problem vault and load an optional exceptions file so that
+    # we don't warn about the past
+    if args.regen:
+        tmpname = exceptions_file + ".tmp"
+        tmpfile = open(tmpname, "w")
+        problem_file = tmpfile
+        problem_file.write(HEADER)
+        ProblemVault = problem.ProblemVault()
+    else:
+        ProblemVault = problem.ProblemVault(exceptions_file)
+        problem_file = sys.stdout
+
+    if args.list_overbroad:
+        # If we're listing overbroad exceptions, don't list problems.
+        problem_file = util.NullFile()
+
+    # 2.1) Adjust the exceptions so that we warn only about small problems,
+    # and produce errors on big ones.
+    if not (args.regen or args.list_overbroad or args.strict):
+        ProblemVault.set_tolerances(TOLERANCE_FNS)
+
+    # 3) Go through all the files and report problems if they are not exceptions
+    found_new_issues = 0
+    for item in filt.filter(consider_all_metrics(files_list)):
+        status = ProblemVault.register_problem(item)
+        if status == problem.STATUS_ERR:
+            print(item, file=problem_file)
+            found_new_issues += 1
+        elif status == problem.STATUS_WARN:
+            # warnings always go to stdout.
+            print("(warning) {}".format(item))
+
+    if args.regen:
+        tmpfile.close()
+        os.rename(tmpname, exceptions_file)
+        sys.exit(0)
+
+    # If new issues were found, try to give out some advice to the developer on how to resolve it.
+    if found_new_issues and not args.regen and not args.terse:
+        new_issues_str = """\
+FAILURE: practracker found {} new problem(s) in the code: see warnings above.
+
+Please fix the problems if you can, and update the exceptions file
+({}) if you can't.
+
+See doc/HACKING/HelpfulTools.md for more information on using practracker.\
+
+You can disable this message by setting the TOR_DISABLE_PRACTRACKER environment
+variable.
+""".format(found_new_issues, exceptions_file)
+        print(new_issues_str)
+
+    if args.list_overbroad:
+        def k_fn(tup):
+            return tup[0].key()
+        for (ex,p) in sorted(ProblemVault.list_overbroad_exceptions(), key=k_fn):
+            if p is None:
+                print(ex, "->", 0)
+            else:
+                print(ex, "->", p.metric_value)
+
+    sys.exit(found_new_issues)
+
+if __name__ == '__main__':
+    if os.environ.get("TOR_DISABLE_PRACTRACKER"):
+        print("TOR_DISABLE_PRACTRACKER is set, skipping practracker tests.",
+              file=sys.stderr)
+        sys.exit(0)
+    main(sys.argv)
diff --git a/scripts/maint/practracker/practracker_tests.py b/scripts/maint/practracker/practracker_tests.py
new file mode 100755
index 0000000000..45719d6cb7
--- /dev/null
+++ b/scripts/maint/practracker/practracker_tests.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+
+"""Some simple tests for practracker metrics"""
+
+import unittest
+
+try:
+    # python 2 names the module this way...
+    from StringIO import StringIO
+except ImportError:
+    # python 3 names the module this way.
+    from io import StringIO
+
+import metrics
+
+function_file = """static void
+fun(directory_request_t *req, const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+}
+
+static void
+fun(directory_request_t *req,
+      const char *resource)
+{
+  time_t if_modified_since = 0;
+  uint8_t or_diff_from[DIGEST256_LEN];
+}
+
+MOCK_IMPL(void,
+fun,(
+       uint8_t dir_purpose,
+       uint8_t router_purpose,
+       const char *resource,
+       int pds_flags,
+       download_want_authority_t want_authority))
+{
+  const routerstatus_t *rs = NULL;
+  const or_options_t *options = get_options();
+}
+"""
+
+class TestFunctionLength(unittest.TestCase):
+    def test_function_length(self):
+        funcs = StringIO(function_file)
+        # All functions should have length 2
+        for name, lines in metrics.get_function_lines(funcs):
+            self.assertEqual(name, "fun")
+
+        funcs.seek(0)
+
+        for name, lines in metrics.get_function_lines(funcs):
+            self.assertEqual(lines, 4)
+
+class TestIncludeCount(unittest.TestCase):
+    def test_include_count(self):
+        f = StringIO("""
+  #   include <abc.h>
+  #   include "def.h"
+#include "ghi.h"
+\t#\t include "jkl.h"
+""")
+        self.assertEqual(metrics.get_include_count(f),4)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/scripts/maint/practracker/problem.py b/scripts/maint/practracker/problem.py
new file mode 100644
index 0000000000..d21840a213
--- /dev/null
+++ b/scripts/maint/practracker/problem.py
@@ -0,0 +1,242 @@
+"""
+In this file we define a ProblemVault class where we store all the
+exceptions and all the problems we find with the code.
+
+The ProblemVault is capable of registering problems and also figuring out if a
+problem is worse than a registered exception so that it only warns when things
+get worse.
+"""
+
+from __future__ import print_function
+
+import os.path
+import re
+import sys
+
+STATUS_ERR = 2
+STATUS_WARN = 1
+STATUS_OK = 0
+
+class ProblemVault(object):
+    """
+    Singleton where we store the various new problems we
+    found in the code, and also the old problems we read from the exception
+    file.
+    """
+    def __init__(self, exception_fname=None):
+        # Exception dictionary: { problem.key() : Problem object }
+        self.exceptions = {}
+        # Exception dictionary: maps key to the problem it was used to
+        # suppress.
+        self.used_exception_for = {}
+
+        if exception_fname == None:
+            return
+
+        try:
+            with open(exception_fname, 'r') as exception_f:
+                self.register_exceptions(exception_f)
+        except IOError:
+            print("No exception file provided", file=sys.stderr)
+
+    def register_exceptions(self, exception_file):
+        # Register exceptions
+        for lineno, line in enumerate(exception_file, 1):
+            try:
+                problem = get_old_problem_from_exception_str(line)
+            except ValueError as v:
+                print("Exception file line {} not recognized: {}"
+                      .format(lineno,v),
+                      file=sys.stderr)
+                continue
+
+            if problem is None:
+                continue
+
+            # Fail if we see dup exceptions. There is really no reason to have dup exceptions.
+            if problem.key() in self.exceptions:
+                print("Duplicate exceptions lines found in exception file:\n\t{}\n\t{}\nAborting...".format(problem, self.exceptions[problem.key()]),
+                      file=sys.stderr)
+                sys.exit(1)
+
+            self.exceptions[problem.key()] = problem
+            #print "Registering exception: %s" % problem
+
+    def register_problem(self, problem):
+        """
+        Register this problem to the problem value. Return true if it was a new
+        problem or it worsens an already existing problem.  A true
+        value may be STATUS_ERR to indicate a hard violation, or STATUS_WARN
+        to indicate a warning.
+        """
+        # This is a new problem, print it
+        if problem.key() not in self.exceptions:
+            return STATUS_ERR
+
+        # If it's an old problem, we don't warn if the situation got better
+        # (e.g. we went from 4k LoC to 3k LoC), but we do warn if the
+        # situation worsened (e.g. we went from 60 includes to 80).
+        status = problem.is_worse_than(self.exceptions[problem.key()])
+
+        # Remember that we used this exception, so that we can later
+        # determine whether the exception was overbroad.
+        self.used_exception_for[problem.key()] = problem
+
+        return status
+
+    def list_overbroad_exceptions(self):
+        """Return an iterator of tuples containing (ex,prob) where ex is an
+           exceptions in this vault that are stricter than it needs to be, and
+           prob is the worst problem (if any) that it covered.
+        """
+        for k in self.exceptions:
+            e = self.exceptions[k]
+            p = self.used_exception_for.get(k)
+            if p is None or e.is_worse_than(p):
+                yield (e, p)
+
+    def set_tolerances(self, fns):
+        """Adjust the tolerances for the exceptions in this vault.  Takes
+           a map of problem type to a function that adjusts the permitted
+           function to its new maximum value."""
+        for k in self.exceptions:
+            ex = self.exceptions[k]
+            fn = fns.get(ex.problem_type)
+            if fn is not None:
+                ex.metric_value = fn(ex.metric_value)
+
+class ProblemFilter(object):
+    def __init__(self):
+        self.thresholds = dict()
+
+    def addThreshold(self, item):
+        self.thresholds[(item.get_type(),item.get_file_type())] = item
+
+    def matches(self, item):
+        key = (item.get_type(), item.get_file_type())
+        filt = self.thresholds.get(key, None)
+        if filt is None:
+            return False
+        return item.is_worse_than(filt)
+
+    def filter(self, sequence):
+        for item in iter(sequence):
+            if self.matches(item):
+                yield item
+
+class Item(object):
+    """
+    A generic measurement about some aspect of our source code. See
+    the subclasses below for the specific problems we are trying to tackle.
+    """
+    def __init__(self, problem_type, problem_location, metric_value):
+        self.problem_location = problem_location
+        self.metric_value = int(metric_value)
+        self.warning_threshold = self.metric_value
+        self.problem_type = problem_type
+
+    def is_worse_than(self, other_problem):
+        """Return STATUS_ERR if this is a worse problem than other_problem.
+           Return STATUS_WARN if it is a little worse, but falls within the
+           warning threshold.  Return STATUS_OK if this problem is not
+           at all worse than other_problem.
+        """
+        if self.metric_value > other_problem.metric_value:
+            return STATUS_ERR
+        elif self.metric_value > other_problem.warning_threshold:
+            return STATUS_WARN
+        else:
+            return STATUS_OK
+
+    def key(self):
+        """Generate a unique key that describes this problem that can be used as a dictionary key"""
+        # Item location is a filesystem path, so we need to normalize this
+        # across platforms otherwise same paths are not gonna match.
+        canonical_location = os.path.normcase(self.problem_location)
+        return "%s:%s" % (canonical_location, self.problem_type)
+
+    def __str__(self):
+        return "problem %s %s %s" % (self.problem_type, self.problem_location, self.metric_value)
+
+    def get_type(self):
+        return self.problem_type
+
+    def get_file_type(self):
+        if self.problem_location.endswith(".h"):
+            return "*.h"
+        else:
+            return "*.c"
+
+class FileSizeItem(Item):
+    """
+    Denotes a problem with the size of a .c file.
+
+    The 'problem_location' is the filesystem path of the .c file, and the
+    'metric_value' is the number of lines in the .c file.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(FileSizeItem, self).__init__("file-size", problem_location, metric_value)
+
+class IncludeCountItem(Item):
+    """
+    Denotes a problem with the number of #includes in a .c file.
+
+    The 'problem_location' is the filesystem path of the .c file, and the
+    'metric_value' is the number of #includes in the .c file.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(IncludeCountItem, self).__init__("include-count", problem_location, metric_value)
+
+class FunctionSizeItem(Item):
+    """
+    Denotes a problem with a size of a function in a .c file.
+
+    The 'problem_location' is "<path>:<function>()" where <path> is the
+    filesystem path of the .c file and <function> is the name of the offending
+    function.
+
+    The 'metric_value' is the size of the offending function in lines.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(FunctionSizeItem, self).__init__("function-size", problem_location, metric_value)
+
+class DependencyViolationItem(Item):
+    """
+    Denotes a dependency violation in a .c or .h file.  A dependency violation
+    occurs when a file includes a file from some module that is not listed
+    in its .may_include file.
+
+    The 'problem_location' is the file that contains the problem.
+
+    The 'metric_value' is the number of forbidden includes.
+    """
+    def __init__(self, problem_location, metric_value):
+        super(DependencyViolationItem, self).__init__("dependency-violation",
+                                                      problem_location,
+                                                      metric_value)
+
+comment_re = re.compile(r'#.*$')
+
+def get_old_problem_from_exception_str(exception_str):
+    orig_str = exception_str
+    exception_str = comment_re.sub("", exception_str)
+    fields = exception_str.split()
+    if len(fields) == 0:
+        # empty line or comment
+        return None
+    elif len(fields) == 4:
+        # valid line
+        _, problem_type, problem_location, metric_value = fields
+    else:
+        raise ValueError("Misformatted line {!r}".format(orig_str))
+
+    if problem_type == "file-size":
+        return FileSizeItem(problem_location, metric_value)
+    elif problem_type == "include-count":
+        return IncludeCountItem(problem_location, metric_value)
+    elif problem_type == "function-size":
+        return FunctionSizeItem(problem_location, metric_value)
+    elif problem_type == "dependency-violation":
+        return DependencyViolationItem(problem_location, metric_value)
+    else:
+        raise ValueError("Unknown exception type {!r}".format(orig_str))
diff --git a/scripts/maint/practracker/test_practracker.sh b/scripts/maint/practracker/test_practracker.sh
new file mode 100755
index 0000000000..207a5ceded
--- /dev/null
+++ b/scripts/maint/practracker/test_practracker.sh
@@ -0,0 +1,70 @@
+#!/bin/sh
+
+umask 077
+unset TOR_DISABLE_PRACTRACKER
+
+TMPDIR=""
+clean () {
+  if [ -n "$TMPDIR" ] && [ -d "$TMPDIR" ]; then
+    rm -rf "$TMPDIR"
+  fi
+}
+trap clean EXIT HUP INT TERM
+
+if test "${PRACTRACKER_DIR}" = "" ||
+        test ! -e "${PRACTRACKER_DIR}/practracker.py" ; then
+    PRACTRACKER_DIR=$(dirname "$0")
+fi
+
+TMPDIR="$(mktemp -d -t pracktracker.test.XXXXXX)"
+if test -z "${TMPDIR}" || test ! -d "${TMPDIR}" ; then
+    echo >&2 "mktemp failed."
+    exit 1;
+fi
+
+DATA="${PRACTRACKER_DIR}/testdata"
+
+run_practracker() {
+    "${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker.py" \
+        --include-dir "" \
+        --max-file-size=0 \
+        --max-function-size=0 \
+        --max-h-file-size=0 \
+        --max-h-include-count=0 \
+        --max-include-count=0 \
+        --terse \
+        "${DATA}/" "$@";
+}
+compare() {
+    # we can't use cmp because we need to use -b for windows
+    diff -b -u "$@" > "${TMPDIR}/test-diff"
+    if test -z "$(cat "${TMPDIR}"/test-diff)"; then
+        echo "OK"
+    else
+        cat "${TMPDIR}/test-diff"
+        echo "FAILED"
+        exit 1
+    fi
+}
+
+echo "unit tests:"
+
+"${PYTHON:-python}" "${PRACTRACKER_DIR}/practracker_tests.py" || exit 1
+
+echo "ex0:"
+
+run_practracker --exceptions "${DATA}/ex0.txt" > "${TMPDIR}/ex0-received.txt"
+
+compare "${TMPDIR}/ex0-received.txt" "${DATA}/ex0-expected.txt"
+
+echo "ex1:"
+
+run_practracker --exceptions "${DATA}/ex1.txt" > "${TMPDIR}/ex1-received.txt"
+
+compare "${TMPDIR}/ex1-received.txt" "${DATA}/ex1-expected.txt"
+
+echo "ex1.overbroad:"
+
+run_practracker --exceptions "${DATA}/ex1.txt" --list-overbroad > "${TMPDIR}/ex1-overbroad-received.txt"
+
+compare "${TMPDIR}/ex1-overbroad-received.txt" "${DATA}/ex1-overbroad-expected.txt"
diff --git a/scripts/maint/practracker/testdata/.may_include b/scripts/maint/practracker/testdata/.may_include
new file mode 100644
index 0000000000..40bf8155d9
--- /dev/null
+++ b/scripts/maint/practracker/testdata/.may_include
@@ -0,0 +1,3 @@
+!advisory
+
+permitted.h
diff --git a/scripts/maint/practracker/testdata/a.c b/scripts/maint/practracker/testdata/a.c
new file mode 100644
index 0000000000..1939773f57
--- /dev/null
+++ b/scripts/maint/practracker/testdata/a.c
@@ -0,0 +1,38 @@
+
+#include "one.h"
+#include "two.h"
+#incldue "three.h"
+
+# include "permitted.h"
+
+int
+i_am_a_function(void)
+{
+  call();
+  call();
+  /* comment
+
+     another */
+
+  return 3;
+}
+
+#	include  "five.h"
+
+long
+another_function(long x,
+                 long y)
+{
+  int abcd;
+
+  abcd = x+y;
+  abcd *= abcd;
+
+  /* comment here */
+
+  return abcd +
+    abcd +
+    abcd;
+}
+
+/* And a comment to grow! */
diff --git a/scripts/maint/practracker/testdata/b.c b/scripts/maint/practracker/testdata/b.c
new file mode 100644
index 0000000000..bef277aaae
--- /dev/null
+++ b/scripts/maint/practracker/testdata/b.c
@@ -0,0 +1,15 @@
+
+MOCK_IMPL(int,
+foo,(void))
+{
+  // blah1
+  return 0;
+}
+
+MOCK_IMPL(int,
+bar,( long z))
+{
+  // blah2
+
+  return (int)(z+2);
+}
diff --git a/scripts/maint/practracker/testdata/ex.txt b/scripts/maint/practracker/testdata/ex.txt
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex.txt
diff --git a/scripts/maint/practracker/testdata/ex0-expected.txt b/scripts/maint/practracker/testdata/ex0-expected.txt
new file mode 100644
index 0000000000..5f3d9e5aec
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex0-expected.txt
@@ -0,0 +1,11 @@
+problem file-size a.c 38
+problem include-count a.c 4
+problem function-size a.c:i_am_a_function() 9
+problem function-size a.c:another_function() 12
+problem dependency-violation a.c 3
+problem file-size b.c 15
+problem function-size b.c:foo() 4
+problem function-size b.c:bar() 5
+problem file-size header.h 8
+problem include-count header.h 4
+problem dependency-violation header.h 3
diff --git a/scripts/maint/practracker/testdata/ex0.txt b/scripts/maint/practracker/testdata/ex0.txt
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex0.txt
diff --git a/scripts/maint/practracker/testdata/ex1-expected.txt b/scripts/maint/practracker/testdata/ex1-expected.txt
new file mode 100644
index 0000000000..58140a4d9a
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1-expected.txt
@@ -0,0 +1,3 @@
+problem function-size a.c:i_am_a_function() 9
+(warning) problem function-size a.c:another_function() 12
+problem function-size b.c:foo() 4
diff --git a/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt
new file mode 100644
index 0000000000..f69c608f40
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1-overbroad-expected.txt
@@ -0,0 +1,2 @@
+problem file-size a.c 40 -> 38
+problem file-size z.c 100 -> 0
diff --git a/scripts/maint/practracker/testdata/ex1.txt b/scripts/maint/practracker/testdata/ex1.txt
new file mode 100644
index 0000000000..c698005d07
--- /dev/null
+++ b/scripts/maint/practracker/testdata/ex1.txt
@@ -0,0 +1,18 @@
+
+problem file-size a.c 40
+problem include-count a.c 4
+# this problem will produce an error
+problem function-size a.c:i_am_a_function() 8
+# this problem will produce a warning
+problem function-size a.c:another_function() 11
+problem file-size b.c 15
+# This is removed, and so will produce an error.
+# problem function-size b.c:foo() 4
+# This exception isn't used.
+problem file-size z.c 100
+
+problem function-size b.c:bar() 5
+problem dependency-violation a.c 3
+problem dependency-violation header.h 3
+problem file-size header.h 8
+problem include-count header.h 4
diff --git a/scripts/maint/practracker/testdata/header.h b/scripts/maint/practracker/testdata/header.h
new file mode 100644
index 0000000000..1183f5db9a
--- /dev/null
+++ b/scripts/maint/practracker/testdata/header.h
@@ -0,0 +1,8 @@
+
+// some forbidden includes
+#include "foo.h"
+#include "quux.h"
+#include "quup.h"
+
+// a permitted include
+#include "permitted.h"
diff --git a/scripts/maint/practracker/testdata/not_c_file b/scripts/maint/practracker/testdata/not_c_file
new file mode 100644
index 0000000000..e150962c02
--- /dev/null
+++ b/scripts/maint/practracker/testdata/not_c_file
@@ -0,0 +1,2 @@
+
+This isn't a C file, so practracker shouldn't care about it.
diff --git a/scripts/maint/practracker/util.py b/scripts/maint/practracker/util.py
new file mode 100644
index 0000000000..df629110c2
--- /dev/null
+++ b/scripts/maint/practracker/util.py
@@ -0,0 +1,50 @@
+import os
+
+# We don't want to run metrics for unittests, automatically-generated C files,
+# external libraries or git leftovers.
+EXCLUDE_SOURCE_DIRS = {"src/test/", "src/trunnel/", "src/rust/",
+                       "src/ext/" }
+
+EXCLUDE_FILES = {"orconfig.h"}
+
+def _norm(p):
+    return os.path.normcase(os.path.normpath(p))
+
+def get_tor_c_files(tor_topdir, include_dirs=None):
+    """
+    Return a list with the .c and .h filenames we want to get metrics of.
+    """
+    files_list = []
+    exclude_dirs = { _norm(os.path.join(tor_topdir, p)) for p in EXCLUDE_SOURCE_DIRS }
+
+    if include_dirs is None:
+        topdirs = [ tor_topdir ]
+    else:
+        topdirs = [ os.path.join(tor_topdir, inc) for inc in include_dirs ]
+
+    for topdir in topdirs:
+        for root, directories, filenames in os.walk(topdir):
+            # Remove all the directories that are excluded.
+            directories[:] = [ d for d in directories
+                               if _norm(os.path.join(root,d)) not in exclude_dirs ]
+            directories.sort()
+            filenames.sort()
+            for filename in filenames:
+                # We only care about .c and .h files
+                if not (filename.endswith(".c") or filename.endswith(".h")):
+                    continue
+                if filename in EXCLUDE_FILES:
+                    continue
+
+                full_path = os.path.join(root,filename)
+
+                files_list.append(full_path)
+
+    return files_list
+
+class NullFile:
+    """A file-like object that we can us to suppress output."""
+    def __init__(self):
+        pass
+    def write(self, s):
+        pass
diff --git a/scripts/maint/rectify_include_paths.py b/scripts/maint/rectify_include_paths.py
index 401fadae6d..1140e8cd22 100755
--- a/scripts/maint/rectify_include_paths.py
+++ b/scripts/maint/rectify_include_paths.py
@@ -1,8 +1,12 @@
-#!/usr/bin/python3
+#!/usr/bin/python
 
 import os
 import os.path
 import re
+import sys
+
+def warn(msg):
+    sys.stderr.write("WARNING: %s\n"%msg)
 
 # Find all the include files, map them to their real names.
 
@@ -11,6 +15,8 @@ def exclude(paths, dirnames):
         if p in dirnames:
             dirnames.remove(p)
 
+DUPLICATE = object()
+
 def get_include_map():
     includes = { }
 
@@ -19,7 +25,10 @@ def get_include_map():
 
         for fname in fnames:
             if fname.endswith(".h"):
-                assert fname not in includes
+                if fname in includes:
+                    warn("Multiple headers named %s"%fname)
+                    includes[fname] = DUPLICATE
+                    continue
                 include = os.path.join(dirpath, fname)
                 assert include.startswith("src/")
                 includes[fname] = include[4:]
@@ -37,7 +46,7 @@ def fix_includes(inp, out, mapping):
         if m:
             include,hdr,rest = m.groups()
             basehdr = get_base_header_name(hdr)
-            if basehdr in mapping:
+            if basehdr in mapping and mapping[basehdr] is not DUPLICATE:
                 out.write('{}{}{}\n'.format(include,mapping[basehdr],rest))
                 continue
 
diff --git a/scripts/maint/run_calltool.sh b/scripts/maint/run_calltool.sh
index efb8706fea..b0268322f4 100755
--- a/scripts/maint/run_calltool.sh
+++ b/scripts/maint/run_calltool.sh
@@ -15,10 +15,10 @@ SUBITEMS="fn_graph fn_invgraph fn_scc fn_scc_weaklinks module_graph module_invgr
 
 for calculation in $SUBITEMS; do
   echo "======== $calculation"
-  python -m calltool $calculation > callgraph/$calculation
+  python -m calltool "$calculation" > callgraph/"$calculation"
 done
 
-echo <<EOF > callgraph/README
+cat <<EOF > callgraph/README
 This directory holds output from calltool, as run on Tor.  For more
 information about each of these files, see the NOTES and README files in
 the calltool distribution.
diff --git a/scripts/maint/updateCopyright.pl b/scripts/maint/updateCopyright.pl
index bd24377d38..6800032f87 100755
--- a/scripts/maint/updateCopyright.pl
+++ b/scripts/maint/updateCopyright.pl
@@ -1,7 +1,9 @@
 #!/usr/bin/perl -i -w -p
 
-$NEWYEAR=2018;
+@now = gmtime();
 
-s/Copyright(.*) (201[^8]), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
+$NEWYEAR=$now[5]+1900;
+
+s/Copyright([^-]*) (20[^-]*), The Tor Project/Copyright$1 $2-${NEWYEAR}, The Tor Project/;
 
 s/Copyright(.*)-(20..), The Tor Project/Copyright$1-${NEWYEAR}, The Tor Project/;
diff --git a/scripts/maint/updateFallbackDirs.py b/scripts/maint/updateFallbackDirs.py
deleted file mode 100755
index 0ea3992d8f..0000000000
--- a/scripts/maint/updateFallbackDirs.py
+++ /dev/null
@@ -1,2216 +0,0 @@
-#!/usr/bin/env python
-
-# Usage:
-#
-# Regenerate the list:
-# scripts/maint/updateFallbackDirs.py > src/app/config/fallback_dirs.inc 2> fallback_dirs.log
-#
-# Check the existing list:
-# scripts/maint/updateFallbackDirs.py check_existing > fallback_dirs.inc.ok 2> fallback_dirs.log
-# mv fallback_dirs.inc.ok src/app/config/fallback_dirs.inc
-#
-# This script should be run from a stable, reliable network connection,
-# with no other network activity (and not over tor).
-# If this is not possible, please disable:
-# PERFORM_IPV4_DIRPORT_CHECKS and PERFORM_IPV6_DIRPORT_CHECKS
-#
-# Needs dateutil, stem, and potentially other python packages.
-# Optionally uses ipaddress (python 3 builtin) or py2-ipaddress (package)
-# for netblock analysis.
-#
-# Then read the logs to make sure the fallbacks aren't dominated by a single
-# netblock or port.
-
-# Script by weasel, April 2015
-# Portions by gsathya & karsten, 2013
-# https://trac.torproject.org/projects/tor/attachment/ticket/8374/dir_list.2.py
-# Modifications by teor, 2015
-
-import StringIO
-import string
-import re
-import datetime
-import gzip
-import os.path
-import json
-import math
-import sys
-import urllib
-import urllib2
-import hashlib
-import dateutil.parser
-# bson_lazy provides bson
-#from bson import json_util
-import copy
-import re
-
-from stem.descriptor import DocumentHandler
-from stem.descriptor.remote import get_consensus, get_server_descriptors, MAX_FINGERPRINTS
-
-import logging
-logging.root.name = ''
-
-HAVE_IPADDRESS = False
-try:
-  # python 3 builtin, or install package py2-ipaddress
-  # there are several ipaddress implementations for python 2
-  # with slightly different semantics with str typed text
-  # fortunately, all our IP addresses are in unicode
-  import ipaddress
-  HAVE_IPADDRESS = True
-except ImportError:
-  # if this happens, we avoid doing netblock analysis
-  logging.warning('Unable to import ipaddress, please install py2-ipaddress.' +
-                  ' A fallback list will be created, but optional netblock' +
-                  ' analysis will not be performed.')
-
-## Top-Level Configuration
-
-# We use semantic versioning: https://semver.org
-# In particular:
-# * major changes include removing a mandatory field, or anything else that
-#   would break an appropriately tolerant parser,
-# * minor changes include adding a field,
-# * patch changes include changing header comments or other unstructured
-#   content
-FALLBACK_FORMAT_VERSION = '2.0.0'
-SECTION_SEPARATOR_BASE = '====='
-SECTION_SEPARATOR_COMMENT = '/* ' + SECTION_SEPARATOR_BASE + ' */'
-
-# Output all candidate fallbacks, or only output selected fallbacks?
-OUTPUT_CANDIDATES = False
-
-# Perform DirPort checks over IPv4?
-# Change this to False if IPv4 doesn't work for you, or if you don't want to
-# download a consensus for each fallback
-# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
-PERFORM_IPV4_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else True
-
-# Perform DirPort checks over IPv6?
-# If you know IPv6 works for you, set this to True
-# This will exclude IPv6 relays without an IPv6 DirPort configured
-# So it's best left at False until #18394 is implemented
-# Don't check ~1000 candidates when OUTPUT_CANDIDATES is True
-PERFORM_IPV6_DIRPORT_CHECKS = False if OUTPUT_CANDIDATES else False
-
-# Must relays be running now?
-MUST_BE_RUNNING_NOW = (PERFORM_IPV4_DIRPORT_CHECKS
-                       or PERFORM_IPV6_DIRPORT_CHECKS)
-
-# Clients have been using microdesc consensuses by default for a while now
-DOWNLOAD_MICRODESC_CONSENSUS = True
-
-# If a relay delivers an expired consensus, if it expired less than this many
-# seconds ago, we still allow the relay. This should never be less than -90,
-# as all directory mirrors should have downloaded a consensus 90 minutes
-# before it expires. It should never be more than 24 hours, because clients
-# reject consensuses that are older than REASONABLY_LIVE_TIME.
-# For the consensus expiry check to be accurate, the machine running this
-# script needs an accurate clock.
-#
-# Relays on 0.3.0 and later return a 404 when they are about to serve an
-# expired consensus. This makes them fail the download check.
-# We use a tolerance of 0, so that 0.2.x series relays also fail the download
-# check if they serve an expired consensus.
-CONSENSUS_EXPIRY_TOLERANCE = 0
-
-# Output fallback name, flags, bandwidth, and ContactInfo in a C comment?
-OUTPUT_COMMENTS = True if OUTPUT_CANDIDATES else False
-
-# Output matching ContactInfo in fallbacks list?
-# Useful if you're trying to contact operators
-CONTACT_COUNT = True if OUTPUT_CANDIDATES else False
-
-# How the list should be sorted:
-# fingerprint: is useful for stable diffs of fallback lists
-# measured_bandwidth: is useful when pruning the list based on bandwidth
-# contact: is useful for contacting operators once the list has been pruned
-OUTPUT_SORT_FIELD = 'contact' if OUTPUT_CANDIDATES else 'fingerprint'
-
-## OnionOO Settings
-
-ONIONOO = 'https://onionoo.torproject.org/'
-#ONIONOO = 'https://onionoo.thecthulhu.com/'
-
-# Don't bother going out to the Internet, just use the files available locally,
-# even if they're very old
-LOCAL_FILES_ONLY = False
-
-## Whitelist / Blacklist Filter Settings
-
-# The whitelist contains entries that are included if all attributes match
-# (IPv4, dirport, orport, id, and optionally IPv6 and IPv6 orport)
-
-# What happens to entries not in whitelist?
-# When True, they are included, when False, they are excluded
-INCLUDE_UNLISTED_ENTRIES = True if OUTPUT_CANDIDATES else False
-
-WHITELIST_FILE_NAME = 'scripts/maint/fallback.whitelist'
-FALLBACK_FILE_NAME  = 'src/app/config/fallback_dirs.inc'
-
-# The number of bytes we'll read from a filter file before giving up
-MAX_LIST_FILE_SIZE = 1024 * 1024
-
-## Eligibility Settings
-
-# Require fallbacks to have the same address and port for a set amount of time
-# We used to have this at 1 week, but that caused many fallback failures, which
-# meant that we had to rebuild the list more often. We want fallbacks to be
-# stable for 2 years, so we set it to a few months.
-#
-# If a relay changes address or port, that's it, it's not useful any more,
-# because clients can't find it
-ADDRESS_AND_PORT_STABLE_DAYS = 90
-# We ignore relays that have been down for more than this period
-MAX_DOWNTIME_DAYS = 0 if MUST_BE_RUNNING_NOW else 7
-# FallbackDirs must have a time-weighted-fraction that is greater than or
-# equal to:
-# Mirrors that are down half the time are still useful half the time
-CUTOFF_RUNNING = .50
-CUTOFF_V2DIR = .50
-# Guard flags are removed for some time after a relay restarts, so we ignore
-# the guard flag.
-CUTOFF_GUARD = .00
-# FallbackDirs must have a time-weighted-fraction that is less than or equal
-# to:
-# .00 means no bad exits
-PERMITTED_BADEXIT = .00
-
-# older entries' weights are adjusted with ALPHA^(age in days)
-AGE_ALPHA = 0.99
-
-# this factor is used to scale OnionOO entries to [0,1]
-ONIONOO_SCALE_ONE = 999.
-
-## Fallback Count Limits
-
-# The target for these parameters is 20% of the guards in the network
-# This is around 200 as of October 2015
-_FB_POG = 0.2
-FALLBACK_PROPORTION_OF_GUARDS = None if OUTPUT_CANDIDATES else _FB_POG
-
-# Limit the number of fallbacks (eliminating lowest by advertised bandwidth)
-MAX_FALLBACK_COUNT = None if OUTPUT_CANDIDATES else 200
-# Emit a C #error if the number of fallbacks is less than expected
-MIN_FALLBACK_COUNT = 0 if OUTPUT_CANDIDATES else MAX_FALLBACK_COUNT*0.5
-
-# The maximum number of fallbacks on the same address, contact, or family
-#
-# With 150 fallbacks, this means each operator sees 5% of client bootstraps.
-# For comparison:
-#  - We try to limit guard and exit operators to 5% of the network
-#  - The directory authorities used to see 11% of client bootstraps each
-#
-# We also don't want too much of the list to go down if a single operator
-# has to move all their relays.
-MAX_FALLBACKS_PER_IP = 1
-MAX_FALLBACKS_PER_IPV4 = MAX_FALLBACKS_PER_IP
-MAX_FALLBACKS_PER_IPV6 = MAX_FALLBACKS_PER_IP
-MAX_FALLBACKS_PER_CONTACT = 7
-MAX_FALLBACKS_PER_FAMILY = 7
-
-## Fallback Bandwidth Requirements
-
-# Any fallback with the Exit flag has its bandwidth multiplied by this fraction
-# to make sure we aren't further overloading exits
-# (Set to 1.0, because we asked that only lightly loaded exits opt-in,
-# and the extra load really isn't that much for large relays.)
-EXIT_BANDWIDTH_FRACTION = 1.0
-
-# If a single fallback's bandwidth is too low, it's pointless adding it
-# We expect fallbacks to handle an extra 10 kilobytes per second of traffic
-# Make sure they can support fifty times the expected extra load
-#
-# We convert this to a consensus weight before applying the filter,
-# because all the bandwidth amounts are specified by the relay
-MIN_BANDWIDTH = 50.0 * 10.0 * 1024.0
-
-# Clients will time out after 30 seconds trying to download a consensus
-# So allow fallback directories half that to deliver a consensus
-# The exact download times might change based on the network connection
-# running this script, but only by a few seconds
-# There is also about a second of python overhead
-CONSENSUS_DOWNLOAD_SPEED_MAX = 15.0
-# If the relay fails a consensus check, retry the download
-# This avoids delisting a relay due to transient network conditions
-CONSENSUS_DOWNLOAD_RETRY = True
-
-## Parsing Functions
-
-def parse_ts(t):
-  return datetime.datetime.strptime(t, "%Y-%m-%d %H:%M:%S")
-
-def remove_bad_chars(raw_string, bad_char_list):
-  # Remove each character in the bad_char_list
-  cleansed_string = raw_string
-  for c in bad_char_list:
-    cleansed_string = cleansed_string.replace(c, '')
-  return cleansed_string
-
-def cleanse_unprintable(raw_string):
-  # Remove all unprintable characters
-  cleansed_string = ''
-  for c in raw_string:
-    if c in string.printable:
-      cleansed_string += c
-  return cleansed_string
-
-def cleanse_whitespace(raw_string):
-  # Replace all whitespace characters with a space
-  cleansed_string = raw_string
-  for c in string.whitespace:
-    cleansed_string = cleansed_string.replace(c, ' ')
-  return cleansed_string
-
-def cleanse_c_multiline_comment(raw_string):
-  cleansed_string = raw_string
-  # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
-  cleansed_string = cleanse_whitespace(cleansed_string)
-  # ContactInfo and Version can be arbitrary binary data
-  cleansed_string = cleanse_unprintable(cleansed_string)
-  # Prevent a malicious / unanticipated string from breaking out
-  # of a C-style multiline comment
-  # This removes '/*' and '*/' and '//'
-  bad_char_list = '*/'
-  # Prevent a malicious string from using C nulls
-  bad_char_list += '\0'
-  # Avoid confusing parsers by making sure there is only one comma per fallback
-  bad_char_list += ','
-  # Avoid confusing parsers by making sure there is only one equals per field
-  bad_char_list += '='
-  # Be safer by removing bad characters entirely
-  cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
-  # Some compilers may further process the content of comments
-  # There isn't much we can do to cover every possible case
-  # But comment-based directives are typically only advisory
-  return cleansed_string
-
-def cleanse_c_string(raw_string):
-  cleansed_string = raw_string
-  # Embedded newlines should be removed by tor/onionoo, but let's be paranoid
-  cleansed_string = cleanse_whitespace(cleansed_string)
-  # ContactInfo and Version can be arbitrary binary data
-  cleansed_string = cleanse_unprintable(cleansed_string)
-  # Prevent a malicious address/fingerprint string from breaking out
-  # of a C-style string
-  bad_char_list = '"'
-  # Prevent a malicious string from using escapes
-  bad_char_list += '\\'
-  # Prevent a malicious string from using C nulls
-  bad_char_list += '\0'
-  # Avoid confusing parsers by making sure there is only one comma per fallback
-  bad_char_list += ','
-  # Avoid confusing parsers by making sure there is only one equals per field
-  bad_char_list += '='
-  # Be safer by removing bad characters entirely
-  cleansed_string = remove_bad_chars(cleansed_string, bad_char_list)
-  # Some compilers may further process the content of strings
-  # There isn't much we can do to cover every possible case
-  # But this typically only results in changes to the string data
-  return cleansed_string
-
-## OnionOO Source Functions
-
-# a dictionary of source metadata for each onionoo query we've made
-fetch_source = {}
-
-# register source metadata for 'what'
-# assumes we only retrieve one document for each 'what'
-def register_fetch_source(what, url, relays_published, version):
-  fetch_source[what] = {}
-  fetch_source[what]['url'] = url
-  fetch_source[what]['relays_published'] = relays_published
-  fetch_source[what]['version'] = version
-
-# list each registered source's 'what'
-def fetch_source_list():
-  return sorted(fetch_source.keys())
-
-# given 'what', provide a multiline C comment describing the source
-def describe_fetch_source(what):
-  desc = '/*'
-  desc += '\n'
-  desc += 'Onionoo Source: '
-  desc += cleanse_c_multiline_comment(what)
-  desc += ' Date: '
-  desc += cleanse_c_multiline_comment(fetch_source[what]['relays_published'])
-  desc += ' Version: '
-  desc += cleanse_c_multiline_comment(fetch_source[what]['version'])
-  desc += '\n'
-  desc += 'URL: '
-  desc += cleanse_c_multiline_comment(fetch_source[what]['url'])
-  desc += '\n'
-  desc += '*/'
-  return desc
-
-## File Processing Functions
-
-def write_to_file(str, file_name, max_len):
-  try:
-    with open(file_name, 'w') as f:
-      f.write(str[0:max_len])
-  except EnvironmentError, error:
-    logging.error('Writing file %s failed: %d: %s'%
-                  (file_name,
-                   error.errno,
-                   error.strerror)
-                  )
-
-def read_from_file(file_name, max_len):
-  try:
-    if os.path.isfile(file_name):
-      with open(file_name, 'r') as f:
-        return f.read(max_len)
-  except EnvironmentError, error:
-    logging.info('Loading file %s failed: %d: %s'%
-                 (file_name,
-                  error.errno,
-                  error.strerror)
-                 )
-  return None
-
-def parse_fallback_file(file_name):
-  file_data = read_from_file(file_name, MAX_LIST_FILE_SIZE)
-  file_data = cleanse_unprintable(file_data)
-  file_data = remove_bad_chars(file_data, '\n"\0')
-  file_data = re.sub('/\*.*?\*/', '', file_data)
-  file_data = file_data.replace(',', '\n')
-  file_data = file_data.replace(' weight=10', '')
-  return file_data
-
-def load_possibly_compressed_response_json(response):
-    if response.info().get('Content-Encoding') == 'gzip':
-      buf = StringIO.StringIO( response.read() )
-      f = gzip.GzipFile(fileobj=buf)
-      return json.load(f)
-    else:
-      return json.load(response)
-
-def load_json_from_file(json_file_name):
-    # An exception here may be resolved by deleting the .last_modified
-    # and .json files, and re-running the script
-    try:
-      with open(json_file_name, 'r') as f:
-        return json.load(f)
-    except EnvironmentError, error:
-      raise Exception('Reading not-modified json file %s failed: %d: %s'%
-                    (json_file_name,
-                     error.errno,
-                     error.strerror)
-                    )
-
-## OnionOO Functions
-
-def datestr_to_datetime(datestr):
-  # Parse datetimes like: Fri, 02 Oct 2015 13:34:14 GMT
-  if datestr is not None:
-    dt = dateutil.parser.parse(datestr)
-  else:
-    # Never modified - use start of epoch
-    dt = datetime.datetime.utcfromtimestamp(0)
-  # strip any timezone out (in case they're supported in future)
-  dt = dt.replace(tzinfo=None)
-  return dt
-
-def onionoo_fetch(what, **kwargs):
-  params = kwargs
-  params['type'] = 'relay'
-  #params['limit'] = 10
-  params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
-  params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
-  params['flag'] = 'V2Dir'
-  url = ONIONOO + what + '?' + urllib.urlencode(params)
-
-  # Unfortunately, the URL is too long for some OS filenames,
-  # but we still don't want to get files from different URLs mixed up
-  base_file_name = what + '-' + hashlib.sha1(url).hexdigest()
-
-  full_url_file_name = base_file_name + '.full_url'
-  MAX_FULL_URL_LENGTH = 1024
-
-  last_modified_file_name = base_file_name + '.last_modified'
-  MAX_LAST_MODIFIED_LENGTH = 64
-
-  json_file_name = base_file_name + '.json'
-
-  if LOCAL_FILES_ONLY:
-    # Read from the local file, don't write to anything
-    response_json = load_json_from_file(json_file_name)
-  else:
-    # store the full URL to a file for debugging
-    # no need to compare as long as you trust SHA-1
-    write_to_file(url, full_url_file_name, MAX_FULL_URL_LENGTH)
-
-    request = urllib2.Request(url)
-    request.add_header('Accept-encoding', 'gzip')
-
-    # load the last modified date from the file, if it exists
-    last_mod_date = read_from_file(last_modified_file_name,
-                                   MAX_LAST_MODIFIED_LENGTH)
-    if last_mod_date is not None:
-      request.add_header('If-modified-since', last_mod_date)
-
-    # Parse last modified date
-    last_mod = datestr_to_datetime(last_mod_date)
-
-    # Not Modified and still recent enough to be useful
-    # Onionoo / Globe used to use 6 hours, but we can afford a day
-    required_freshness = datetime.datetime.utcnow()
-    # strip any timezone out (to match dateutil.parser)
-    required_freshness = required_freshness.replace(tzinfo=None)
-    required_freshness -= datetime.timedelta(hours=24)
-
-    # Make the OnionOO request
-    response_code = 0
-    try:
-      response = urllib2.urlopen(request)
-      response_code = response.getcode()
-    except urllib2.HTTPError, error:
-      response_code = error.code
-      if response_code == 304: # not modified
-        pass
-      else:
-        raise Exception("Could not get " + url + ": "
-                        + str(error.code) + ": " + error.reason)
-
-    if response_code == 200: # OK
-      last_mod = datestr_to_datetime(response.info().get('Last-Modified'))
-
-    # Check for freshness
-    if last_mod < required_freshness:
-      if last_mod_date is not None:
-        # This check sometimes fails transiently, retry the script if it does
-        date_message = "Outdated data: last updated " + last_mod_date
-      else:
-        date_message = "No data: never downloaded "
-      raise Exception(date_message + " from " + url)
-
-    # Process the data
-    if response_code == 200: # OK
-
-      response_json = load_possibly_compressed_response_json(response)
-
-      with open(json_file_name, 'w') as f:
-        # use the most compact json representation to save space
-        json.dump(response_json, f, separators=(',',':'))
-
-      # store the last modified date in its own file
-      if response.info().get('Last-modified') is not None:
-        write_to_file(response.info().get('Last-Modified'),
-                      last_modified_file_name,
-                      MAX_LAST_MODIFIED_LENGTH)
-
-    elif response_code == 304: # Not Modified
-
-      response_json = load_json_from_file(json_file_name)
-
-    else: # Unexpected HTTP response code not covered in the HTTPError above
-      raise Exception("Unexpected HTTP response code to " + url + ": "
-                      + str(response_code))
-
-  register_fetch_source(what,
-                        url,
-                        response_json['relays_published'],
-                        response_json['version'])
-
-  return response_json
-
-def fetch(what, **kwargs):
-  #x = onionoo_fetch(what, **kwargs)
-  # don't use sort_keys, as the order of or_addresses is significant
-  #print json.dumps(x, indent=4, separators=(',', ': '))
-  #sys.exit(0)
-
-  return onionoo_fetch(what, **kwargs)
-
-## Fallback Candidate Class
-
-class Candidate(object):
-  CUTOFF_ADDRESS_AND_PORT_STABLE = (datetime.datetime.utcnow()
-                            - datetime.timedelta(ADDRESS_AND_PORT_STABLE_DAYS))
-
-  def __init__(self, details):
-    for f in ['fingerprint', 'nickname', 'last_changed_address_or_port',
-              'consensus_weight', 'or_addresses', 'dir_address']:
-      if not f in details: raise Exception("Document has no %s field."%(f,))
-
-    if not 'contact' in details:
-      details['contact'] = None
-    if not 'flags' in details or details['flags'] is None:
-      details['flags'] = []
-    if (not 'advertised_bandwidth' in details
-        or details['advertised_bandwidth'] is None):
-      # relays without advertised bandwidth have it calculated from their
-      # consensus weight
-      details['advertised_bandwidth'] = 0
-    if (not 'effective_family' in details
-        or details['effective_family'] is None):
-      details['effective_family'] = []
-    if not 'platform' in details:
-      details['platform'] = None
-    details['last_changed_address_or_port'] = parse_ts(
-                                      details['last_changed_address_or_port'])
-    self._data = details
-    self._stable_sort_or_addresses()
-
-    self._fpr = self._data['fingerprint']
-    self._running = self._guard = self._v2dir = 0.
-    self._split_dirport()
-    self._compute_orport()
-    if self.orport is None:
-      raise Exception("Failed to get an orport for %s."%(self._fpr,))
-    self._compute_ipv6addr()
-    if not self.has_ipv6():
-      logging.debug("Failed to get an ipv6 address for %s."%(self._fpr,))
-    self._compute_version()
-    self._extra_info_cache = None
-
-  def _stable_sort_or_addresses(self):
-    # replace self._data['or_addresses'] with a stable ordering,
-    # sorting the secondary addresses in string order
-    # leave the received order in self._data['or_addresses_raw']
-    self._data['or_addresses_raw'] = self._data['or_addresses']
-    or_address_primary = self._data['or_addresses'][:1]
-    # subsequent entries in the or_addresses array are in an arbitrary order
-    # so we stabilise the addresses by sorting them in string order
-    or_addresses_secondaries_stable = sorted(self._data['or_addresses'][1:])
-    or_addresses_stable = or_address_primary + or_addresses_secondaries_stable
-    self._data['or_addresses'] = or_addresses_stable
-
-  def get_fingerprint(self):
-    return self._fpr
-
-  # is_valid_ipv[46]_address by gsathya, karsten, 2013
-  @staticmethod
-  def is_valid_ipv4_address(address):
-    if not isinstance(address, (str, unicode)):
-      return False
-
-    # check if there are four period separated values
-    if address.count(".") != 3:
-      return False
-
-    # checks that each value in the octet are decimal values between 0-255
-    for entry in address.split("."):
-      if not entry.isdigit() or int(entry) < 0 or int(entry) > 255:
-        return False
-      elif entry[0] == "0" and len(entry) > 1:
-        return False  # leading zeros, for instance in "1.2.3.001"
-
-    return True
-
-  @staticmethod
-  def is_valid_ipv6_address(address):
-    if not isinstance(address, (str, unicode)):
-      return False
-
-    # remove brackets
-    address = address[1:-1]
-
-    # addresses are made up of eight colon separated groups of four hex digits
-    # with leading zeros being optional
-    # https://en.wikipedia.org/wiki/IPv6#Address_format
-
-    colon_count = address.count(":")
-
-    if colon_count > 7:
-      return False  # too many groups
-    elif colon_count != 7 and not "::" in address:
-      return False  # not enough groups and none are collapsed
-    elif address.count("::") > 1 or ":::" in address:
-      return False  # multiple groupings of zeros can't be collapsed
-
-    found_ipv4_on_previous_entry = False
-    for entry in address.split(":"):
-      # If an IPv6 address has an embedded IPv4 address,
-      # it must be the last entry
-      if found_ipv4_on_previous_entry:
-        return False
-      if not re.match("^[0-9a-fA-f]{0,4}$", entry):
-        if not Candidate.is_valid_ipv4_address(entry):
-          return False
-        else:
-          found_ipv4_on_previous_entry = True
-
-    return True
-
-  def _split_dirport(self):
-    # Split the dir_address into dirip and dirport
-    (self.dirip, _dirport) = self._data['dir_address'].split(':', 2)
-    self.dirport = int(_dirport)
-
-  def _compute_orport(self):
-    # Choose the first ORPort that's on the same IPv4 address as the DirPort.
-    # In rare circumstances, this might not be the primary ORPort address.
-    # However, _stable_sort_or_addresses() ensures we choose the same one
-    # every time, even if onionoo changes the order of the secondaries.
-    self._split_dirport()
-    self.orport = None
-    for i in self._data['or_addresses']:
-      if i != self._data['or_addresses'][0]:
-        logging.debug('Secondary IPv4 Address Used for %s: %s'%(self._fpr, i))
-      (ipaddr, port) = i.rsplit(':', 1)
-      if (ipaddr == self.dirip) and Candidate.is_valid_ipv4_address(ipaddr):
-        self.orport = int(port)
-        return
-
-  def _compute_ipv6addr(self):
-    # Choose the first IPv6 address that uses the same port as the ORPort
-    # Or, choose the first IPv6 address in the list
-    # _stable_sort_or_addresses() ensures we choose the same IPv6 address
-    # every time, even if onionoo changes the order of the secondaries.
-    self.ipv6addr = None
-    self.ipv6orport = None
-    # Choose the first IPv6 address that uses the same port as the ORPort
-    for i in self._data['or_addresses']:
-      (ipaddr, port) = i.rsplit(':', 1)
-      if (port == self.orport) and Candidate.is_valid_ipv6_address(ipaddr):
-        self.ipv6addr = ipaddr
-        self.ipv6orport = int(port)
-        return
-    # Choose the first IPv6 address in the list
-    for i in self._data['or_addresses']:
-      (ipaddr, port) = i.rsplit(':', 1)
-      if Candidate.is_valid_ipv6_address(ipaddr):
-        self.ipv6addr = ipaddr
-        self.ipv6orport = int(port)
-        return
-
-  def _compute_version(self):
-    # parse the version out of the platform string
-    # The platform looks like: "Tor 0.2.7.6 on Linux"
-    self._data['version'] = None
-    if self._data['platform'] is None:
-      return
-    # be tolerant of weird whitespacing, use a whitespace split
-    tokens = self._data['platform'].split()
-    for token in tokens:
-      vnums = token.split('.')
-      # if it's at least a.b.c.d, with potentially an -alpha-dev, -alpha, -rc
-      if (len(vnums) >= 4 and vnums[0].isdigit() and vnums[1].isdigit() and
-          vnums[2].isdigit()):
-        self._data['version'] = token
-        return
-
-  # From #20509
-  # bug #20499 affects versions from 0.2.9.1-alpha-dev to 0.2.9.4-alpha-dev
-  # and version 0.3.0.0-alpha-dev
-  # Exhaustive lists are hard to get wrong
-  STALE_CONSENSUS_VERSIONS = ['0.2.9.1-alpha-dev',
-                              '0.2.9.2-alpha',
-                              '0.2.9.2-alpha-dev',
-                              '0.2.9.3-alpha',
-                              '0.2.9.3-alpha-dev',
-                              '0.2.9.4-alpha',
-                              '0.2.9.4-alpha-dev',
-                              '0.3.0.0-alpha-dev'
-                              ]
-
-  def is_valid_version(self):
-    # call _compute_version before calling this
-    # is the version of the relay a version we want as a fallback?
-    # checks both recommended versions and bug #20499 / #20509
-    #
-    # if the relay doesn't have a recommended version field, exclude the relay
-    if not self._data.has_key('recommended_version'):
-      log_excluded('%s not a candidate: no recommended_version field',
-                   self._fpr)
-      return False
-    if not self._data['recommended_version']:
-      log_excluded('%s not a candidate: version not recommended', self._fpr)
-      return False
-    # if the relay doesn't have version field, exclude the relay
-    if not self._data.has_key('version'):
-      log_excluded('%s not a candidate: no version field', self._fpr)
-      return False
-    if self._data['version'] in Candidate.STALE_CONSENSUS_VERSIONS:
-      logging.warning('%s not a candidate: version delivers stale consensuses',
-                      self._fpr)
-      return False
-    return True
-
-  @staticmethod
-  def _extract_generic_history(history, which='unknown'):
-    # given a tree like this:
-    #   {
-    #     "1_month": {
-    #         "count": 187,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2015-02-27 06:00:00",
-    #         "interval": 14400,
-    #         "last": "2015-03-30 06:00:00",
-    #         "values": [
-    #             999,
-    #             999
-    #         ]
-    #     },
-    #     "1_week": {
-    #         "count": 169,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2015-03-23 07:30:00",
-    #         "interval": 3600,
-    #         "last": "2015-03-30 07:30:00",
-    #         "values": [ ...]
-    #     },
-    #     "1_year": {
-    #         "count": 177,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2014-04-11 00:00:00",
-    #         "interval": 172800,
-    #         "last": "2015-03-29 00:00:00",
-    #         "values": [ ...]
-    #     },
-    #     "3_months": {
-    #         "count": 185,
-    #         "factor": 0.001001001001001001,
-    #         "first": "2014-12-28 06:00:00",
-    #         "interval": 43200,
-    #         "last": "2015-03-30 06:00:00",
-    #         "values": [ ...]
-    #     }
-    #   },
-    # extract exactly one piece of data per time interval,
-    # using smaller intervals where available.
-    #
-    # returns list of (age, length, value) dictionaries.
-
-    generic_history = []
-
-    periods = history.keys()
-    periods.sort(key = lambda x: history[x]['interval'])
-    now = datetime.datetime.utcnow()
-    newest = now
-    for p in periods:
-      h = history[p]
-      interval = datetime.timedelta(seconds = h['interval'])
-      this_ts = parse_ts(h['last'])
-
-      if (len(h['values']) != h['count']):
-        logging.warning('Inconsistent value count in %s document for %s'
-                        %(p, which))
-      for v in reversed(h['values']):
-        if (this_ts <= newest):
-          agt1 = now - this_ts
-          agt2 = interval
-          agetmp1 = (agt1.microseconds + (agt1.seconds + agt1.days * 24 * 3600)
-                     * 10**6) / 10**6
-          agetmp2 = (agt2.microseconds + (agt2.seconds + agt2.days * 24 * 3600)
-                     * 10**6) / 10**6
-          generic_history.append(
-            { 'age': agetmp1,
-              'length': agetmp2,
-              'value': v
-            })
-          newest = this_ts
-        this_ts -= interval
-
-      if (this_ts + interval != parse_ts(h['first'])):
-        logging.warning('Inconsistent time information in %s document for %s'
-                        %(p, which))
-
-    #print json.dumps(generic_history, sort_keys=True,
-    #                  indent=4, separators=(',', ': '))
-    return generic_history
-
-  @staticmethod
-  def _avg_generic_history(generic_history):
-    a = []
-    for i in generic_history:
-      if i['age'] > (ADDRESS_AND_PORT_STABLE_DAYS * 24 * 3600):
-        continue
-      if (i['length'] is not None
-          and i['age'] is not None
-          and i['value'] is not None):
-        w = i['length'] * math.pow(AGE_ALPHA, i['age']/(3600*24))
-        a.append( (i['value'] * w, w) )
-
-    sv = math.fsum(map(lambda x: x[0], a))
-    sw = math.fsum(map(lambda x: x[1], a))
-
-    if sw == 0.0:
-      svw = 0.0
-    else:
-      svw = sv/sw
-    return svw
-
-  def _add_generic_history(self, history):
-    periods = r['read_history'].keys()
-    periods.sort(key = lambda x: r['read_history'][x]['interval'] )
-
-    print periods
-
-  def add_running_history(self, history):
-    pass
-
-  def add_uptime(self, uptime):
-    logging.debug('Adding uptime %s.'%(self._fpr,))
-
-    # flags we care about: Running, V2Dir, Guard
-    if not 'flags' in uptime:
-      logging.debug('No flags in document for %s.'%(self._fpr,))
-      return
-
-    for f in ['Running', 'Guard', 'V2Dir']:
-      if not f in uptime['flags']:
-        logging.debug('No %s in flags for %s.'%(f, self._fpr,))
-        return
-
-    running = self._extract_generic_history(uptime['flags']['Running'],
-                                            '%s-Running'%(self._fpr))
-    guard = self._extract_generic_history(uptime['flags']['Guard'],
-                                          '%s-Guard'%(self._fpr))
-    v2dir = self._extract_generic_history(uptime['flags']['V2Dir'],
-                                          '%s-V2Dir'%(self._fpr))
-    if 'BadExit' in uptime['flags']:
-      badexit = self._extract_generic_history(uptime['flags']['BadExit'],
-                                              '%s-BadExit'%(self._fpr))
-
-    self._running = self._avg_generic_history(running) / ONIONOO_SCALE_ONE
-    self._guard = self._avg_generic_history(guard) / ONIONOO_SCALE_ONE
-    self._v2dir = self._avg_generic_history(v2dir) / ONIONOO_SCALE_ONE
-    self._badexit = None
-    if 'BadExit' in uptime['flags']:
-      self._badexit = self._avg_generic_history(badexit) / ONIONOO_SCALE_ONE
-
-  def is_candidate(self):
-    try:
-      if (MUST_BE_RUNNING_NOW and not self.is_running()):
-        log_excluded('%s not a candidate: not running now, unable to check ' +
-                     'DirPort consensus download', self._fpr)
-        return False
-      if (self._data['last_changed_address_or_port'] >
-          self.CUTOFF_ADDRESS_AND_PORT_STABLE):
-        log_excluded('%s not a candidate: changed address/port recently (%s)',
-                     self._fpr, self._data['last_changed_address_or_port'])
-        return False
-      if self._running < CUTOFF_RUNNING:
-        log_excluded('%s not a candidate: running avg too low (%lf)',
-                     self._fpr, self._running)
-        return False
-      if self._v2dir < CUTOFF_V2DIR:
-        log_excluded('%s not a candidate: v2dir avg too low (%lf)',
-                     self._fpr, self._v2dir)
-        return False
-      if self._badexit is not None and self._badexit > PERMITTED_BADEXIT:
-        log_excluded('%s not a candidate: badexit avg too high (%lf)',
-                     self._fpr, self._badexit)
-        return False
-      # this function logs a message depending on which check fails
-      if not self.is_valid_version():
-        return False
-      if self._guard < CUTOFF_GUARD:
-        log_excluded('%s not a candidate: guard avg too low (%lf)',
-                     self._fpr, self._guard)
-        return False
-      if (not self._data.has_key('consensus_weight')
-          or self._data['consensus_weight'] < 1):
-        log_excluded('%s not a candidate: consensus weight invalid', self._fpr)
-        return False
-    except BaseException as e:
-      logging.warning("Exception %s when checking if fallback is a candidate",
-                      str(e))
-      return False
-    return True
-
-  def is_in_whitelist(self, relaylist):
-    """ A fallback matches if each key in the whitelist line matches:
-          ipv4
-          dirport
-          orport
-          id
-          ipv6 address and port (if present)
-        If the fallback has an ipv6 key, the whitelist line must also have
-        it, and vice versa, otherwise they don't match. """
-    ipv6 = None
-    if self.has_ipv6():
-      ipv6 = '%s:%d'%(self.ipv6addr, self.ipv6orport)
-    for entry in relaylist:
-      if entry['id'] != self._fpr:
-        # can't log here unless we match an IP and port, because every relay's
-        # fingerprint is compared to every entry's fingerprint
-        if entry['ipv4'] == self.dirip and int(entry['orport']) == self.orport:
-          logging.warning('%s excluded: has OR %s:%d changed fingerprint to ' +
-                          '%s?', entry['id'], self.dirip, self.orport,
-                          self._fpr)
-        if self.has_ipv6() and entry.has_key('ipv6') and entry['ipv6'] == ipv6:
-          logging.warning('%s excluded: has OR %s changed fingerprint to ' +
-                          '%s?', entry['id'], ipv6, self._fpr)
-        continue
-      if entry['ipv4'] != self.dirip:
-        logging.warning('%s excluded: has it changed IPv4 from %s to %s?',
-                        self._fpr, entry['ipv4'], self.dirip)
-        continue
-      if int(entry['dirport']) != self.dirport:
-        logging.warning('%s excluded: has it changed DirPort from %s:%d to ' +
-                        '%s:%d?', self._fpr, self.dirip, int(entry['dirport']),
-                        self.dirip, self.dirport)
-        continue
-      if int(entry['orport']) != self.orport:
-        logging.warning('%s excluded: has it changed ORPort from %s:%d to ' +
-                        '%s:%d?', self._fpr, self.dirip, int(entry['orport']),
-                        self.dirip, self.orport)
-        continue
-      if entry.has_key('ipv6') and self.has_ipv6():
-        # if both entry and fallback have an ipv6 address, compare them
-        if entry['ipv6'] != ipv6:
-          logging.warning('%s excluded: has it changed IPv6 ORPort from %s ' +
-                          'to %s?', self._fpr, entry['ipv6'], ipv6)
-          continue
-      # if the fallback has an IPv6 address but the whitelist entry
-      # doesn't, or vice versa, the whitelist entry doesn't match
-      elif entry.has_key('ipv6') and not self.has_ipv6():
-        logging.warning('%s excluded: has it lost its former IPv6 address %s?',
-                        self._fpr, entry['ipv6'])
-        continue
-      elif not entry.has_key('ipv6') and self.has_ipv6():
-        logging.warning('%s excluded: has it gained an IPv6 address %s?',
-                        self._fpr, ipv6)
-        continue
-      return True
-    return False
-
-  def cw_to_bw_factor(self):
-    # any relays with a missing or zero consensus weight are not candidates
-    # any relays with a missing advertised bandwidth have it set to zero
-    return self._data['advertised_bandwidth'] / self._data['consensus_weight']
-
-  # since advertised_bandwidth is reported by the relay, it can be gamed
-  # to avoid this, use the median consensus weight to bandwidth factor to
-  # estimate this relay's measured bandwidth, and make that the upper limit
-  def measured_bandwidth(self, median_cw_to_bw_factor):
-    cw_to_bw= median_cw_to_bw_factor
-    # Reduce exit bandwidth to make sure we're not overloading them
-    if self.is_exit():
-      cw_to_bw *= EXIT_BANDWIDTH_FRACTION
-    measured_bandwidth = self._data['consensus_weight'] * cw_to_bw
-    if self._data['advertised_bandwidth'] != 0:
-      # limit advertised bandwidth (if available) to measured bandwidth
-      return min(measured_bandwidth, self._data['advertised_bandwidth'])
-    else:
-      return measured_bandwidth
-
-  def set_measured_bandwidth(self, median_cw_to_bw_factor):
-    self._data['measured_bandwidth'] = self.measured_bandwidth(
-                                                      median_cw_to_bw_factor)
-
-  def is_exit(self):
-    return 'Exit' in self._data['flags']
-
-  def is_guard(self):
-    return 'Guard' in self._data['flags']
-
-  def is_running(self):
-    return 'Running' in self._data['flags']
-
-  # does this fallback have an IPv6 address and orport?
-  def has_ipv6(self):
-    return self.ipv6addr is not None and self.ipv6orport is not None
-
-  # strip leading and trailing brackets from an IPv6 address
-  # safe to use on non-bracketed IPv6 and on IPv4 addresses
-  # also convert to unicode, and make None appear as ''
-  @staticmethod
-  def strip_ipv6_brackets(ip):
-    if ip is None:
-      return unicode('')
-    if len(ip) < 2:
-      return unicode(ip)
-    if ip[0] == '[' and ip[-1] == ']':
-      return unicode(ip[1:-1])
-    return unicode(ip)
-
-  # are ip_a and ip_b in the same netblock?
-  # mask_bits is the size of the netblock
-  # takes both IPv4 and IPv6 addresses
-  # the versions of ip_a and ip_b must be the same
-  # the mask must be valid for the IP version
-  @staticmethod
-  def netblocks_equal(ip_a, ip_b, mask_bits):
-    if ip_a is None or ip_b is None:
-      return False
-    ip_a = Candidate.strip_ipv6_brackets(ip_a)
-    ip_b = Candidate.strip_ipv6_brackets(ip_b)
-    a = ipaddress.ip_address(ip_a)
-    b = ipaddress.ip_address(ip_b)
-    if a.version != b.version:
-      raise Exception('Mismatching IP versions in %s and %s'%(ip_a, ip_b))
-    if mask_bits > a.max_prefixlen:
-      logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
-      mask_bits = a.max_prefixlen
-    if mask_bits < 0:
-      logging.error('Bad IP mask %d for %s and %s'%(mask_bits, ip_a, ip_b))
-      mask_bits = 0
-    a_net = ipaddress.ip_network('%s/%d'%(ip_a, mask_bits), strict=False)
-    return b in a_net
-
-  # is this fallback's IPv4 address (dirip) in the same netblock as other's
-  # IPv4 address?
-  # mask_bits is the size of the netblock
-  def ipv4_netblocks_equal(self, other, mask_bits):
-    return Candidate.netblocks_equal(self.dirip, other.dirip, mask_bits)
-
-  # is this fallback's IPv6 address (ipv6addr) in the same netblock as
-  # other's IPv6 address?
-  # Returns False if either fallback has no IPv6 address
-  # mask_bits is the size of the netblock
-  def ipv6_netblocks_equal(self, other, mask_bits):
-    if not self.has_ipv6() or not other.has_ipv6():
-      return False
-    return Candidate.netblocks_equal(self.ipv6addr, other.ipv6addr, mask_bits)
-
-  # is this fallback's IPv4 DirPort the same as other's IPv4 DirPort?
-  def dirport_equal(self, other):
-    return self.dirport == other.dirport
-
-  # is this fallback's IPv4 ORPort the same as other's IPv4 ORPort?
-  def ipv4_orport_equal(self, other):
-    return self.orport == other.orport
-
-  # is this fallback's IPv6 ORPort the same as other's IPv6 ORPort?
-  # Returns False if either fallback has no IPv6 address
-  def ipv6_orport_equal(self, other):
-    if not self.has_ipv6() or not other.has_ipv6():
-      return False
-    return self.ipv6orport == other.ipv6orport
-
-  # does this fallback have the same DirPort, IPv4 ORPort, or
-  # IPv6 ORPort as other?
-  # Ignores IPv6 ORPort if either fallback has no IPv6 address
-  def port_equal(self, other):
-    return (self.dirport_equal(other) or self.ipv4_orport_equal(other)
-            or self.ipv6_orport_equal(other))
-
-  # return a list containing IPv4 ORPort, DirPort, and IPv6 ORPort (if present)
-  def port_list(self):
-    ports = [self.dirport, self.orport]
-    if self.has_ipv6() and not self.ipv6orport in ports:
-      ports.append(self.ipv6orport)
-    return ports
-
-  # does this fallback share a port with other, regardless of whether the
-  # port types match?
-  # For example, if self's IPv4 ORPort is 80 and other's DirPort is 80,
-  # return True
-  def port_shared(self, other):
-    for p in self.port_list():
-      if p in other.port_list():
-        return True
-    return False
-
-  # log how long it takes to download a consensus from dirip:dirport
-  # returns True if the download failed, False if it succeeded within max_time
-  @staticmethod
-  def fallback_consensus_download_speed(dirip, dirport, nickname, fingerprint,
-                                        max_time):
-    download_failed = False
-    # some directory mirrors respond to requests in ways that hang python
-    # sockets, which is why we log this line here
-    logging.info('Initiating %sconsensus download from %s (%s:%d) %s.',
-                 'microdesc ' if DOWNLOAD_MICRODESC_CONSENSUS else '',
-                 nickname, dirip, dirport, fingerprint)
-    # there appears to be about 1 second of overhead when comparing stem's
-    # internal trace time and the elapsed time calculated here
-    TIMEOUT_SLOP = 1.0
-    start = datetime.datetime.utcnow()
-    try:
-      consensus = get_consensus(
-                              endpoints = [(dirip, dirport)],
-                              timeout = (max_time + TIMEOUT_SLOP),
-                              validate = True,
-                              retries = 0,
-                              fall_back_to_authority = False,
-                              document_handler = DocumentHandler.BARE_DOCUMENT,
-                              microdescriptor = DOWNLOAD_MICRODESC_CONSENSUS
-                                ).run()[0]
-      end = datetime.datetime.utcnow()
-      time_since_expiry = (end - consensus.valid_until).total_seconds()
-    except Exception, stem_error:
-      end = datetime.datetime.utcnow()
-      log_excluded('Unable to retrieve a consensus from %s: %s', nickname,
-                    stem_error)
-      status = 'error: "%s"' % (stem_error)
-      level = logging.WARNING
-      download_failed = True
-    elapsed = (end - start).total_seconds()
-    if download_failed:
-      # keep the error failure status, and avoid using the variables
-      pass
-    elif elapsed > max_time:
-      status = 'too slow'
-      level = logging.WARNING
-      download_failed = True
-    elif (time_since_expiry > 0):
-      status = 'outdated consensus, expired %ds ago'%(int(time_since_expiry))
-      if time_since_expiry <= CONSENSUS_EXPIRY_TOLERANCE:
-        status += ', tolerating up to %ds'%(CONSENSUS_EXPIRY_TOLERANCE)
-        level = logging.INFO
-      else:
-        status += ', invalid'
-        level = logging.WARNING
-        download_failed = True
-    else:
-      status = 'ok'
-      level = logging.DEBUG
-    logging.log(level, 'Consensus download: %0.1fs %s from %s (%s:%d) %s, ' +
-                 'max download time %0.1fs.', elapsed, status, nickname,
-                 dirip, dirport, fingerprint, max_time)
-    return download_failed
-
-  # does this fallback download the consensus fast enough?
-  def check_fallback_download_consensus(self):
-    # include the relay if we're not doing a check, or we can't check (IPv6)
-    ipv4_failed = False
-    ipv6_failed = False
-    if PERFORM_IPV4_DIRPORT_CHECKS:
-      ipv4_failed = Candidate.fallback_consensus_download_speed(self.dirip,
-                                                self.dirport,
-                                                self._data['nickname'],
-                                                self._fpr,
-                                                CONSENSUS_DOWNLOAD_SPEED_MAX)
-    if self.has_ipv6() and PERFORM_IPV6_DIRPORT_CHECKS:
-      # Clients assume the IPv6 DirPort is the same as the IPv4 DirPort
-      ipv6_failed = Candidate.fallback_consensus_download_speed(self.ipv6addr,
-                                                self.dirport,
-                                                self._data['nickname'],
-                                                self._fpr,
-                                                CONSENSUS_DOWNLOAD_SPEED_MAX)
-    return ((not ipv4_failed) and (not ipv6_failed))
-
-  # if this fallback has not passed a download check, try it again,
-  # and record the result, available in get_fallback_download_consensus
-  def try_fallback_download_consensus(self):
-    if not self.get_fallback_download_consensus():
-      self._data['download_check'] = self.check_fallback_download_consensus()
-
-  # did this fallback pass the download check?
-  def get_fallback_download_consensus(self):
-    # if we're not performing checks, return True
-    if not PERFORM_IPV4_DIRPORT_CHECKS and not PERFORM_IPV6_DIRPORT_CHECKS:
-      return True
-    # if we are performing checks, but haven't done one, return False
-    if not self._data.has_key('download_check'):
-      return False
-    return self._data['download_check']
-
-  # output an optional header comment and info for this fallback
-  # try_fallback_download_consensus before calling this
-  def fallbackdir_line(self, fallbacks, prefilter_fallbacks):
-    s = ''
-    if OUTPUT_COMMENTS:
-      s += self.fallbackdir_comment(fallbacks, prefilter_fallbacks)
-    # if the download speed is ok, output a C string
-    # if it's not, but we OUTPUT_COMMENTS, output a commented-out C string
-    if self.get_fallback_download_consensus() or OUTPUT_COMMENTS:
-      s += self.fallbackdir_info(self.get_fallback_download_consensus())
-    return s
-
-  # output a header comment for this fallback
-  def fallbackdir_comment(self, fallbacks, prefilter_fallbacks):
-    # /*
-    # nickname
-    # flags
-    # adjusted bandwidth, consensus weight
-    # [contact]
-    # [identical contact counts]
-    # */
-    # Multiline C comment
-    s = '/*'
-    s += '\n'
-    s += cleanse_c_multiline_comment(self._data['nickname'])
-    s += '\n'
-    s += 'Flags: '
-    s += cleanse_c_multiline_comment(' '.join(sorted(self._data['flags'])))
-    s += '\n'
-    # this is an adjusted bandwidth, see calculate_measured_bandwidth()
-    bandwidth = self._data['measured_bandwidth']
-    weight = self._data['consensus_weight']
-    s += 'Bandwidth: %.1f MByte/s, Consensus Weight: %d'%(
-        bandwidth/(1024.0*1024.0),
-        weight)
-    s += '\n'
-    if self._data['contact'] is not None:
-      s += cleanse_c_multiline_comment(self._data['contact'])
-      if CONTACT_COUNT:
-        fallback_count = len([f for f in fallbacks
-                              if f._data['contact'] == self._data['contact']])
-        if fallback_count > 1:
-          s += '\n'
-          s += '%d identical contacts listed' % (fallback_count)
-
-  # output the fallback info C string for this fallback
-  # this is the text that would go after FallbackDir in a torrc
-  # if this relay failed the download test and we OUTPUT_COMMENTS,
-  # comment-out the returned string
-  def fallbackdir_info(self, dl_speed_ok):
-    # "address:dirport orport=port id=fingerprint"
-    # (insert additional madatory fields here)
-    # "[ipv6=addr:orport]"
-    # (insert additional optional fields here)
-    # /* nickname=name */
-    # /* extrainfo={0,1} */
-    # (insert additional comment fields here)
-    # /* ===== */
-    # ,
-    #
-    # Do we want a C string, or a commented-out string?
-    c_string = dl_speed_ok
-    comment_string = not dl_speed_ok and OUTPUT_COMMENTS
-    # If we don't want either kind of string, bail
-    if not c_string and not comment_string:
-      return ''
-    s = ''
-    # Comment out the fallback directory entry if it's too slow
-    # See the debug output for which address and port is failing
-    if comment_string:
-      s += '/* Consensus download failed or was too slow:\n'
-    # Multi-Line C string with trailing comma (part of a string list)
-    # This makes it easier to diff the file, and remove IPv6 lines using grep
-    # Integers don't need escaping
-    s += '"%s orport=%d id=%s"'%(
-            cleanse_c_string(self._data['dir_address']),
-            self.orport,
-            cleanse_c_string(self._fpr))
-    s += '\n'
-    # (insert additional madatory fields here)
-    if self.has_ipv6():
-      s += '" ipv6=%s:%d"'%(cleanse_c_string(self.ipv6addr), self.ipv6orport)
-      s += '\n'
-    # (insert additional optional fields here)
-    if not comment_string:
-      s += '/* '
-    s += 'nickname=%s'%(cleanse_c_string(self._data['nickname']))
-    if not comment_string:
-      s += ' */'
-    s += '\n'
-    # if we know that the fallback is an extrainfo cache, flag it
-    # and if we don't know, assume it is not
-    if not comment_string:
-      s += '/* '
-    s += 'extrainfo=%d'%(1 if self._extra_info_cache else 0)
-    if not comment_string:
-      s += ' */'
-    s += '\n'
-    # (insert additional comment fields here)
-    # The terminator and comma must be the last line in each fallback entry
-    if not comment_string:
-      s += '/* '
-    s += SECTION_SEPARATOR_BASE
-    if not comment_string:
-      s += ' */'
-    s += '\n'
-    s += ','
-    if comment_string:
-      s += '\n'
-      s += '*/'
-    return s
-
-## Fallback Candidate List Class
-
-class CandidateList(dict):
-  def __init__(self):
-    pass
-
-  def _add_relay(self, details):
-    if not 'dir_address' in details: return
-    c = Candidate(details)
-    self[ c.get_fingerprint() ] = c
-
-  def _add_uptime(self, uptime):
-    try:
-      fpr = uptime['fingerprint']
-    except KeyError:
-      raise Exception("Document has no fingerprint field.")
-
-    try:
-      c = self[fpr]
-    except KeyError:
-      logging.debug('Got unknown relay %s in uptime document.'%(fpr,))
-      return
-
-    c.add_uptime(uptime)
-
-  def _add_details(self):
-    logging.debug('Loading details document.')
-    d = fetch('details',
-        fields=('fingerprint,nickname,contact,last_changed_address_or_port,' +
-                'consensus_weight,advertised_bandwidth,or_addresses,' +
-                'dir_address,recommended_version,flags,effective_family,' +
-                'platform'))
-    logging.debug('Loading details document done.')
-
-    if not 'relays' in d: raise Exception("No relays found in document.")
-
-    for r in d['relays']: self._add_relay(r)
-
-  def _add_uptimes(self):
-    logging.debug('Loading uptime document.')
-    d = fetch('uptime')
-    logging.debug('Loading uptime document done.')
-
-    if not 'relays' in d: raise Exception("No relays found in document.")
-    for r in d['relays']: self._add_uptime(r)
-
-  def add_relays(self):
-    self._add_details()
-    self._add_uptimes()
-
-  def count_guards(self):
-    guard_count = 0
-    for fpr in self.keys():
-      if self[fpr].is_guard():
-        guard_count += 1
-    return guard_count
-
-  # Find fallbacks that fit the uptime, stability, and flags criteria,
-  # and make an array of them in self.fallbacks
-  def compute_fallbacks(self):
-    self.fallbacks = map(lambda x: self[x],
-                         filter(lambda x: self[x].is_candidate(),
-                                self.keys()))
-
-  # sort fallbacks by their consensus weight to advertised bandwidth factor,
-  # lowest to highest
-  # used to find the median cw_to_bw_factor()
-  def sort_fallbacks_by_cw_to_bw_factor(self):
-    self.fallbacks.sort(key=lambda f: f.cw_to_bw_factor())
-
-  # sort fallbacks by their measured bandwidth, highest to lowest
-  # calculate_measured_bandwidth before calling this
-  # this is useful for reviewing candidates in priority order
-  def sort_fallbacks_by_measured_bandwidth(self):
-    self.fallbacks.sort(key=lambda f: f._data['measured_bandwidth'],
-                        reverse=True)
-
-  # sort fallbacks by the data field data_field, lowest to highest
-  def sort_fallbacks_by(self, data_field):
-    self.fallbacks.sort(key=lambda f: f._data[data_field])
-
-  @staticmethod
-  def load_relaylist(file_obj):
-    """ Read each line in the file, and parse it like a FallbackDir line:
-        an IPv4 address and optional port:
-          <IPv4 address>:<port>
-        which are parsed into dictionary entries:
-          ipv4=<IPv4 address>
-          dirport=<port>
-        followed by a series of key=value entries:
-          orport=<port>
-          id=<fingerprint>
-          ipv6=<IPv6 address>:<IPv6 orport>
-        each line's key/value pairs are placed in a dictonary,
-        (of string -> string key/value pairs),
-        and these dictionaries are placed in an array.
-        comments start with # and are ignored """
-    file_data = file_obj['data']
-    file_name = file_obj['name']
-    relaylist = []
-    if file_data is None:
-      return relaylist
-    for line in file_data.split('\n'):
-      relay_entry = {}
-      # ignore comments
-      line_comment_split = line.split('#')
-      line = line_comment_split[0]
-      # cleanup whitespace
-      line = cleanse_whitespace(line)
-      line = line.strip()
-      if len(line) == 0:
-        continue
-      for item in line.split(' '):
-        item = item.strip()
-        if len(item) == 0:
-          continue
-        key_value_split = item.split('=')
-        kvl = len(key_value_split)
-        if kvl < 1 or kvl > 2:
-          print '#error Bad %s item: %s, format is key=value.'%(
-                                                 file_name, item)
-        if kvl == 1:
-          # assume that entries without a key are the ipv4 address,
-          # perhaps with a dirport
-          ipv4_maybe_dirport = key_value_split[0]
-          ipv4_maybe_dirport_split = ipv4_maybe_dirport.split(':')
-          dirl = len(ipv4_maybe_dirport_split)
-          if dirl < 1 or dirl > 2:
-            print '#error Bad %s IPv4 item: %s, format is ipv4:port.'%(
-                                                        file_name, item)
-          if dirl >= 1:
-            relay_entry['ipv4'] = ipv4_maybe_dirport_split[0]
-          if dirl == 2:
-            relay_entry['dirport'] = ipv4_maybe_dirport_split[1]
-        elif kvl == 2:
-          relay_entry[key_value_split[0]] = key_value_split[1]
-      relaylist.append(relay_entry)
-    return relaylist
-
-  # apply the fallback whitelist
-  def apply_filter_lists(self, whitelist_obj):
-    excluded_count = 0
-    logging.debug('Applying whitelist')
-    # parse the whitelist
-    whitelist = self.load_relaylist(whitelist_obj)
-    filtered_fallbacks = []
-    for f in self.fallbacks:
-      in_whitelist = f.is_in_whitelist(whitelist)
-      if in_whitelist:
-        # include
-        filtered_fallbacks.append(f)
-      elif INCLUDE_UNLISTED_ENTRIES:
-          # include
-          filtered_fallbacks.append(f)
-      else:
-          # exclude
-          excluded_count += 1
-          log_excluded('Excluding %s: not in whitelist.',
-                       f._fpr)
-    self.fallbacks = filtered_fallbacks
-    return excluded_count
-
-  @staticmethod
-  def summarise_filters(initial_count, excluded_count):
-    return '/* Whitelist excluded %d of %d candidates. */'%(
-                                                excluded_count, initial_count)
-
-  # calculate each fallback's measured bandwidth based on the median
-  # consensus weight to advertised bandwidth ratio
-  def calculate_measured_bandwidth(self):
-    self.sort_fallbacks_by_cw_to_bw_factor()
-    median_fallback = self.fallback_median(True)
-    if median_fallback is not None:
-      median_cw_to_bw_factor = median_fallback.cw_to_bw_factor()
-    else:
-      # this will never be used, because there are no fallbacks
-      median_cw_to_bw_factor = None
-    for f in self.fallbacks:
-      f.set_measured_bandwidth(median_cw_to_bw_factor)
-
-  # remove relays with low measured bandwidth from the fallback list
-  # calculate_measured_bandwidth for each relay before calling this
-  def remove_low_bandwidth_relays(self):
-    if MIN_BANDWIDTH is None:
-      return
-    above_min_bw_fallbacks = []
-    for f in self.fallbacks:
-      if f._data['measured_bandwidth'] >= MIN_BANDWIDTH:
-        above_min_bw_fallbacks.append(f)
-      else:
-        # the bandwidth we log here is limited by the relay's consensus weight
-        # as well as its adverttised bandwidth. See set_measured_bandwidth
-        # for details
-        log_excluded('%s not a candidate: bandwidth %.1fMByte/s too low, ' +
-                     'must be at least %.1fMByte/s', f._fpr,
-                     f._data['measured_bandwidth']/(1024.0*1024.0),
-                     MIN_BANDWIDTH/(1024.0*1024.0))
-    self.fallbacks = above_min_bw_fallbacks
-
-  # the minimum fallback in the list
-  # call one of the sort_fallbacks_* functions before calling this
-  def fallback_min(self):
-    if len(self.fallbacks) > 0:
-      return self.fallbacks[-1]
-    else:
-      return None
-
-  # the median fallback in the list
-  # call one of the sort_fallbacks_* functions before calling this
-  def fallback_median(self, require_advertised_bandwidth):
-    # use the low-median when there are an evan number of fallbacks,
-    # for consistency with the bandwidth authorities
-    if len(self.fallbacks) > 0:
-      median_position = (len(self.fallbacks) - 1) / 2
-      if not require_advertised_bandwidth:
-        return self.fallbacks[median_position]
-      # if we need advertised_bandwidth but this relay doesn't have it,
-      # move to a fallback with greater consensus weight until we find one
-      while not self.fallbacks[median_position]._data['advertised_bandwidth']:
-        median_position += 1
-        if median_position >= len(self.fallbacks):
-          return None
-      return self.fallbacks[median_position]
-    else:
-      return None
-
-  # the maximum fallback in the list
-  # call one of the sort_fallbacks_* functions before calling this
-  def fallback_max(self):
-    if len(self.fallbacks) > 0:
-      return self.fallbacks[0]
-    else:
-      return None
-
-  # return a new bag suitable for storing attributes
-  @staticmethod
-  def attribute_new():
-    return dict()
-
-  # get the count of attribute in attribute_bag
-  # if attribute is None or the empty string, return 0
-  @staticmethod
-  def attribute_count(attribute, attribute_bag):
-    if attribute is None or attribute == '':
-      return 0
-    if attribute not in attribute_bag:
-      return 0
-    return attribute_bag[attribute]
-
-  # does attribute_bag contain more than max_count instances of attribute?
-  # if so, return False
-  # if not, return True
-  # if attribute is None or the empty string, or max_count is invalid,
-  # always return True
-  @staticmethod
-  def attribute_allow(attribute, attribute_bag, max_count=1):
-    if attribute is None or attribute == '' or max_count <= 0:
-      return True
-    elif CandidateList.attribute_count(attribute, attribute_bag) >= max_count:
-      return False
-    else:
-      return True
-
-  # add attribute to attribute_bag, incrementing the count if it is already
-  # present
-  # if attribute is None or the empty string, or count is invalid,
-  # do nothing
-  @staticmethod
-  def attribute_add(attribute, attribute_bag, count=1):
-    if attribute is None or attribute == '' or count <= 0:
-      pass
-    attribute_bag.setdefault(attribute, 0)
-    attribute_bag[attribute] += count
-
-  # make sure there are only MAX_FALLBACKS_PER_IP fallbacks per IPv4 address,
-  # and per IPv6 address
-  # there is only one IPv4 address on each fallback: the IPv4 DirPort address
-  # (we choose the IPv4 ORPort which is on the same IPv4 as the DirPort)
-  # there is at most one IPv6 address on each fallback: the IPv6 ORPort address
-  # we try to match the IPv4 ORPort, but will use any IPv6 address if needed
-  # (clients only use the IPv6 ORPort)
-  # if there is no IPv6 address, only the IPv4 address is checked
-  # return the number of candidates we excluded
-  def limit_fallbacks_same_ip(self):
-    ip_limit_fallbacks = []
-    ip_list = CandidateList.attribute_new()
-    for f in self.fallbacks:
-      if (CandidateList.attribute_allow(f.dirip, ip_list,
-                                        MAX_FALLBACKS_PER_IPV4)
-          and CandidateList.attribute_allow(f.ipv6addr, ip_list,
-                                            MAX_FALLBACKS_PER_IPV6)):
-        ip_limit_fallbacks.append(f)
-        CandidateList.attribute_add(f.dirip, ip_list)
-        if f.has_ipv6():
-          CandidateList.attribute_add(f.ipv6addr, ip_list)
-      elif not CandidateList.attribute_allow(f.dirip, ip_list,
-                                             MAX_FALLBACKS_PER_IPV4):
-        log_excluded('Eliminated %s: already have %d fallback(s) on IPv4 %s'
-                     %(f._fpr, CandidateList.attribute_count(f.dirip, ip_list),
-                       f.dirip))
-      elif (f.has_ipv6() and
-            not CandidateList.attribute_allow(f.ipv6addr, ip_list,
-                                              MAX_FALLBACKS_PER_IPV6)):
-        log_excluded('Eliminated %s: already have %d fallback(s) on IPv6 %s'
-                     %(f._fpr, CandidateList.attribute_count(f.ipv6addr,
-                                                             ip_list),
-                       f.ipv6addr))
-    original_count = len(self.fallbacks)
-    self.fallbacks = ip_limit_fallbacks
-    return original_count - len(self.fallbacks)
-
-  # make sure there are only MAX_FALLBACKS_PER_CONTACT fallbacks for each
-  # ContactInfo
-  # if there is no ContactInfo, allow the fallback
-  # this check can be gamed by providing no ContactInfo, or by setting the
-  # ContactInfo to match another fallback
-  # However, given the likelihood that relays with the same ContactInfo will
-  # go down at similar times, its usefulness outweighs the risk
-  def limit_fallbacks_same_contact(self):
-    contact_limit_fallbacks = []
-    contact_list = CandidateList.attribute_new()
-    for f in self.fallbacks:
-      if CandidateList.attribute_allow(f._data['contact'], contact_list,
-                                       MAX_FALLBACKS_PER_CONTACT):
-        contact_limit_fallbacks.append(f)
-        CandidateList.attribute_add(f._data['contact'], contact_list)
-      else:
-        log_excluded(
-          'Eliminated %s: already have %d fallback(s) on ContactInfo %s'
-          %(f._fpr, CandidateList.attribute_count(f._data['contact'],
-                                                  contact_list),
-            f._data['contact']))
-    original_count = len(self.fallbacks)
-    self.fallbacks = contact_limit_fallbacks
-    return original_count - len(self.fallbacks)
-
-  # make sure there are only MAX_FALLBACKS_PER_FAMILY fallbacks per effective
-  # family
-  # if there is no family, allow the fallback
-  # we use effective family, which ensures mutual family declarations
-  # but the check can be gamed by not declaring a family at all
-  # if any indirect families exist, the result depends on the order in which
-  # fallbacks are sorted in the list
-  def limit_fallbacks_same_family(self):
-    family_limit_fallbacks = []
-    fingerprint_list = CandidateList.attribute_new()
-    for f in self.fallbacks:
-      if CandidateList.attribute_allow(f._fpr, fingerprint_list,
-                                       MAX_FALLBACKS_PER_FAMILY):
-        family_limit_fallbacks.append(f)
-        CandidateList.attribute_add(f._fpr, fingerprint_list)
-        for family_fingerprint in f._data['effective_family']:
-          CandidateList.attribute_add(family_fingerprint, fingerprint_list)
-      else:
-        # we already have a fallback with this fallback in its effective
-        # family
-        log_excluded(
-          'Eliminated %s: already have %d fallback(s) in effective family'
-          %(f._fpr, CandidateList.attribute_count(f._fpr, fingerprint_list)))
-    original_count = len(self.fallbacks)
-    self.fallbacks = family_limit_fallbacks
-    return original_count - len(self.fallbacks)
-
-  # try once to get the descriptors for fingerprint_list using stem
-  # returns an empty list on exception
-  @staticmethod
-  def get_fallback_descriptors_once(fingerprint_list):
-    desc_list = get_server_descriptors(fingerprints=fingerprint_list).run(suppress=True)
-    return desc_list
-
-  # try up to max_retries times to get the descriptors for fingerprint_list
-  # using stem. Stops retrying when all descriptors have been retrieved.
-  # returns a list containing the descriptors that were retrieved
-  @staticmethod
-  def get_fallback_descriptors(fingerprint_list, max_retries=5):
-    # we can't use stem's retries=, because we want to support more than 96
-    # descriptors
-    #
-    # add an attempt for every MAX_FINGERPRINTS (or part thereof) in the list
-    max_retries += (len(fingerprint_list) + MAX_FINGERPRINTS - 1) / MAX_FINGERPRINTS
-    remaining_list = fingerprint_list
-    desc_list = []
-    for _ in xrange(max_retries):
-      if len(remaining_list) == 0:
-        break
-      new_desc_list = CandidateList.get_fallback_descriptors_once(remaining_list[0:MAX_FINGERPRINTS])
-      for d in new_desc_list:
-        try:
-          remaining_list.remove(d.fingerprint)
-        except ValueError:
-          # warn and ignore if a directory mirror returned a bad descriptor
-          logging.warning("Directory mirror returned unwanted descriptor %s, ignoring",
-                          d.fingerprint)
-          continue
-        desc_list.append(d)
-    return desc_list
-
-  # find the fallbacks that cache extra-info documents
-  # Onionoo doesn't know this, so we have to use stem
-  def mark_extra_info_caches(self):
-    fingerprint_list = [ f._fpr for f in self.fallbacks ]
-    logging.info("Downloading fallback descriptors to find extra-info caches")
-    desc_list = CandidateList.get_fallback_descriptors(fingerprint_list)
-    for d in desc_list:
-      self[d.fingerprint]._extra_info_cache = d.extra_info_cache
-    missing_descriptor_list = [ f._fpr for f in self.fallbacks
-                                if f._extra_info_cache is None ]
-    for f in missing_descriptor_list:
-      logging.warning("No descriptor for {}. Assuming extrainfo=0.".format(f))
-
-  # try a download check on each fallback candidate in order
-  # stop after max_count successful downloads
-  # but don't remove any candidates from the array
-  def try_download_consensus_checks(self, max_count):
-    dl_ok_count = 0
-    for f in self.fallbacks:
-      f.try_fallback_download_consensus()
-      if f.get_fallback_download_consensus():
-        # this fallback downloaded a consensus ok
-        dl_ok_count += 1
-        if dl_ok_count >= max_count:
-          # we have enough fallbacks
-          return
-
-  # put max_count successful candidates in the fallbacks array:
-  # - perform download checks on each fallback candidate
-  # - retry failed candidates if CONSENSUS_DOWNLOAD_RETRY is set
-  # - eliminate failed candidates
-  # - if there are more than max_count candidates, eliminate lowest bandwidth
-  # - if there are fewer than max_count candidates, leave only successful
-  # Return the number of fallbacks that failed the consensus check
-  def perform_download_consensus_checks(self, max_count):
-    self.sort_fallbacks_by_measured_bandwidth()
-    self.try_download_consensus_checks(max_count)
-    if CONSENSUS_DOWNLOAD_RETRY:
-      # try unsuccessful candidates again
-      # we could end up with more than max_count successful candidates here
-      self.try_download_consensus_checks(max_count)
-    # now we have at least max_count successful candidates,
-    # or we've tried them all
-    original_count = len(self.fallbacks)
-    self.fallbacks = filter(lambda x: x.get_fallback_download_consensus(),
-                            self.fallbacks)
-    # some of these failed the check, others skipped the check,
-    # if we already had enough successful downloads
-    failed_count = original_count - len(self.fallbacks)
-    self.fallbacks = self.fallbacks[:max_count]
-    return failed_count
-
-  # return a string that describes a/b as a percentage
-  @staticmethod
-  def describe_percentage(a, b):
-    if b != 0:
-      return '%d/%d = %.0f%%'%(a, b, (a*100.0)/b)
-    else:
-      # technically, 0/0 is undefined, but 0.0% is a sensible result
-      return '%d/%d = %.0f%%'%(a, b, 0.0)
-
-  # return a dictionary of lists of fallbacks by IPv4 netblock
-  # the dictionary is keyed by the fingerprint of an arbitrary fallback
-  # in each netblock
-  # mask_bits is the size of the netblock
-  def fallbacks_by_ipv4_netblock(self, mask_bits):
-    netblocks = {}
-    for f in self.fallbacks:
-      found_netblock = False
-      for b in netblocks.keys():
-        # we found an existing netblock containing this fallback
-        if f.ipv4_netblocks_equal(self[b], mask_bits):
-          # add it to the list
-          netblocks[b].append(f)
-          found_netblock = True
-          break
-      # make a new netblock based on this fallback's fingerprint
-      if not found_netblock:
-        netblocks[f._fpr] = [f]
-    return netblocks
-
-  # return a dictionary of lists of fallbacks by IPv6 netblock
-  # where mask_bits is the size of the netblock
-  def fallbacks_by_ipv6_netblock(self, mask_bits):
-    netblocks = {}
-    for f in self.fallbacks:
-      # skip fallbacks without IPv6 addresses
-      if not f.has_ipv6():
-        continue
-      found_netblock = False
-      for b in netblocks.keys():
-        # we found an existing netblock containing this fallback
-        if f.ipv6_netblocks_equal(self[b], mask_bits):
-          # add it to the list
-          netblocks[b].append(f)
-          found_netblock = True
-          break
-      # make a new netblock based on this fallback's fingerprint
-      if not found_netblock:
-        netblocks[f._fpr] = [f]
-    return netblocks
-
-  # log a message about the proportion of fallbacks in each IPv4 netblock,
-  # where mask_bits is the size of the netblock
-  def describe_fallback_ipv4_netblock_mask(self, mask_bits):
-    fallback_count = len(self.fallbacks)
-    shared_netblock_fallback_count = 0
-    most_frequent_netblock = None
-    netblocks = self.fallbacks_by_ipv4_netblock(mask_bits)
-    for b in netblocks.keys():
-      if len(netblocks[b]) > 1:
-        # how many fallbacks are in a netblock with other fallbacks?
-        shared_netblock_fallback_count += len(netblocks[b])
-        # what's the netblock with the most fallbacks?
-        if (most_frequent_netblock is None
-            or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
-          most_frequent_netblock = b
-        logging.debug('Fallback IPv4 addresses in the same /%d:'%(mask_bits))
-        for f in netblocks[b]:
-          logging.debug('%s - %s', f.dirip, f._fpr)
-    if most_frequent_netblock is not None:
-      logging.warning('There are %s fallbacks in the IPv4 /%d containing %s'%(
-                                    CandidateList.describe_percentage(
-                                      len(netblocks[most_frequent_netblock]),
-                                      fallback_count),
-                                    mask_bits,
-                                    self[most_frequent_netblock].dirip))
-    if shared_netblock_fallback_count > 0:
-      logging.warning(('%s of fallbacks are in an IPv4 /%d with other ' +
-                       'fallbacks')%(CandidateList.describe_percentage(
-                                                shared_netblock_fallback_count,
-                                                fallback_count),
-                                     mask_bits))
-
-  # log a message about the proportion of fallbacks in each IPv6 netblock,
-  # where mask_bits is the size of the netblock
-  def describe_fallback_ipv6_netblock_mask(self, mask_bits):
-    fallback_count = len(self.fallbacks_with_ipv6())
-    shared_netblock_fallback_count = 0
-    most_frequent_netblock = None
-    netblocks = self.fallbacks_by_ipv6_netblock(mask_bits)
-    for b in netblocks.keys():
-      if len(netblocks[b]) > 1:
-        # how many fallbacks are in a netblock with other fallbacks?
-        shared_netblock_fallback_count += len(netblocks[b])
-        # what's the netblock with the most fallbacks?
-        if (most_frequent_netblock is None
-            or len(netblocks[b]) > len(netblocks[most_frequent_netblock])):
-          most_frequent_netblock = b
-        logging.debug('Fallback IPv6 addresses in the same /%d:'%(mask_bits))
-        for f in netblocks[b]:
-          logging.debug('%s - %s', f.ipv6addr, f._fpr)
-    if most_frequent_netblock is not None:
-      logging.warning('There are %s fallbacks in the IPv6 /%d containing %s'%(
-                                    CandidateList.describe_percentage(
-                                      len(netblocks[most_frequent_netblock]),
-                                      fallback_count),
-                                    mask_bits,
-                                    self[most_frequent_netblock].ipv6addr))
-    if shared_netblock_fallback_count > 0:
-      logging.warning(('%s of fallbacks are in an IPv6 /%d with other ' +
-                       'fallbacks')%(CandidateList.describe_percentage(
-                                                shared_netblock_fallback_count,
-                                                fallback_count),
-                                     mask_bits))
-
-  # log a message about the proportion of fallbacks in each IPv4 /8, /16,
-  # and /24
-  def describe_fallback_ipv4_netblocks(self):
-   # this doesn't actually tell us anything useful
-   #self.describe_fallback_ipv4_netblock_mask(8)
-   self.describe_fallback_ipv4_netblock_mask(16)
-   #self.describe_fallback_ipv4_netblock_mask(24)
-
-  # log a message about the proportion of fallbacks in each IPv6 /12 (RIR),
-  # /23 (smaller RIR blocks), /32 (LIR), /48 (Customer), and /64 (Host)
-  # https://www.iana.org/assignments/ipv6-unicast-address-assignments/
-  def describe_fallback_ipv6_netblocks(self):
-    # these don't actually tell us anything useful
-    #self.describe_fallback_ipv6_netblock_mask(12)
-    #self.describe_fallback_ipv6_netblock_mask(23)
-    self.describe_fallback_ipv6_netblock_mask(32)
-    #self.describe_fallback_ipv6_netblock_mask(48)
-    self.describe_fallback_ipv6_netblock_mask(64)
-
-  # log a message about the proportion of fallbacks in each IPv4 and IPv6
-  # netblock
-  def describe_fallback_netblocks(self):
-    self.describe_fallback_ipv4_netblocks()
-    self.describe_fallback_ipv6_netblocks()
-
-  # return a list of fallbacks which are on the IPv4 ORPort port
-  def fallbacks_on_ipv4_orport(self, port):
-    return filter(lambda x: x.orport == port, self.fallbacks)
-
-  # return a list of fallbacks which are on the IPv6 ORPort port
-  def fallbacks_on_ipv6_orport(self, port):
-    return filter(lambda x: x.ipv6orport == port, self.fallbacks_with_ipv6())
-
-  # return a list of fallbacks which are on the DirPort port
-  def fallbacks_on_dirport(self, port):
-    return filter(lambda x: x.dirport == port, self.fallbacks)
-
-  # log a message about the proportion of fallbacks on IPv4 ORPort port
-  # and return that count
-  def describe_fallback_ipv4_orport(self, port):
-    port_count = len(self.fallbacks_on_ipv4_orport(port))
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks are on IPv4 ORPort %d'%(
-                    CandidateList.describe_percentage(port_count,
-                                                      fallback_count),
-                    port))
-    return port_count
-
-  # log a message about the proportion of IPv6 fallbacks on IPv6 ORPort port
-  # and return that count
-  def describe_fallback_ipv6_orport(self, port):
-    port_count = len(self.fallbacks_on_ipv6_orport(port))
-    fallback_count = len(self.fallbacks_with_ipv6())
-    logging.warning('%s of IPv6 fallbacks are on IPv6 ORPort %d'%(
-                    CandidateList.describe_percentage(port_count,
-                                                      fallback_count),
-                    port))
-    return port_count
-
-  # log a message about the proportion of fallbacks on DirPort port
-  # and return that count
-  def describe_fallback_dirport(self, port):
-    port_count = len(self.fallbacks_on_dirport(port))
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks are on DirPort %d'%(
-                    CandidateList.describe_percentage(port_count,
-                                                      fallback_count),
-                    port))
-    return port_count
-
-  # log a message about the proportion of fallbacks on each dirport,
-  # each IPv4 orport, and each IPv6 orport
-  def describe_fallback_ports(self):
-    fallback_count = len(self.fallbacks)
-    ipv4_or_count = fallback_count
-    ipv4_or_count -= self.describe_fallback_ipv4_orport(443)
-    ipv4_or_count -= self.describe_fallback_ipv4_orport(9001)
-    logging.warning('%s of fallbacks are on other IPv4 ORPorts'%(
-                    CandidateList.describe_percentage(ipv4_or_count,
-                                                      fallback_count)))
-    ipv6_fallback_count = len(self.fallbacks_with_ipv6())
-    ipv6_or_count = ipv6_fallback_count
-    ipv6_or_count -= self.describe_fallback_ipv6_orport(443)
-    ipv6_or_count -= self.describe_fallback_ipv6_orport(9001)
-    logging.warning('%s of IPv6 fallbacks are on other IPv6 ORPorts'%(
-                    CandidateList.describe_percentage(ipv6_or_count,
-                                                      ipv6_fallback_count)))
-    dir_count = fallback_count
-    dir_count -= self.describe_fallback_dirport(80)
-    dir_count -= self.describe_fallback_dirport(9030)
-    logging.warning('%s of fallbacks are on other DirPorts'%(
-                    CandidateList.describe_percentage(dir_count,
-                                                      fallback_count)))
-
-  # return a list of fallbacks which cache extra-info documents
-  def fallbacks_with_extra_info_cache(self):
-    return filter(lambda x: x._extra_info_cache, self.fallbacks)
-
-  # log a message about the proportion of fallbacks that cache extra-info docs
-  def describe_fallback_extra_info_caches(self):
-    extra_info_falback_count = len(self.fallbacks_with_extra_info_cache())
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks cache extra-info documents'%(
-                    CandidateList.describe_percentage(extra_info_falback_count,
-                                                      fallback_count)))
-
-  # return a list of fallbacks which have the Exit flag
-  def fallbacks_with_exit(self):
-    return filter(lambda x: x.is_exit(), self.fallbacks)
-
-  # log a message about the proportion of fallbacks with an Exit flag
-  def describe_fallback_exit_flag(self):
-    exit_falback_count = len(self.fallbacks_with_exit())
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks have the Exit flag'%(
-                    CandidateList.describe_percentage(exit_falback_count,
-                                                      fallback_count)))
-
-  # return a list of fallbacks which have an IPv6 address
-  def fallbacks_with_ipv6(self):
-    return filter(lambda x: x.has_ipv6(), self.fallbacks)
-
-  # log a message about the proportion of fallbacks on IPv6
-  def describe_fallback_ip_family(self):
-    ipv6_falback_count = len(self.fallbacks_with_ipv6())
-    fallback_count = len(self.fallbacks)
-    logging.warning('%s of fallbacks are on IPv6'%(
-                    CandidateList.describe_percentage(ipv6_falback_count,
-                                                      fallback_count)))
-
-  def summarise_fallbacks(self, eligible_count, operator_count, failed_count,
-                          guard_count, target_count):
-    s = ''
-    # Report:
-    #  whether we checked consensus download times
-    #  the number of fallback directories (and limits/exclusions, if relevant)
-    #  min & max fallback bandwidths
-    #  #error if below minimum count
-    if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
-      s += '/* Checked %s%s%s DirPorts served a consensus within %.1fs. */'%(
-            'IPv4' if PERFORM_IPV4_DIRPORT_CHECKS else '',
-            ' and ' if (PERFORM_IPV4_DIRPORT_CHECKS
-                        and PERFORM_IPV6_DIRPORT_CHECKS) else '',
-            'IPv6' if PERFORM_IPV6_DIRPORT_CHECKS else '',
-            CONSENSUS_DOWNLOAD_SPEED_MAX)
-    else:
-      s += '/* Did not check IPv4 or IPv6 DirPort consensus downloads. */'
-    s += '\n'
-    # Multiline C comment with #error if things go bad
-    s += '/*'
-    s += '\n'
-    # Integers don't need escaping in C comments
-    fallback_count = len(self.fallbacks)
-    if FALLBACK_PROPORTION_OF_GUARDS is None:
-      fallback_proportion = ''
-    else:
-      fallback_proportion = ', Target %d (%d * %.2f)'%(target_count,
-                                                guard_count,
-                                                FALLBACK_PROPORTION_OF_GUARDS)
-    s += 'Final Count: %d (Eligible %d%s'%(fallback_count, eligible_count,
-                                           fallback_proportion)
-    if MAX_FALLBACK_COUNT is not None:
-      s += ', Max %d'%(MAX_FALLBACK_COUNT)
-    s += ')\n'
-    if eligible_count != fallback_count:
-      removed_count = eligible_count - fallback_count
-      excess_to_target_or_max = (eligible_count - operator_count - failed_count
-                                 - fallback_count)
-      # some 'Failed' failed the check, others 'Skipped' the check,
-      # if we already had enough successful downloads
-      s += ('Excluded: %d (Same Operator %d, Failed/Skipped Download %d, ' +
-            'Excess %d)')%(removed_count, operator_count, failed_count,
-                           excess_to_target_or_max)
-      s += '\n'
-    min_fb = self.fallback_min()
-    min_bw = min_fb._data['measured_bandwidth']
-    max_fb = self.fallback_max()
-    max_bw = max_fb._data['measured_bandwidth']
-    s += 'Bandwidth Range: %.1f - %.1f MByte/s'%(min_bw/(1024.0*1024.0),
-                                                 max_bw/(1024.0*1024.0))
-    s += '\n'
-    s += '*/'
-    if fallback_count < MIN_FALLBACK_COUNT:
-      # We must have a minimum number of fallbacks so they are always
-      # reachable, and are in diverse locations
-      s += '\n'
-      s += '#error Fallback Count %d is too low. '%(fallback_count)
-      s += 'Must be at least %d for diversity. '%(MIN_FALLBACK_COUNT)
-      s += 'Try adding entries to the whitelist, '
-      s += 'or setting INCLUDE_UNLISTED_ENTRIES = True.'
-    return s
-
-def process_existing():
-  logging.basicConfig(level=logging.INFO)
-  logging.getLogger('stem').setLevel(logging.INFO)
-  whitelist = {'data': parse_fallback_file(FALLBACK_FILE_NAME),
-               'name': FALLBACK_FILE_NAME}
-  list_fallbacks(whitelist)
-
-def process_default():
-  logging.basicConfig(level=logging.WARNING)
-  logging.getLogger('stem').setLevel(logging.WARNING)
-  whitelist = {'data': read_from_file(WHITELIST_FILE_NAME, MAX_LIST_FILE_SIZE),
-               'name': WHITELIST_FILE_NAME}
-  list_fallbacks(whitelist)
-
-## Main Function
-def main():
-  if get_command() == 'check_existing':
-    process_existing()
-  else:
-    process_default()
-
-def get_command():
-  if len(sys.argv) == 2:
-    return sys.argv[1]
-  else:
-    return None
-
-def log_excluded(msg, *args):
-  if get_command() == 'check_existing':
-    logging.warning(msg, *args)
-  else:
-    logging.info(msg, *args)
-
-def list_fallbacks(whitelist):
-  """ Fetches required onionoo documents and evaluates the
-      fallback directory criteria for each of the relays """
-
-  print "/* type=fallback */"
-  print ("/* version={} */"
-         .format(cleanse_c_multiline_comment(FALLBACK_FORMAT_VERSION)))
-  now = datetime.datetime.utcnow()
-  timestamp = now.strftime('%Y%m%d%H%M%S')
-  print ("/* timestamp={} */"
-         .format(cleanse_c_multiline_comment(timestamp)))
-  # end the header with a separator, to make it easier for parsers
-  print SECTION_SEPARATOR_COMMENT
-
-  logging.warning('Downloading and parsing Onionoo data. ' +
-                  'This may take some time.')
-  # find relays that could be fallbacks
-  candidates = CandidateList()
-  candidates.add_relays()
-
-  # work out how many fallbacks we want
-  guard_count = candidates.count_guards()
-  if FALLBACK_PROPORTION_OF_GUARDS is None:
-    target_count = guard_count
-  else:
-    target_count = int(guard_count * FALLBACK_PROPORTION_OF_GUARDS)
-  # the maximum number of fallbacks is the least of:
-  # - the target fallback count (FALLBACK_PROPORTION_OF_GUARDS * guard count)
-  # - the maximum fallback count (MAX_FALLBACK_COUNT)
-  if MAX_FALLBACK_COUNT is None:
-    max_count = target_count
-  else:
-    max_count = min(target_count, MAX_FALLBACK_COUNT)
-
-  candidates.compute_fallbacks()
-  prefilter_fallbacks = copy.copy(candidates.fallbacks)
-
-  # filter with the whitelist
-  # if a relay has changed IPv4 address or ports recently, it will be excluded
-  # as ineligible before we call apply_filter_lists, and so there will be no
-  # warning that the details have changed from those in the whitelist.
-  # instead, there will be an info-level log during the eligibility check.
-  initial_count = len(candidates.fallbacks)
-  excluded_count = candidates.apply_filter_lists(whitelist)
-  print candidates.summarise_filters(initial_count, excluded_count)
-  eligible_count = len(candidates.fallbacks)
-
-  # calculate the measured bandwidth of each relay,
-  # then remove low-bandwidth relays
-  candidates.calculate_measured_bandwidth()
-  candidates.remove_low_bandwidth_relays()
-
-  # print the raw fallback list
-  #for x in candidates.fallbacks:
-  #  print x.fallbackdir_line(True)
-  #  print json.dumps(candidates[x]._data, sort_keys=True, indent=4,
-  #                   separators=(',', ': '), default=json_util.default)
-
-  # impose mandatory conditions here, like one per contact, family, IP
-  # in measured bandwidth order
-  candidates.sort_fallbacks_by_measured_bandwidth()
-  operator_count = 0
-  # only impose these limits on the final list - operators can nominate
-  # multiple candidate fallbacks, and then we choose the best set
-  if not OUTPUT_CANDIDATES:
-    operator_count += candidates.limit_fallbacks_same_ip()
-    operator_count += candidates.limit_fallbacks_same_contact()
-    operator_count += candidates.limit_fallbacks_same_family()
-
-  # check if each candidate can serve a consensus
-  # there's a small risk we've eliminated relays from the same operator that
-  # can serve a consensus, in favour of one that can't
-  # but given it takes up to 15 seconds to check each consensus download,
-  # the risk is worth it
-  if PERFORM_IPV4_DIRPORT_CHECKS or PERFORM_IPV6_DIRPORT_CHECKS:
-    logging.warning('Checking consensus download speeds. ' +
-                    'This may take some time.')
-  failed_count = candidates.perform_download_consensus_checks(max_count)
-
-  # work out which fallbacks cache extra-infos
-  candidates.mark_extra_info_caches()
-
-  # analyse and log interesting diversity metrics
-  # like netblock, ports, exit, IPv4-only
-  # (we can't easily analyse AS, and it's hard to accurately analyse country)
-  candidates.describe_fallback_ip_family()
-  # if we can't import the ipaddress module, we can't do netblock analysis
-  if HAVE_IPADDRESS:
-    candidates.describe_fallback_netblocks()
-  candidates.describe_fallback_ports()
-  candidates.describe_fallback_extra_info_caches()
-  candidates.describe_fallback_exit_flag()
-
-  # output C comments summarising the fallback selection process
-  if len(candidates.fallbacks) > 0:
-    print candidates.summarise_fallbacks(eligible_count, operator_count,
-                                         failed_count, guard_count,
-                                         target_count)
-  else:
-    print '/* No Fallbacks met criteria */'
-
-  # output C comments specifying the OnionOO data used to create the list
-  for s in fetch_source_list():
-    print describe_fetch_source(s)
-
-  # start the list with a separator, to make it easy for parsers
-  print SECTION_SEPARATOR_COMMENT
-
-  # sort the list differently depending on why we've created it:
-  # if we're outputting the final fallback list, sort by fingerprint
-  # this makes diffs much more stable
-  # otherwise, if we're trying to find a bandwidth cutoff, or we want to
-  # contact operators in priority order, sort by bandwidth (not yet
-  # implemented)
-  # otherwise, if we're contacting operators, sort by contact
-  candidates.sort_fallbacks_by(OUTPUT_SORT_FIELD)
-
-  for x in candidates.fallbacks:
-    print x.fallbackdir_line(candidates.fallbacks, prefilter_fallbacks)
-
-if __name__ == "__main__":
-  main()
diff --git a/scripts/maint/updateRustDependencies.sh b/scripts/maint/updateRustDependencies.sh
index a5a92579d3..6d0587351f 100755
--- a/scripts/maint/updateRustDependencies.sh
+++ b/scripts/maint/updateRustDependencies.sh
@@ -20,26 +20,26 @@
 
 set -e
 
-HERE=`dirname $(realpath $0)`
-TOPLEVEL=`dirname $(dirname $HERE)`
+HERE=$(dirname "$(realpath "$0")")
+TOPLEVEL=$(dirname "$(dirname "$HERE")")
 TOML="$TOPLEVEL/src/rust/Cargo.toml"
 VENDORED="$TOPLEVEL/src/ext/rust/crates"
-CARGO=`which cargo`
+CARGO=$(command -v cargo)
 
 if ! test -f "$TOML"  ; then
-    printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\n" "$TOML"
+    printf "Error: Couldn't find workspace Cargo.toml in expected location: %s\\n" "$TOML"
 fi
 
 if ! test -d "$VENDORED" ; then
-    printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\n" "$VENDORED"
+    printf "Error: Couldn't find directory for Rust dependencies! Expected location: %s\\n" "$VENDORED"
 fi
 
 if test -z "$CARGO" ; then
-    printf "Error: cargo must be installed and in your \$PATH\n"
+    printf "Error: cargo must be installed and in your \$PATH\\n"
 fi
 
-if test -z `cargo --list | grep vendor` ; then
-    printf "Error: cargo-vendor not installed\n"
+if test -z "$(cargo --list | grep vendor)" ; then
+    printf "Error: cargo-vendor not installed\\n"
 fi
 
-$CARGO vendor -v --locked --explicit-version --no-delete --sync $TOML $VENDORED
+$CARGO vendor -v --locked --explicit-version --no-delete --sync "$TOML" "$VENDORED"
diff --git a/scripts/maint/updateVersions.pl.in b/scripts/maint/updateVersions.pl.in
deleted file mode 100755
index 65c51a1f2d..0000000000
--- a/scripts/maint/updateVersions.pl.in
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/usr/bin/perl -w
-
-$CONFIGURE_IN = '@abs_top_srcdir@/configure.ac';
-$ORCONFIG_H = '@abs_top_srcdir@/src/win32/orconfig.h';
-$TOR_NSI = '@abs_top_srcdir@/contrib/win32build/tor-mingw.nsi.in';
-
-$quiet = 1;
-
-sub demand {
-    my $fn = shift;
-    die "Missing file $fn" unless (-f $fn);
-}
-
-demand($CONFIGURE_IN);
-demand($ORCONFIG_H);
-demand($TOR_NSI);
-
-# extract version from configure.ac
-
-open(F, $CONFIGURE_IN) or die "$!";
-$version = undef;
-while (<F>) {
-    if (/AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)/) {
-	$version = $1;
-	last;
-    }
-}
-die "No version found" unless $version;
-print "Tor version is $version\n" unless $quiet;
-close F;
-
-sub correctversion {
-    my ($fn, $defchar) = @_;
-    undef $/;
-    open(F, $fn) or die "$!";
-    my $s = <F>;
-    close F;
-    if ($s =~ /^$defchar(?:)define\s+VERSION\s+\"([^\"]+)\"/m) {
-	$oldver = $1;
-	if ($oldver ne $version) {
-	    print "Version mismatch in $fn: It thinks that the version is $oldver.  I think it's $version.  Fixing.\n";
-	    $line = $defchar . "define VERSION \"$version\"";
-	    open(F, ">$fn.bak");
-	    print F $s;
-	    close F;
-	    $s =~ s/^$defchar(?:)define\s+VERSION.*?$/$line/m;
-	    open(F, ">$fn");
-	    print F $s;
-	    close F;
-	} else {
-	    print "$fn has the correct version. Good.\n" unless $quiet;
-	}
-    } else {
-	print "Didn't find a version line in $fn -- uh oh.\n";
-    }
-}
-
-correctversion($TOR_NSI, "!");
-correctversion($ORCONFIG_H, "#");
diff --git a/scripts/maint/update_versions.py b/scripts/maint/update_versions.py
new file mode 100755
index 0000000000..8067f2c6c8
--- /dev/null
+++ b/scripts/maint/update_versions.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import io
+import os
+import re
+import sys
+import time
+
+def P(path):
+    """
+    Give 'path' as a path relative to the abs_top_srcdir environment
+    variable.
+    """
+    return os.path.join(
+        os.environ.get('abs_top_srcdir', "."),
+        path)
+
+def warn(msg):
+    """
+    Print an warning message.
+    """
+    print("WARNING: {}".format(msg), file=sys.stderr)
+
+def find_version(infile):
+    """
+    Given an open file (or some other iterator of lines) holding a
+    configure.ac file, find the current version line.
+    """
+    for line in infile:
+        m = re.search(r'AC_INIT\(\[tor\],\s*\[([^\]]*)\]\)', line)
+        if m:
+            return m.group(1)
+
+    return None
+
+def update_version_in(infile, outfile, regex, versionline):
+    """
+    Copy every line from infile to outfile. If any line matches 'regex',
+    replace it with 'versionline'.  Return True if any line was changed;
+    false otherwise.
+
+    'versionline' is either a string -- in which case it is used literally,
+    or a function that receives the output of 'regex.match'.
+    """
+    found = False
+    have_changed = False
+    for line in infile:
+        m = regex.match(line)
+        if m:
+            found = True
+            oldline = line
+            if type(versionline) == type(u""):
+                line = versionline
+            else:
+                line = versionline(m)
+            if not line.endswith("\n"):
+                line += "\n"
+            if oldline != line:
+                have_changed = True
+        outfile.write(line)
+
+    if not found:
+        warn("didn't find any version line to replace in {}".format(infile.name))
+
+    return have_changed
+
+def replace_on_change(fname, change):
+    """
+    If "change" is true, replace fname with fname.tmp.  Otherwise,
+    delete fname.tmp.  Log what we're doing to stderr.
+    """
+    if not change:
+        print("No change in {}".format(fname))
+        os.unlink(fname+".tmp")
+    else:
+        print("Updating {}".format(fname))
+        os.rename(fname+".tmp", fname)
+
+
+def update_file(fname,
+                regex,
+                versionline,
+                encoding="utf-8"):
+    """
+    Replace any line matching 'regex' in 'fname' with 'versionline'.
+    Do not modify 'fname' if there are no changes made.  Use the
+    provided encoding to read and write.
+    """
+    with io.open(fname, "r", encoding=encoding) as f, \
+         io.open(fname+".tmp", "w", encoding=encoding) as outf:
+        have_changed = update_version_in(f, outf, regex, versionline)
+
+    replace_on_change(fname, have_changed)
+
+# Find out our version
+with open("configure.ac") as f:
+    version = find_version(f)
+
+# If we have no version, we can't proceed.
+if version == None:
+    print("No version found in configure.ac", file=sys.stderr())
+    sys.exit(1)
+
+print("The version is {}".format(version))
+
+today = time.strftime("%Y-%m-%d", time.gmtime())
+
+# In configure.ac, we replace the definition of APPROX_RELEASE_DATE
+# with "{today} for {version}", but only if the version does not match
+# what is already there.
+def replace_fn(m):
+    if m.group(1) != version:
+        # The version changed -- we change the date.
+        return u'AC_DEFINE(APPROX_RELEASE_DATE, ["{}"], # for {}'.format(today, version)
+    else:
+        # No changes.
+        return m.group(0)
+update_file(P("configure.ac"),
+            re.compile(r'AC_DEFINE\(APPROX_RELEASE_DATE.* for (.*)'),
+            replace_fn)
+
+# In tor-mingw.nsi.in, we replace the definition of VERSION.
+update_file(P("contrib/win32build/tor-mingw.nsi.in"),
+            re.compile(r'!define VERSION .*'),
+            u'!define VERSION "{}"'.format(version),
+            encoding="iso-8859-1")
+
+# In src/win32/orconfig.h, we replace the definition of VERSION.
+update_file(P("src/win32/orconfig.h"),
+            re.compile(r'#define VERSION .*'),
+            u'#define VERSION "{}"'.format(version))
diff --git a/scripts/test/chutney-git-bisect.sh b/scripts/test/chutney-git-bisect.sh
index 8a3f2c70c8..dcf8ab1102 100755
--- a/scripts/test/chutney-git-bisect.sh
+++ b/scripts/test/chutney-git-bisect.sh
@@ -15,21 +15,21 @@
 # Skips the test if <skip-flavour> fails (default no skip).
 
 CHUTNEY_TRIES=3
-if [ ! -z "$1" ]; then
+if [ -n "$1" ]; then
     CHUTNEY_TRIES="$1"
 fi
 
-if [ ! -z "$2" ]; then
-    cd "$2"
+if [ -n "$2" ]; then
+    cd "$2" || exit
 fi
 
 CHUTNEY_TEST_CMD="make test-network-all"
-if [ ! -z "$3" ]; then
+if [ -n "$3" ]; then
     CHUTNEY_TEST_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $3"
 fi
 
 CHUTNEY_SKIP_ON_FAIL_CMD="true"
-if [ ! -z "$4" ]; then
+if [ -n "$4" ]; then
     CHUTNEY_SKIP_ON_FAIL_CMD="$CHUTNEY_PATH/tools/test-network.sh --flavour $4"
 fi
 
@@ -54,9 +54,9 @@ while [ "$i" -le "$CHUTNEY_TRIES" ]; do
         echo "test '$CHUTNEY_TEST_CMD' succeeded after $i/$CHUTNEY_TRIES attempts, good"
         exit 0
     fi
-    i=$[$i+1]
+    i=$((i+1))
 done
 
-i=$[$i-1]
+i=$((i-1))
 echo "test '$CHUTNEY_TEST_CMD' failed $i/$CHUTNEY_TRIES attempts, bad"
 exit 1
diff --git a/scripts/test/cov-diff b/scripts/test/cov-diff
index 6179dff63e..8751800966 100755
--- a/scripts/test/cov-diff
+++ b/scripts/test/cov-diff
@@ -7,15 +7,14 @@
 DIRA="$1"
 DIRB="$2"
 
-for B in $DIRB/*; do
-  A=$DIRA/`basename $B`
-  if [ -f $A ]; then
+for B in "$DIRB"/*; do
+  A=$DIRA/$(basename "$B")
+  if [ -f "$A" ]; then
       perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$A" > "$A.tmp"
   else
       cat /dev/null > "$A.tmp"
   fi
   perl -pe 's/^\s*\!*\d+(\*?):/        1$1:/; s/^([^:]+:)[\d\s]+:/$1/; s/^ *-:(Runs|Programs):.*//;' "$B" > "$B.tmp"
   diff -u "$A.tmp" "$B.tmp" |perl -pe 's/^((?:\+\+\+|---)(?:.*tmp))\s+.*/$1/;'
-  rm "$A.tmp" "$B.tmp"
+  rm -f "$A.tmp" "$B.tmp"
 done
-
diff --git a/scripts/test/cov-test-determinism.sh b/scripts/test/cov-test-determinism.sh
new file mode 100755
index 0000000000..3458f96968
--- /dev/null
+++ b/scripts/test/cov-test-determinism.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# To use this script, build Tor with coverage enabled, and then say:
+#  ./scripts/test/cov-test-determinism.sh run
+#
+# Let it run for a long time so it can run the tests over and over.  It
+# will put their coverage outputs in coverage-raw/coverage-*/.
+#
+# Then say:
+#  ./scripts/test/cov-test-determinism.sh check
+#
+# It will diff the other coverage outputs to the first one, and put their
+# diffs in coverage-raw/diff-coverage-*.
+
+run=0
+check=0
+
+if test "$1" = run; then
+    run=1
+elif test "$1" = check; then
+    check=1
+else
+    echo "First use 'run' with this script, then use 'check'."
+    exit 1
+fi
+
+if test "$run" = 1; then
+    # same seed as in travis.yml
+    TOR_TEST_RNG_SEED="636f766572616765"
+    export TOR_TEST_RNG_SEED
+    while true; do
+        make reset-gcov
+        CD=coverage-raw/coverage-$(date +%s)
+        make -j5 check
+        mkdir -p "$CD"
+        ./scripts/test/coverage "$CD"
+    done
+fi
+
+if test "$check" = 1; then
+    cd coverage-raw || exit 1
+
+    FIRST="$(find . -name "coverage-*" -type d | head -1)"
+    rm -f A
+    ln -sf "$FIRST" A
+    for dir in coverage-*; do
+        rm -f B
+        ln -sf "$dir" B
+        ../scripts/test/cov-diff A B > "diff-$dir"
+    done
+fi
diff --git a/scripts/test/coverage b/scripts/test/coverage
index b6e17abe25..f61c83bc72 100755
--- a/scripts/test/coverage
+++ b/scripts/test/coverage
@@ -8,9 +8,9 @@
 dst=$1
 
 for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do
-    BN=`basename $fn`
-    DN=`dirname $fn`
-    F=`echo $BN | sed -e 's/\.c$//;'`
+    BN=$(basename "$fn")
+    DN=$(dirname "$fn")
+    F=$(echo "$BN" | sed -e 's/\.c$//;')
     GC="${BN}.gcov"
     # Figure out the object file names
     ONS=$(echo "${DN}"/*testing_a-"${F}".o)
@@ -20,18 +20,18 @@ for fn in src/core/*/*.c src/feature/*/*.c src/app/*/*.c src/lib/*/*.c; do
     then
       for on in $ONS; do
         # We should have a gcno file
-        GCNO=`echo $on | sed -e 's/\.o$/\.gcno/;'`
-        if [ -e $GCNO ]
+        GCNO=$(echo "$on" | sed -e 's/\.o$/\.gcno/;')
+        if [ -e "$GCNO" ]
         then
           # No need to test for gcda, since gcov assumes no execution
           # if it's absent
-          rm -f $GC
-          gcov -o $on $fn
-          if [ -e $GC ]
+          rm -f "$GC"
+          gcov -o "$on" "$fn"
+          if [ -e "$GC" ]
           then
             if [ -d "$dst" ]
             then
-              mv $GC $dst/$GC
+              mv "$GC" "$dst"/"$GC"
             fi
           else
             echo "gcov -o $on $fn didn't make a .gcov file"
diff --git a/scripts/test/scan-build.sh b/scripts/test/scan-build.sh
index 8d126cbcee..26e05ff101 100755
--- a/scripts/test/scan-build.sh
+++ b/scripts/test/scan-build.sh
@@ -33,6 +33,7 @@ CHECKERS="\
     -enable-checker security.insecureAPI.strcpy \
 "
 
+# shellcheck disable=SC2034
 # These have high false-positive rates.
 EXTRA_CHECKERS="\
     -enable-checker alpha.security.ArrayBoundV2 \
@@ -40,6 +41,7 @@ EXTRA_CHECKERS="\
     -enable-checker alpha.core.CastSize \
 "
 
+# shellcheck disable=SC2034
 # These don't seem to generate anything useful
 NOISY_CHECKERS="\
     -enable-checker alpha.clone.CloneChecker \
@@ -52,6 +54,7 @@ else
    OUTPUTARG=""
 fi
 
+# shellcheck disable=SC2086
 scan-build \
     $CHECKERS \
     ./configure
@@ -61,11 +64,13 @@ scan-build \
 
 # Make this not get scanned for dead assignments, since it has lots of
 # dead assignments we don't care about.
+# shellcheck disable=SC2086
 scan-build \
     $CHECKERS \
     -disable-checker deadcode.DeadStores \
     make -j5 -k ./src/ext/ed25519/ref10/libed25519_ref10.a
 
+# shellcheck disable=SC2086
 scan-build \
     $CHECKERS $OUTPUTARG \
     make -j5 -k
diff --git a/src/app/config/config.c b/src/app/config/config.c
index 1d61b76310..f27fcd3108 100644
--- a/src/app/config/config.c
+++ b/src/app/config/config.c
@@ -61,9 +61,10 @@
 #define CONFIG_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "app/config/statefile.h"
 #include "app/main/main.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
@@ -85,6 +86,8 @@
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/guardfraction.h"
 #include "feature/dircache/consdiffmgr.h"
@@ -108,13 +111,14 @@
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
+#include "lib/confmgt/structvar.h"
 #include "lib/crypt_ops/crypto_init.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/encoding/confline.h"
-#include "lib/log/git_revision.h"
 #include "lib/net/resolve.h"
 #include "lib/sandbox/sandbox.h"
+#include "lib/version/torversion.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
@@ -144,7 +148,7 @@
 #include "lib/process/pidfile.h"
 #include "lib/process/restrict.h"
 #include "lib/process/setuid.h"
-#include "lib/process/subprocess.h"
+#include "lib/process/process.h"
 #include "lib/net/gethostname.h"
 #include "lib/thread/numcpus.h"
 
@@ -153,6 +157,7 @@
 #include "lib/evloop/procmon.h"
 
 #include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
 #include "feature/dirauth/recommend_pkg.h"
 #include "feature/dirauth/authmode.h"
 
@@ -186,7 +191,7 @@ static const char unix_q_socket_prefix[] = "unix:\"";
 
 /** A list of abbreviations and aliases to map command-line options, obsolete
  * option names, or alternative option names, to their current values. */
-static config_abbrev_t option_abbrevs_[] = {
+static const config_abbrev_t option_abbrevs_[] = {
   PLURAL(AuthDirBadDirCC),
   PLURAL(AuthDirBadExitCC),
   PLURAL(AuthDirInvalidCC),
@@ -249,22 +254,33 @@ static config_abbrev_t option_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_options_t);
 
-/** An entry for config_vars: "The option <b>name</b> has type
+/** An entry for config_vars: "The option <b>varname</b> has type
  * CONFIG_TYPE_<b>conftype</b>, and corresponds to
  * or_options_t.<b>member</b>"
  */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member),     \
-      initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, 0, initvalue)
+
+/* As VAR, but uses a type definition in addition to a type enum. */
+#define VAR_D(varname,conftype,member,initvalue)                        \
+  CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, 0, initvalue)
+
+#define VAR_NODUMP(varname,conftype,member,initvalue)             \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP, initvalue)
+#define VAR_INVIS(varname,conftype,member,initvalue)              \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP | CFLG_NOSET | CFLG_NOLIST, initvalue)
+
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
-/** An entry for config_vars: "The option <b>name</b> is obsolete." */
-#ifdef TOR_UNIT_TESTS
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} }
-#else
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL }
-#endif
+
+/** As V, but uses a type definition instead of a type enum */
+#define V_D(member,type,initvalue)              \
+  VAR_D(#member, type, member, initvalue)
+
+/** An entry for config_vars: "The option <b>varname</b> is obsolete." */
+#define OBSOLETE(varname) CONFIG_VAR_OBSOLETE(varname)
 
 /**
  * Macro to declare *Port options.  Each one comes in three entries.
@@ -276,7 +292,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
 #define VPORT(member)                                           \
   VAR(#member "Lines", LINELIST_V, member ## _lines, NULL),     \
   VAR(#member, LINELIST_S, member ## _lines, NULL),             \
-  VAR("__" #member, LINELIST_S, member ## _lines, NULL)
+  VAR_NODUMP("__" #member, LINELIST_S, member ## _lines, NULL)
 
 /** UINT64_MAX as a decimal string */
 #define UINT64_MAX_STRING "18446744073709551615"
@@ -285,7 +301,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
  * abbreviations, order is significant, since the first matching option will
  * be chosen first.
  */
-static config_var_t option_vars_[] = {
+static const config_var_t option_vars_[] = {
   V(AccountingMax,               MEMUNIT,  "0 bytes"),
   VAR("AccountingRule",          STRING,   AccountingRule_option,  "max"),
   V(AccountingStart,             STRING,   NULL),
@@ -313,7 +329,7 @@ static config_var_t option_vars_[] = {
   OBSOLETE("AuthDirRejectUnlisted"),
   OBSOLETE("AuthDirListBadDirs"),
   V(AuthDirListBadExits,         BOOL,     "0"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "2"),
+  V(AuthDirMaxServersPerAddr,    POSINT,     "2"),
   OBSOLETE("AuthDirMaxServersPerAuthAddr"),
   V(AuthDirHasIPv6Connectivity,  BOOL,     "0"),
   VAR("AuthoritativeDirectory",  BOOL, AuthoritativeDir,    "0"),
@@ -342,12 +358,13 @@ static config_var_t option_vars_[] = {
   V(ClientOnly,                  BOOL,     "0"),
   V(ClientPreferIPv6ORPort,      AUTOBOOL, "auto"),
   V(ClientPreferIPv6DirPort,     AUTOBOOL, "auto"),
+  V(ClientAutoIPv6ORPort,        BOOL,     "0"),
   V(ClientRejectInternalAddresses, BOOL,   "1"),
   V(ClientTransportPlugin,       LINELIST, NULL),
   V(ClientUseIPv6,               BOOL,     "0"),
   V(ClientUseIPv4,               BOOL,     "1"),
   V(ConsensusParams,             STRING,   NULL),
-  V(ConnLimit,                   UINT,     "1000"),
+  V(ConnLimit,                   POSINT,     "1000"),
   V(ConnDirectionStatistics,     BOOL,     "0"),
   V(ConstrainedSockets,          BOOL,     "0"),
   V(ConstrainedSockSize,         MEMUNIT,  "8192"),
@@ -391,16 +408,20 @@ static config_var_t option_vars_[] = {
   OBSOLETE("DynamicDHGroups"),
   VPORT(DNSPort),
   OBSOLETE("DNSListenAddress"),
+  V(DormantClientTimeout,         INTERVAL, "24 hours"),
+  V(DormantTimeoutDisabledByIdleStreams, BOOL,     "1"),
+  V(DormantOnFirstStartup,       BOOL,      "0"),
+  V(DormantCanceledByStartup,    BOOL,      "0"),
   /* DoS circuit creation options. */
   V(DoSCircuitCreationEnabled,   AUTOBOOL, "auto"),
-  V(DoSCircuitCreationMinConnections,      UINT, "0"),
-  V(DoSCircuitCreationRate,      UINT,     "0"),
-  V(DoSCircuitCreationBurst,     UINT,     "0"),
+  V(DoSCircuitCreationMinConnections,      POSINT, "0"),
+  V(DoSCircuitCreationRate,      POSINT,     "0"),
+  V(DoSCircuitCreationBurst,     POSINT,     "0"),
   V(DoSCircuitCreationDefenseType,         INT,  "0"),
   V(DoSCircuitCreationDefenseTimePeriod,   INTERVAL, "0"),
   /* DoS connection options. */
   V(DoSConnectionEnabled,        AUTOBOOL, "auto"),
-  V(DoSConnectionMaxConcurrentCount,       UINT, "0"),
+  V(DoSConnectionMaxConcurrentCount,       POSINT, "0"),
   V(DoSConnectionDefenseType,    INT,      "0"),
   /* DoS single hop client options. */
   V(DoSRefuseSingleHopClientRendezvous,    AUTOBOOL, "auto"),
@@ -409,13 +430,17 @@ static config_var_t option_vars_[] = {
   V(TestingEnableCellStatsEvent, BOOL,     "0"),
   OBSOLETE("TestingEnableTbEmptyEvent"),
   V(EnforceDistinctSubnets,      BOOL,     "1"),
-  V(EntryNodes,                  ROUTERSET,   NULL),
+  V_D(EntryNodes,                ROUTERSET,   NULL),
   V(EntryStatistics,             BOOL,     "0"),
   V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"),
-  V(ExcludeNodes,                ROUTERSET, NULL),
-  V(ExcludeExitNodes,            ROUTERSET, NULL),
+  V_D(ExcludeNodes,              ROUTERSET, NULL),
+  V_D(ExcludeExitNodes,          ROUTERSET, NULL),
   OBSOLETE("ExcludeSingleHopRelays"),
-  V(ExitNodes,                   ROUTERSET, NULL),
+  V_D(ExitNodes,                 ROUTERSET, NULL),
+  /* Researchers need a way to tell their clients to use specific
+   * middles that they also control, to allow safe live-network
+   * experimentation with new padding machines. */
+  V_D(MiddleNodes,               ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
   V(ExitPolicyRejectLocalInterfaces, BOOL, "0"),
@@ -471,6 +496,11 @@ static config_var_t option_vars_[] = {
   VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceExportCircuitID", LINELIST_S,  RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSDefense", LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSRatePerSec",
+      LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSBurstPerSec",
+      LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"),
   V(HidServAuth,                 LINELIST, NULL),
   V(ClientOnionAuthDir,          FILENAME, NULL),
@@ -494,8 +524,8 @@ static config_var_t option_vars_[] = {
   V(Socks5ProxyPassword,         STRING,   NULL),
   VAR("KeyDirectory",            FILENAME, KeyDirectory_option, NULL),
   V(KeyDirectoryGroupReadable,   BOOL,     "0"),
-  VAR("HSLayer2Nodes",           ROUTERSET,  HSLayer2Nodes,  NULL),
-  VAR("HSLayer3Nodes",           ROUTERSET,  HSLayer3Nodes,  NULL),
+  VAR_D("HSLayer2Nodes",         ROUTERSET,  HSLayer2Nodes,  NULL),
+  VAR_D("HSLayer3Nodes",         ROUTERSET,  HSLayer3Nodes,  NULL),
   V(KeepalivePeriod,             INTERVAL, "5 minutes"),
   V(KeepBindCapabilities,            AUTOBOOL, "auto"),
   VAR("Log",                     LINELIST, Logs,             NULL),
@@ -509,7 +539,7 @@ static config_var_t option_vars_[] = {
   VAR("MapAddress",              LINELIST, AddressMap,           NULL),
   V(MaxAdvertisedBandwidth,      MEMUNIT,  "1 GB"),
   V(MaxCircuitDirtiness,         INTERVAL, "10 minutes"),
-  V(MaxClientCircuitsPending,    UINT,     "32"),
+  V(MaxClientCircuitsPending,    POSINT,     "32"),
   V(MaxConsensusAgeForDiffs,     INTERVAL, "0 seconds"),
   VAR("MaxMemInQueues",          MEMUNIT,   MaxMemInQueues_raw, "0"),
   OBSOLETE("MaxOnionsPending"),
@@ -526,10 +556,10 @@ static config_var_t option_vars_[] = {
   OBSOLETE("WarnUnsafeSocks"),
   VAR("NodeFamily",              LINELIST, NodeFamilies,         NULL),
   V(NoExec,                      BOOL,     "0"),
-  V(NumCPUs,                     UINT,     "0"),
-  V(NumDirectoryGuards,          UINT,     "0"),
-  V(NumEntryGuards,              UINT,     "0"),
-  V(NumPrimaryGuards,            UINT,     "0"),
+  V(NumCPUs,                     POSINT,     "0"),
+  V(NumDirectoryGuards,          POSINT,     "0"),
+  V(NumEntryGuards,              POSINT,     "0"),
+  V(NumPrimaryGuards,            POSINT,     "0"),
   V(OfflineMasterKey,            BOOL,     "0"),
   OBSOLETE("ORListenAddress"),
   VPORT(ORPort),
@@ -580,10 +610,12 @@ static config_var_t option_vars_[] = {
   V(RecommendedVersions,         LINELIST, NULL),
   V(RecommendedClientVersions,   LINELIST, NULL),
   V(RecommendedServerVersions,   LINELIST, NULL),
-  V(RecommendedPackages,         LINELIST, NULL),
+  OBSOLETE("RecommendedPackages"),
   V(ReducedConnectionPadding,    BOOL,     "0"),
   V(ConnectionPadding,           AUTOBOOL, "auto"),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
+  V(CircuitPadding,              BOOL,     "1"),
+  V(ReducedCircuitPadding,       BOOL,     "0"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
   V(RelayBandwidthRate,          MEMUNIT,  "0"),
@@ -639,6 +671,7 @@ static config_var_t option_vars_[] = {
   OBSOLETE("UseNTorHandshake"),
   V(User,                        STRING,   NULL),
   OBSOLETE("UserspaceIOCPBuffers"),
+  V(AuthDirRejectRequestsUnderLoad, BOOL,  "1"),
   V(AuthDirSharedRandomness,     BOOL,     "1"),
   V(AuthDirTestEd25519LinkKeys,  BOOL,     "1"),
   OBSOLETE("V1AuthoritativeDirectory"),
@@ -651,7 +684,7 @@ static config_var_t option_vars_[] = {
   V(V3AuthVotingInterval,        INTERVAL, "1 hour"),
   V(V3AuthVoteDelay,             INTERVAL, "5 minutes"),
   V(V3AuthDistDelay,             INTERVAL, "5 minutes"),
-  V(V3AuthNIntervalsValid,       UINT,     "3"),
+  V(V3AuthNIntervalsValid,       POSINT,     "3"),
   V(V3AuthUseLegacyKey,          BOOL,     "0"),
   V(V3BandwidthsFile,            FILENAME, NULL),
   V(GuardfractionFile,           FILENAME, NULL),
@@ -662,15 +695,17 @@ static config_var_t option_vars_[] = {
   V(WarnPlaintextPorts,          CSV,      "23,109,110,143"),
   OBSOLETE("UseFilteringSSLBufferevents"),
   OBSOLETE("__UseFilteringSSLBufferevents"),
-  VAR("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
-  VAR("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
-  VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
-  VAR("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
-  VAR("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
-  VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword,
+  VAR_NODUMP("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
+  VAR_NODUMP("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
+  VAR_NODUMP("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
+  VAR_NODUMP("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
+  VAR_NODUMP("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
+  VAR_NODUMP("__HashedControlSessionPassword", LINELIST,
+             HashedControlSessionPassword,
       NULL),
-  VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
-  VAR("__OwningControllerFD", UINT64, OwningControllerFD, UINT64_MAX_STRING),
+  VAR_NODUMP("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
+  VAR_NODUMP("__OwningControllerFD", UINT64, OwningControllerFD,
+             UINT64_MAX_STRING),
   V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"),
   V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
@@ -700,7 +735,7 @@ static config_var_t option_vars_[] = {
    * blocked), but we also don't want to fail if only some mirrors are
    * blackholed. Clients will try 3 directories simultaneously.
    * (Relays never use simultaneous connections.) */
-  V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"),
+  V(ClientBootstrapConsensusMaxInProgressTries, POSINT, "3"),
   /* When a client has any running bridges, check each bridge occasionally,
     * whether or not that bridge is actually up. */
   V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"),
@@ -717,56 +752,40 @@ static config_var_t option_vars_[] = {
   OBSOLETE("TestingDescriptorMaxDownloadTries"),
   OBSOLETE("TestingMicrodescMaxDownloadTries"),
   OBSOLETE("TestingCertMaxDownloadTries"),
-  V(TestingDirAuthVoteExit, ROUTERSET, NULL),
+  V_D(TestingDirAuthVoteExit, ROUTERSET, NULL),
   V(TestingDirAuthVoteExitIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteGuard, ROUTERSET, NULL),
+  V_D(TestingDirAuthVoteGuard, ROUTERSET, NULL),
   V(TestingDirAuthVoteGuardIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteHSDir, ROUTERSET, NULL),
+  V_D(TestingDirAuthVoteHSDir, ROUTERSET, NULL),
   V(TestingDirAuthVoteHSDirIsStrict,  BOOL,     "0"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"),
+  VAR_INVIS("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_,
+            "0"),
 
   END_OF_CONFIG_VARS
 };
 
+/** List of default directory authorities */
+static const char *default_authorities[] = {
+#include "auth_dirs.inc"
+  NULL
+};
+
+/** List of fallback directory authorities. The list is generated by opt-in of
+ * relays that meet certain stability criteria.
+ */
+static const char *default_fallbacks[] = {
+#include "fallback_dirs.inc"
+  NULL
+};
+
 /** Override default values with these if the user sets the TestingTorNetwork
  * option. */
-static const config_var_t testing_tor_network_defaults[] = {
-  V(DirAllowPrivateAddresses,    BOOL,     "1"),
-  V(EnforceDistinctSubnets,      BOOL,     "0"),
-  V(AssumeReachable,             BOOL,     "1"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "0"),
-  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
-    "0"),
-  V(ClientDNSRejectInternalAddresses, BOOL,"0"),
-  V(ClientRejectInternalAddresses, BOOL,   "0"),
-  V(CountPrivateBandwidth,       BOOL,     "1"),
-  V(ExitPolicyRejectPrivate,     BOOL,     "0"),
-  V(ExtendAllowPrivateAddresses, BOOL,     "1"),
-  V(V3AuthVotingInterval,        INTERVAL, "5 minutes"),
-  V(V3AuthVoteDelay,             INTERVAL, "20 seconds"),
-  V(V3AuthDistDelay,             INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"),
-  V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"),
-  V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"),
-  V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"),
-  V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"),
-  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"),
-  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"),
-  V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"),
-  V(TestingEnableConnBwEvent,    BOOL,     "1"),
-  V(TestingEnableCellStatsEvent, BOOL,     "1"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"),
-  V(RendPostPeriod,              INTERVAL, "2 minutes"),
-
-  END_OF_CONFIG_VARS
+static const struct {
+  const char *k;
+  const char *v;
+} testing_tor_network_defaults[] = {
+#include "testnet.inc"
+  { NULL, NULL }
 };
 
 #undef VAR
@@ -830,24 +849,28 @@ static void config_maybe_load_geoip_files_(const or_options_t *options,
 static int options_validate_cb(void *old_options, void *options,
                                void *default_options,
                                int from_setconf, char **msg);
-static void options_free_cb(void *options);
 static void cleanup_protocol_warning_severity_level(void);
 static void set_protocol_warning_severity_level(int warning_severity);
+static void options_clear_cb(const config_mgr_t *mgr, void *opts);
 
 /** Magic value for or_options_t. */
 #define OR_OPTIONS_MAGIC 9090909
 
 /** Configuration format for or_options_t. */
-STATIC config_format_t options_format = {
+static const config_format_t options_format = {
   sizeof(or_options_t),
-  OR_OPTIONS_MAGIC,
-  offsetof(or_options_t, magic_),
+  {
+   "or_options_t",
+   OR_OPTIONS_MAGIC,
+   offsetof(or_options_t, magic_),
+  },
   option_abbrevs_,
   option_deprecation_notes_,
   option_vars_,
   options_validate_cb,
-  options_free_cb,
-  NULL
+  options_clear_cb,
+  NULL,
+  offsetof(or_options_t, subconfigs_),
 };
 
 /*
@@ -879,6 +902,20 @@ static int in_option_validation = 0;
 /* True iff we've initialized libevent */
 static int libevent_initialized = 0;
 
+/* A global configuration manager to handle all configuration objects. */
+static config_mgr_t *options_mgr = NULL;
+
+/** Return the global configuration manager object for torrc options. */
+STATIC const config_mgr_t *
+get_options_mgr(void)
+{
+  if (PREDICT_UNLIKELY(options_mgr == NULL)) {
+    options_mgr = config_mgr_new(&options_format);
+    config_mgr_freeze(options_mgr);
+  }
+  return options_mgr;
+}
+
 /** Return the contents of our frontpage string, or NULL if not configured. */
 MOCK_IMPL(const char*,
 get_dirportfrontpage, (void))
@@ -902,6 +939,32 @@ get_options,(void))
   return get_options_mutable();
 }
 
+/**
+ * True iff we have noticed that this is a testing tor network, and we
+ * should use the corresponding defaults.
+ **/
+static bool testing_network_configured = false;
+
+/** Return a set of lines for any default options that we want to override
+ * from those set in our config_var_t values. */
+static config_line_t *
+get_options_defaults(void)
+{
+  int i;
+  config_line_t *result = NULL, **next = &result;
+
+  if (testing_network_configured) {
+    for (i = 0; testing_tor_network_defaults[i].k; ++i) {
+      config_line_append(next,
+                         testing_tor_network_defaults[i].k,
+                         testing_tor_network_defaults[i].v);
+      next = &(*next)->next;
+    }
+  }
+
+  return result;
+}
+
 /** Change the current global options to contain <b>new_val</b> instead of
  * their current value; take action based on the new value; free the old value
  * as necessary.  Returns 0 on success, -1 on failure.
@@ -909,9 +972,6 @@ get_options,(void))
 int
 set_options(or_options_t *new_val, char **msg)
 {
-  int i;
-  smartlist_t *elements;
-  config_line_t *line;
   or_options_t *old_options = global_options;
   global_options = new_val;
   /* Note that we pass the *old* options below, for comparison. It
@@ -933,35 +993,16 @@ set_options(or_options_t *new_val, char **msg)
   /* Issues a CONF_CHANGED event to notify controller of the change. If Tor is
    * just starting up then the old_options will be undefined. */
   if (old_options && old_options != global_options) {
-    elements = smartlist_new();
-    for (i=0; options_format.vars[i].name; ++i) {
-      const config_var_t *var = &options_format.vars[i];
-      const char *var_name = var->name;
-      if (var->type == CONFIG_TYPE_LINELIST_S ||
-          var->type == CONFIG_TYPE_OBSOLETE) {
-        continue;
-      }
-      if (!config_is_same(&options_format, new_val, old_options, var_name)) {
-        line = config_get_assigned_option(&options_format, new_val,
-                                          var_name, 1);
-
-        if (line) {
-          config_line_t *next;
-          for (; line; line = next) {
-            next = line->next;
-            smartlist_add(elements, line->key);
-            smartlist_add(elements, line->value);
-            tor_free(line);
-          }
-        } else {
-          smartlist_add_strdup(elements, options_format.vars[i].name);
-          smartlist_add(elements, NULL);
-        }
-      }
+    smartlist_t *elements = smartlist_new();
+    config_line_t *changes =
+      config_get_changes(get_options_mgr(), old_options, new_val);
+    for (config_line_t *line = changes; line; line = line->next) {
+      smartlist_add(elements, line->key);
+      smartlist_add(elements, line->value);
     }
     control_event_conf_changed(elements);
-    SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
     smartlist_free(elements);
+    config_free_lines(changes);
   }
 
   if (old_options != global_options) {
@@ -975,49 +1016,13 @@ set_options(or_options_t *new_val, char **msg)
   return 0;
 }
 
-/** The version of this Tor process, as parsed. */
-static char *the_tor_version = NULL;
-/** A shorter version of this Tor process's version, for export in our router
- *  descriptor.  (Does not include the git version, if any.) */
-static char *the_short_tor_version = NULL;
-
-/** Return the current Tor version. */
-const char *
-get_version(void)
-{
-  if (the_tor_version == NULL) {
-    if (strlen(tor_git_revision)) {
-      tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(),
-                   tor_git_revision);
-    } else {
-      the_tor_version = tor_strdup(get_short_version());
-    }
-  }
-  return the_tor_version;
-}
-
-/** Return the current Tor version, without any git tag. */
-const char *
-get_short_version(void)
-{
-
-  if (the_short_tor_version == NULL) {
-#ifdef TOR_BUILD_TAG
-    tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG);
-#else
-    the_short_tor_version = tor_strdup(VERSION);
-#endif
-  }
-  return the_short_tor_version;
-}
-
 /** Release additional memory allocated in options
  */
-STATIC void
-or_options_free_(or_options_t *options)
+static void
+options_clear_cb(const config_mgr_t *mgr, void *opts)
 {
-  if (!options)
-    return;
+  (void)mgr;
+  or_options_t *options = opts;
 
   routerset_free(options->ExcludeExitNodesUnion_);
   if (options->NodeFamilySets) {
@@ -1040,7 +1045,14 @@ or_options_free_(or_options_t *options)
   tor_free(options->command_arg);
   tor_free(options->master_key_fname);
   config_free_lines(options->MyFamily);
-  config_free(&options_format, options);
+}
+
+/** Release all memory allocated in options
+ */
+STATIC void
+or_options_free_(or_options_t *options)
+{
+  config_free(get_options_mgr(), options);
 }
 
 /** Release all memory and resources held by global configuration structures.
@@ -1070,13 +1082,12 @@ config_free_all(void)
   tor_free(torrc_defaults_fname);
   tor_free(global_dirfrontpagecontents);
 
-  tor_free(the_short_tor_version);
-  tor_free(the_tor_version);
-
   cleanup_protocol_warning_severity_level();
 
   have_parsed_cmdline = 0;
   libevent_initialized = 0;
+
+  config_mgr_free(options_mgr);
 }
 
 /** Make <b>address</b> -- a piece of information related to our operation as
@@ -1187,24 +1198,13 @@ init_protocol_warning_severity_level(void)
 static void
 cleanup_protocol_warning_severity_level(void)
 {
-   atomic_counter_destroy(&protocol_warning_severity_level);
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
+  atomic_counter_destroy(&protocol_warning_severity_level);
 }
 
-/** List of default directory authorities */
-
-static const char *default_authorities[] = {
-#include "auth_dirs.inc"
-  NULL
-};
-
-/** List of fallback directory authorities. The list is generated by opt-in of
- * relays that meet certain stability criteria.
- */
-static const char *default_fallbacks[] = {
-#include "fallback_dirs.inc"
-  NULL
-};
-
 /** Add the default directory authorities directly into the trusted dir list,
  * but only add them insofar as they share bits with <b>type</b>.
  * Each authority's bits are restricted to the bits shared with <b>type</b>.
@@ -1443,10 +1443,10 @@ options_act_reversible(const or_options_t *old_options, char **msg)
    * processes. */
   if (running_tor && options->RunAsDaemon) {
     if (! start_daemon_has_been_called())
-      crypto_prefork();
+      subsystems_prefork();
     /* No need to roll back, since you can't change the value. */
     if (start_daemon())
-      crypto_postfork();
+      subsystems_postfork();
   }
 
 #ifdef HAVE_SYSTEMD
@@ -1460,7 +1460,7 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                       "on this OS/with this build.");
     goto rollback;
   }
-#else /* !(!defined(HAVE_SYS_UN_H)) */
+#else /* defined(HAVE_SYS_UN_H) */
   if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
     *msg = tor_strdup("Setting ControlSocketGroupWritable without setting"
                       "a ControlSocket makes no sense.");
@@ -1735,6 +1735,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out)
   int routerset_usage =
     routerset_needs_geoip(options->EntryNodes) ||
     routerset_needs_geoip(options->ExitNodes) ||
+    routerset_needs_geoip(options->MiddleNodes) ||
     routerset_needs_geoip(options->ExcludeExitNodes) ||
     routerset_needs_geoip(options->ExcludeNodes) ||
     routerset_needs_geoip(options->HSLayer2Nodes) ||
@@ -2042,9 +2043,6 @@ options_act(const or_options_t *old_options)
     finish_daemon(options->DataDirectory);
   }
 
-  /* See whether we need to enable/disable our once-a-second timer. */
-  reschedule_per_second_timer();
-
   /* We want to reinit keys as needed before we do much of anything else:
      keys are important, and other things can depend on them. */
   if (transition_affects_workers ||
@@ -2177,6 +2175,7 @@ options_act(const or_options_t *old_options)
                          options->HSLayer2Nodes) ||
         !routerset_equal(old_options->HSLayer3Nodes,
                          options->HSLayer3Nodes) ||
+        !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) ||
         options->StrictNodes != old_options->StrictNodes) {
       log_info(LD_CIRC,
                "Changed to using entry guards or bridges, or changed "
@@ -2560,7 +2559,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
 
     param = tor_malloc_zero(sizeof(config_line_t));
     param->key = is_cmdline ? tor_strdup(argv[i]) :
-                   tor_strdup(config_expand_abbrev(&options_format, s, 1, 1));
+                 tor_strdup(config_expand_abbrev(get_options_mgr(), s, 1, 1));
     param->value = arg;
     param->command = command;
     param->next = NULL;
@@ -2586,8 +2585,7 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
 int
 option_is_recognized(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return (var != NULL);
+  return config_find_option_name(get_options_mgr(), key) != NULL;
 }
 
 /** Return the canonical name of a configuration option, or NULL
@@ -2595,8 +2593,7 @@ option_is_recognized(const char *key)
 const char *
 option_get_canonical_name(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return var ? var->name : NULL;
+  return config_find_option_name(get_options_mgr(), key);
 }
 
 /** Return a canonical list of the options assigned for key.
@@ -2604,7 +2601,7 @@ option_get_canonical_name(const char *key)
 config_line_t *
 option_get_assignment(const or_options_t *options, const char *key)
 {
-  return config_get_assigned_option(&options_format, options, key, 1);
+  return config_get_assigned_option(get_options_mgr(), options, key, 1);
 }
 
 /** Try assigning <b>list</b> to the global options. You do this by duping
@@ -2620,9 +2617,9 @@ setopt_err_t
 options_trial_assign(config_line_t *list, unsigned flags, char **msg)
 {
   int r;
-  or_options_t *trial_options = config_dup(&options_format, get_options());
+  or_options_t *trial_options = config_dup(get_options_mgr(), get_options());
 
-  if ((r=config_assign(&options_format, trial_options,
+  if ((r=config_assign(get_options_mgr(), trial_options,
                        list, flags, msg)) < 0) {
     or_options_free(trial_options);
     return r;
@@ -2677,24 +2674,30 @@ print_usage(void)
 static void
 list_torrc_options(void)
 {
-  int i;
-  for (i = 0; option_vars_[i].name; ++i) {
-    const config_var_t *var = &option_vars_[i];
-    if (var->type == CONFIG_TYPE_OBSOLETE ||
-        var->type == CONFIG_TYPE_LINELIST_V)
+  smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+  SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+    /* Possibly this should check listable, rather than (or in addition to)
+     * settable. See ticket 31654.
+     */
+    if (! config_var_is_settable(var)) {
+      /* This variable cannot be set, or cannot be set by this name. */
       continue;
-    printf("%s\n", var->name);
-  }
+    }
+    printf("%s\n", var->member.name);
+  } SMARTLIST_FOREACH_END(var);
+  smartlist_free(vars);
 }
 
 /** Print all deprecated but non-obsolete torrc options. */
 static void
 list_deprecated_options(void)
 {
-  const config_deprecation_t *d;
-  for (d = option_deprecation_notes_; d->name; ++d) {
-    printf("%s\n", d->name);
-  }
+  smartlist_t *deps = config_mgr_list_deprecated_vars(get_options_mgr());
+  /* Possibly this should check whether the variables are listable,
+   * but currently it does not.  See ticket 31654. */
+  SMARTLIST_FOREACH(deps, const char *, name,
+                    printf("%s\n", name));
+  smartlist_free(deps);
 }
 
 /** Print all compile-time modules and their enabled/disabled status. */
@@ -3004,7 +3007,7 @@ is_local_addr, (const tor_addr_t *addr))
 or_options_t *
 options_new(void)
 {
-  return config_new(&options_format);
+  return config_new(get_options_mgr());
 }
 
 /** Set <b>options</b> to hold reasonable defaults for most options.
@@ -3012,7 +3015,17 @@ options_new(void)
 void
 options_init(or_options_t *options)
 {
-  config_init(&options_format, options);
+  config_init(get_options_mgr(), options);
+  config_line_t *dflts = get_options_defaults();
+  char *msg=NULL;
+  if (config_assign(get_options_mgr(), options, dflts,
+                    CAL_WARN_DEPRECATIONS, &msg)<0) {
+    log_err(LD_BUG, "Unable to set default options: %s", msg);
+    tor_free(msg);
+    tor_assert_unreached();
+  }
+  config_free_lines(dflts);
+  tor_free(msg);
 }
 
 /** Return a string containing a possible configuration file that would give
@@ -3042,7 +3055,7 @@ options_dump(const or_options_t *options, int how_to_dump)
       return NULL;
   }
 
-  return config_dump(&options_format, use_defaults, options, minimal, 0);
+  return config_dump(get_options_mgr(), use_defaults, options, minimal, 0);
 }
 
 /** Return 0 if every element of sl is a string holding a decimal
@@ -3174,13 +3187,6 @@ options_validate_cb(void *old_options, void *options, void *default_options,
   return rv;
 }
 
-/** Callback to free an or_options_t */
-static void
-options_free_cb(void *options)
-{
-  or_options_free_(options);
-}
-
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
 #if defined(__GNUC__) && __GNUC__ <= 3
@@ -3446,6 +3452,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo)))
     REJECT("ContactInfo config option must be UTF-8.");
 
+  check_network_configuration(server_mode(options));
+
   /* Special case on first boot if no Log options are given. */
   if (!options->Logs && !options->RunAsDaemon && !from_setconf) {
     if (quiet_level == 0)
@@ -3516,7 +3524,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("Cannot use TransProxyType without any valid TransPort.");
     }
   }
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   if (options->TransPort_set)
     REJECT("TransPort is disabled in this build.");
 #endif /* defined(USE_TRANSPARENT) */
@@ -3550,13 +3558,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "features to be broken in unpredictable ways.");
   }
 
-  for (cl = options->RecommendedPackages; cl; cl = cl->next) {
-    if (! validate_recommended_package_line(cl->value)) {
-      log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored",
-               escaped(cl->value));
-    }
-  }
-
   if (options->AuthoritativeDir) {
     if (!options->ContactInfo && !options->TestingTorNetwork)
       REJECT("Authoritative directory servers must set ContactInfo");
@@ -3579,7 +3580,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
     tor_free(t);
     t = format_recommended_version_list(options->RecommendedServerVersions, 1);
     tor_free(t);
-#endif
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
     if (options->UseEntryGuards) {
       log_info(LD_CONFIG, "Authoritative directory servers can't set "
@@ -3595,14 +3596,17 @@ options_validate(or_options_t *old_options, or_options_t *options,
           options->V3AuthoritativeDir))
       REJECT("AuthoritativeDir is set, but none of "
              "(Bridge/V3)AuthoritativeDir is set.");
+#ifdef HAVE_MODULE_DIRAUTH
     /* If we have a v3bandwidthsfile and it's broken, complain on startup */
     if (options->V3BandwidthsFile && !old_options) {
-      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
+      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                       NULL);
     }
     /* same for guardfraction file */
     if (options->GuardfractionFile && !old_options) {
       dirserv_read_guardfraction_file(options->GuardfractionFile, NULL);
     }
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
   }
 
   if (options->AuthoritativeDir && !options->DirPort_set)
@@ -3780,6 +3784,14 @@ options_validate(or_options_t *old_options, or_options_t *options,
     REJECT("Relays cannot set ReducedConnectionPadding. ");
   }
 
+  if (server_mode(options) && options->CircuitPadding == 0) {
+    REJECT("Relays cannot set CircuitPadding to 0. ");
+  }
+
+  if (server_mode(options) && options->ReducedCircuitPadding == 1) {
+    REJECT("Relays cannot set ReducedCircuitPadding. ");
+  }
+
   if (options->BridgeDistribution) {
     if (!options->BridgeRelay) {
       REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
@@ -3902,6 +3914,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "default.");
   }
 
+  if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) {
+    REJECT("DormantClientTimeout is too low. It must be at least 10 minutes.");
+  }
+
   if (options->PathBiasNoticeRate > 1.0) {
     tor_asprintf(msg,
               "PathBiasNoticeRate is too high. "
@@ -3953,7 +3969,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   if (options->HeartbeatPeriod &&
-      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) {
+      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD &&
+      !options->TestingTorNetwork) {
     log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; "
              "raising to %d seconds.", MIN_HEARTBEAT_PERIOD);
     options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD;
@@ -4225,6 +4242,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "You should also make sure you aren't listing this bridge's "
              "fingerprint in any other MyFamily.");
   }
+  if (options->MyFamily_lines && !options->ContactInfo) {
+    log_warn(LD_CONFIG, "MyFamily is set but ContactInfo is not configured. "
+             "ContactInfo should always be set when MyFamily option is too.");
+  }
   if (normalize_nickname_list(&options->MyFamily,
                               options->MyFamily_lines, "MyFamily", msg))
     return -1;
@@ -4427,7 +4448,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
   STMT_BEGIN                                                            \
     if (!options->TestingTorNetwork &&                                  \
         !options->UsingTestNetworkDefaults_ &&                          \
-        !config_is_same(&options_format,options,                        \
+        !config_is_same(get_options_mgr(),options,                        \
                         default_options,#arg)) {                        \
       REJECT(#arg " may only be changed in testing Tor "                \
              "networks!");                                              \
@@ -4613,7 +4634,7 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 #else
 /* On a 32-bit platform, we can't have 8GB of ram. */
 #define RAM_IS_VERY_LARGE(x) (0)
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
       if (RAM_IS_VERY_LARGE(ram)) {
         /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues
@@ -5090,7 +5111,7 @@ find_torrc_filename(config_line_t *cmd_arg,
       } else {
         fname = dflt ? tor_strdup(dflt) : NULL;
       }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
       fname = dflt ? tor_strdup(dflt) : NULL;
 #endif /* !defined(_WIN32) */
     }
@@ -5392,6 +5413,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
                          int command, const char *command_arg,
                          char **msg)
 {
+  bool retry = false;
   or_options_t *oldoptions, *newoptions, *newdefaultoptions=NULL;
   config_line_t *cl;
   int retval;
@@ -5402,8 +5424,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   oldoptions = global_options; /* get_options unfortunately asserts if
                                   this is the first time we run*/
 
-  newoptions = tor_malloc_zero(sizeof(or_options_t));
-  newoptions->magic_ = OR_OPTIONS_MAGIC;
+  newoptions = options_new();
   options_init(newoptions);
   newoptions->command = command;
   newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
@@ -5422,7 +5443,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       err = SETOPT_ERR_PARSE;
       goto err;
     }
-    retval = config_assign(&options_format, newoptions, cl,
+    retval = config_assign(get_options_mgr(), newoptions, cl,
                            CAL_WARN_DEPRECATIONS, msg);
     config_free_lines(cl);
     if (retval < 0) {
@@ -5430,15 +5451,15 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       goto err;
     }
     if (i==0)
-      newdefaultoptions = config_dup(&options_format, newoptions);
+      newdefaultoptions = config_dup(get_options_mgr(), newoptions);
   }
 
   if (newdefaultoptions == NULL) {
-    newdefaultoptions = config_dup(&options_format, global_default_options);
+    newdefaultoptions = config_dup(get_options_mgr(), global_default_options);
   }
 
   /* Go through command-line variables too */
-  retval = config_assign(&options_format, newoptions,
+  retval = config_assign(get_options_mgr(), newoptions,
                          global_cmdline_options, CAL_WARN_DEPRECATIONS, msg);
   if (retval < 0) {
     err = SETOPT_ERR_PARSE;
@@ -5449,73 +5470,12 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   newoptions->FilesOpenedByIncludes = opened_files;
 
   /* If this is a testing network configuration, change defaults
-   * for a list of dependent config options, re-initialize newoptions
-   * with the new defaults, and assign all options to it second time. */
-  if (newoptions->TestingTorNetwork) {
-    /* XXXX this is a bit of a kludge.  perhaps there's a better way to do
-     * this?  We could, for example, make the parsing algorithm do two passes
-     * over the configuration.  If it finds any "suite" options like
-     * TestingTorNetwork, it could change the defaults before its second pass.
-     * Not urgent so long as this seems to work, but at any sign of trouble,
-     * let's clean it up.  -NM */
-
-    /* Change defaults. */
-    for (int i = 0; testing_tor_network_defaults[i].name; ++i) {
-      const config_var_t *new_var = &testing_tor_network_defaults[i];
-      config_var_t *old_var =
-          config_find_option_mutable(&options_format, new_var->name);
-      tor_assert(new_var);
-      tor_assert(old_var);
-      old_var->initvalue = new_var->initvalue;
-
-      if ((config_find_deprecation(&options_format, new_var->name))) {
-        log_warn(LD_GENERAL, "Testing options override the deprecated "
-                 "option %s. Is that intentional?",
-                 new_var->name);
-      }
-    }
-
-    /* Clear newoptions and re-initialize them with new defaults. */
-    or_options_free(newoptions);
-    or_options_free(newdefaultoptions);
-    newdefaultoptions = NULL;
-    newoptions = tor_malloc_zero(sizeof(or_options_t));
-    newoptions->magic_ = OR_OPTIONS_MAGIC;
-    options_init(newoptions);
-    newoptions->command = command;
-    newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
-
-    /* Assign all options a second time. */
-    opened_files = smartlist_new();
-    for (int i = 0; i < 2; ++i) {
-      const char *body = i==0 ? cf_defaults : cf;
-      if (!body)
-        continue;
-
-      /* get config lines, assign them */
-      retval = config_get_lines_include(body, &cl, 1,
-                                        body == cf ? &cf_has_include : NULL,
-                                        opened_files);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      retval = config_assign(&options_format, newoptions, cl, 0, msg);
-      config_free_lines(cl);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      if (i==0)
-        newdefaultoptions = config_dup(&options_format, newoptions);
-    }
-    /* Assign command-line variables a second time too */
-    retval = config_assign(&options_format, newoptions,
-                           global_cmdline_options, 0, msg);
-    if (retval < 0) {
-      err = SETOPT_ERR_PARSE;
-      goto err;
-    }
+   * for a list of dependent config options, and try this function again. */
+  if (newoptions->TestingTorNetwork && ! testing_network_configured) {
+    // retry with the testing defaults.
+    testing_network_configured = true;
+    retry = true;
+    goto err;
   }
 
   newoptions->IncludeUsed = cf_has_include;
@@ -5560,6 +5520,9 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg);
     tor_free(old_msg);
   }
+  if (retry)
+    return options_init_from_string(cf_defaults, cf, command, command_arg,
+                                    msg);
   return err;
 }
 
@@ -5785,7 +5748,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
 #else
         log_warn(LD_CONFIG, "Android logging is not supported"
                             " on this system. Sorry.");
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
         goto cleanup;
       }
     }
@@ -7838,7 +7801,7 @@ get_data_directory(const char *val)
   } else {
     return tor_strdup(get_windows_conf_root());
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *d = val;
   if (!d)
     d = "~/.tor";
@@ -8182,72 +8145,48 @@ getinfo_helper_config(control_connection_t *conn,
   (void) errmsg;
   if (!strcmp(question, "config/names")) {
     smartlist_t *sl = smartlist_new();
-    int i;
-    for (i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
-      const char *type;
-      /* don't tell controller about triple-underscore options */
-      if (!strncmp(option_vars_[i].name, "___", 3))
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+      /* don't tell controller about invisible options */
+      if (! config_var_is_listable(var))
         continue;
-      switch (var->type) {
-        case CONFIG_TYPE_STRING: type = "String"; break;
-        case CONFIG_TYPE_FILENAME: type = "Filename"; break;
-        case CONFIG_TYPE_UINT: type = "Integer"; break;
-        case CONFIG_TYPE_UINT64: type = "Integer"; break;
-        case CONFIG_TYPE_INT: type = "SignedInteger"; break;
-        case CONFIG_TYPE_PORT: type = "Port"; break;
-        case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break;
-        case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break;
-        case CONFIG_TYPE_DOUBLE: type = "Float"; break;
-        case CONFIG_TYPE_BOOL: type = "Boolean"; break;
-        case CONFIG_TYPE_AUTOBOOL: type = "Boolean+Auto"; break;
-        case CONFIG_TYPE_ISOTIME: type = "Time"; break;
-        case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break;
-        case CONFIG_TYPE_CSV: type = "CommaList"; break;
-        /* This type accepts more inputs than TimeInterval, but it ignores
-         * everything after the first entry, so we may as well pretend
-         * it's a TimeInterval. */
-        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_LINELIST: type = "LineList"; break;
-        case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break;
-        case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break;
-        default:
-        case CONFIG_TYPE_OBSOLETE:
-          type = NULL; break;
-      }
+      const char *type = struct_var_get_typename(&var->member);
       if (!type)
         continue;
-      smartlist_add_asprintf(sl, "%s %s\n",var->name,type);
-    }
+      smartlist_add_asprintf(sl, "%s %s\n",var->member.name,type);
+    } SMARTLIST_FOREACH_END(var);
     *answer = smartlist_join_strings(sl, "", 0, NULL);
     SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
     smartlist_free(sl);
+    smartlist_free(vars);
   } else if (!strcmp(question, "config/defaults")) {
     smartlist_t *sl = smartlist_new();
     int dirauth_lines_seen = 0, fallback_lines_seen = 0;
-    for (int i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
+    /* Possibly this should check whether the variables are listable,
+     * but currently it does not.  See ticket 31654. */
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
       if (var->initvalue != NULL) {
-        if (strcmp(option_vars_[i].name, "DirAuthority") == 0) {
+        if (strcmp(var->member.name, "DirAuthority") == 0) {
           /*
            * Count dirauth lines we have a default for; we'll use the
            * count later to decide whether to add the defaults manually
            */
           ++dirauth_lines_seen;
         }
-        if (strcmp(option_vars_[i].name, "FallbackDir") == 0) {
+        if (strcmp(var->member.name, "FallbackDir") == 0) {
           /*
-           * Similarly count fallback lines, so that we can decided later
+           * Similarly count fallback lines, so that we can decide later
            * to add the defaults manually.
            */
           ++fallback_lines_seen;
         }
         char *val = esc_for_log(var->initvalue);
-        smartlist_add_asprintf(sl, "%s %s\n",var->name,val);
+        smartlist_add_asprintf(sl, "%s %s\n",var->member.name,val);
         tor_free(val);
       }
-    }
+    } SMARTLIST_FOREACH_END(var);
+    smartlist_free(vars);
 
     if (dirauth_lines_seen == 0) {
       /*
@@ -8408,7 +8347,7 @@ config_load_geoip_file_(sa_family_t family,
   }
   r = geoip_load_file(family, fname, severity);
   tor_free(free_fname);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)default_fname;
   r = geoip_load_file(family, fname, severity);
 #endif /* defined(_WIN32) */
@@ -8496,7 +8435,7 @@ init_cookie_authentication(const char *fname, const char *header,
       log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname));
     }
   }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void) group_readable;
 #endif /* !defined(_WIN32) */
 
diff --git a/src/app/config/config.h b/src/app/config/config.h
index 301faf7067..44f09e5ee9 100644
--- a/src/app/config/config.h
+++ b/src/app/config/config.h
@@ -41,8 +41,6 @@ const char *escaped_safe_str_client(const char *address);
 const char *escaped_safe_str(const char *address);
 void init_protocol_warning_severity_level(void);
 int get_protocol_warning_severity_level(void);
-const char *get_version(void);
-const char *get_short_version(void);
 
 /** An error from options_trial_assign() or options_init_from_string(). */
 typedef enum setopt_err_t {
@@ -249,9 +247,8 @@ int options_any_client_port_set(const or_options_t *options);
 #define CL_PORT_DFLT_GROUP_WRITABLE (1u<<7)
 
 STATIC int options_act(const or_options_t *old_options);
-#ifdef TOR_UNIT_TESTS
-extern struct config_format_t options_format;
-#endif
+struct config_mgr_t;
+STATIC const struct config_mgr_t *get_options_mgr(void);
 
 STATIC port_cfg_t *port_cfg_new(size_t namelen);
 #define port_cfg_free(port) \
diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c
deleted file mode 100644
index efa0c19fa6..0000000000
--- a/src/app/config/confparse.c
+++ /dev/null
@@ -1,1207 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.c
- *
- * \brief Back-end for parsing and generating key-value files, used to
- *   implement the torrc file format and the state file.
- *
- * This module is used by config.c to parse and encode torrc
- * configuration files, and by statefile.c to parse and encode the
- * $DATADIR/state file.
- *
- * To use this module, its callers provide an instance of
- * config_format_t to describe the mappings from a set of configuration
- * options to a number of fields in a C structure.  With this mapping,
- * the functions here can convert back and forth between the C structure
- * specified, and a linked list of key-value pairs.
- */
-
-#include "core/or/or.h"
-#include "app/config/confparse.h"
-#include "feature/nodelist/routerset.h"
-
-#include "lib/container/bitarray.h"
-#include "lib/encoding/confline.h"
-
-static uint64_t config_parse_memunit(const char *s, int *ok);
-static int config_parse_msec_interval(const char *s, int *ok);
-static int config_parse_interval(const char *s, int *ok);
-static void config_reset(const config_format_t *fmt, void *options,
-                         const config_var_t *var, int use_defaults);
-
-/** Allocate an empty configuration object of a given format type. */
-void *
-config_new(const config_format_t *fmt)
-{
-  void *opts = tor_malloc_zero(fmt->size);
-  *(uint32_t*)STRUCT_VAR_P(opts, fmt->magic_offset) = fmt->magic;
-  CONFIG_CHECK(fmt, opts);
-  return opts;
-}
-
-/*
- * Functions to parse config options
- */
-
-/** If <b>option</b> is an official abbreviation for a longer option,
- * return the longer option.  Otherwise return <b>option</b>.
- * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
- * apply abbreviations that work for the config file and the command line.
- * If <b>warn_obsolete</b> is set, warn about deprecated names. */
-const char *
-config_expand_abbrev(const config_format_t *fmt, const char *option,
-                     int command_line, int warn_obsolete)
-{
-  int i;
-  if (! fmt->abbrevs)
-    return option;
-  for (i=0; fmt->abbrevs[i].abbreviated; ++i) {
-    /* Abbreviations are case insensitive. */
-    if (!strcasecmp(option,fmt->abbrevs[i].abbreviated) &&
-        (command_line || !fmt->abbrevs[i].commandline_only)) {
-      if (warn_obsolete && fmt->abbrevs[i].warn) {
-        log_warn(LD_CONFIG,
-                 "The configuration option '%s' is deprecated; "
-                 "use '%s' instead.",
-                 fmt->abbrevs[i].abbreviated,
-                 fmt->abbrevs[i].full);
-      }
-      /* Keep going through the list in case we want to rewrite it more.
-       * (We could imagine recursing here, but I don't want to get the
-       * user into an infinite loop if we craft our list wrong.) */
-      option = fmt->abbrevs[i].full;
-    }
-  }
-  return option;
-}
-
-/** If <b>key</b> is a deprecated configuration option, return the message
- * explaining why it is deprecated (which may be an empty string). Return NULL
- * if it is not deprecated. The <b>key</b> field must be fully expanded. */
-const char *
-config_find_deprecation(const config_format_t *fmt, const char *key)
-{
-  if (BUG(fmt == NULL) || BUG(key == NULL))
-    return NULL;
-  if (fmt->deprecations == NULL)
-    return NULL;
-
-  const config_deprecation_t *d;
-  for (d = fmt->deprecations; d->name; ++d) {
-    if (!strcasecmp(d->name, key)) {
-      return d->why_deprecated ? d->why_deprecated : "";
-    }
-  }
-  return NULL;
-}
-
-/** As config_find_option, but return a non-const pointer. */
-config_var_t *
-config_find_option_mutable(config_format_t *fmt, const char *key)
-{
-  int i;
-  size_t keylen = strlen(key);
-  if (!keylen)
-    return NULL; /* if they say "--" on the command line, it's not an option */
-  /* First, check for an exact (case-insensitive) match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strcasecmp(key, fmt->vars[i].name)) {
-      return &fmt->vars[i];
-    }
-  }
-  /* If none, check for an abbreviated match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strncasecmp(key, fmt->vars[i].name, keylen)) {
-      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
-               "Please use '%s' instead",
-               key, fmt->vars[i].name);
-      return &fmt->vars[i];
-    }
-  }
-  /* Okay, unrecognized option */
-  return NULL;
-}
-
-/** If <b>key</b> is a configuration option, return the corresponding const
- * config_var_t.  Otherwise, if <b>key</b> is a non-standard abbreviation,
- * warn, and return the corresponding const config_var_t.  Otherwise return
- * NULL.
- */
-const config_var_t *
-config_find_option(const config_format_t *fmt, const char *key)
-{
-  return config_find_option_mutable((config_format_t*)fmt, key);
-}
-
-/** Return the number of option entries in <b>fmt</b>. */
-static int
-config_count_options(const config_format_t *fmt)
-{
-  int i;
-  for (i=0; fmt->vars[i].name; ++i)
-    ;
-  return i;
-}
-
-/*
- * Functions to assign config options.
- */
-
-/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
- * with <b>c</b>-\>value and return 0, or return -1 if bad value.
- *
- * Called from config_assign_line() and option_reset().
- */
-static int
-config_assign_value(const config_format_t *fmt, void *options,
-                    config_line_t *c, char **msg)
-{
-  int i, ok;
-  const config_var_t *var;
-  void *lvalue;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  tor_assert(var);
-
-  lvalue = STRUCT_VAR_P(options, var->var_offset);
-
-  switch (var->type) {
-
-  case CONFIG_TYPE_PORT:
-    if (!strcasecmp(c->value, "auto")) {
-      *(int *)lvalue = CFG_AUTO_PORT;
-      break;
-    }
-    FALLTHROUGH;
-  case CONFIG_TYPE_INT:
-  case CONFIG_TYPE_UINT:
-    i = (int)tor_parse_long(c->value, 10,
-                            var->type==CONFIG_TYPE_INT ? INT_MIN : 0,
-                            var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX,
-                            &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Int keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_UINT64: {
-    uint64_t u64 = tor_parse_uint64(c->value, 10,
-                                    0, UINT64_MAX, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "uint64 keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_CSV_INTERVAL: {
-    /* We used to have entire smartlists here.  But now that all of our
-     * download schedules use exponential backoff, only the first part
-     * matters. */
-    const char *comma = strchr(c->value, ',');
-    const char *val = c->value;
-    char *tmp = NULL;
-    if (comma) {
-      tmp = tor_strndup(c->value, comma - c->value);
-      val = tmp;
-    }
-
-    i = config_parse_interval(val, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      tor_free(tmp);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    tor_free(tmp);
-    break;
-  }
-
-  case CONFIG_TYPE_INTERVAL: {
-    i = config_parse_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MSEC_INTERVAL: {
-    i = config_parse_msec_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Msec interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MEMUNIT: {
-    uint64_t u64 = config_parse_memunit(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Value '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_BOOL:
-    i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Boolean '%s %s' expects 0 or 1.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_AUTOBOOL:
-    if (!strcasecmp(c->value, "auto"))
-      *(int *)lvalue = -1;
-    else if (!strcmp(c->value, "0"))
-      *(int *)lvalue = 0;
-    else if (!strcmp(c->value, "1"))
-      *(int *)lvalue = 1;
-    else {
-      tor_asprintf(msg, "Boolean '%s %s' expects 0, 1, or 'auto'.",
-                   c->key, c->value);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_STRING:
-  case CONFIG_TYPE_FILENAME:
-    tor_free(*(char **)lvalue);
-    *(char **)lvalue = tor_strdup(c->value);
-    break;
-
-  case CONFIG_TYPE_DOUBLE:
-    *(double *)lvalue = atof(c->value);
-    break;
-
-  case CONFIG_TYPE_ISOTIME:
-    if (parse_iso_time(c->value, (time_t *)lvalue)) {
-      tor_asprintf(msg,
-          "Invalid time '%s' for keyword '%s'", c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_ROUTERSET:
-    if (*(routerset_t**)lvalue) {
-      routerset_free(*(routerset_t**)lvalue);
-    }
-    *(routerset_t**)lvalue = routerset_new();
-    if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) {
-      tor_asprintf(msg, "Invalid exit list '%s' for option '%s'",
-                   c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_CSV:
-    if (*(smartlist_t**)lvalue) {
-      SMARTLIST_FOREACH(*(smartlist_t**)lvalue, char *, cp, tor_free(cp));
-      smartlist_clear(*(smartlist_t**)lvalue);
-    } else {
-      *(smartlist_t**)lvalue = smartlist_new();
-    }
-
-    smartlist_split_string(*(smartlist_t**)lvalue, c->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    break;
-
-  case CONFIG_TYPE_LINELIST:
-  case CONFIG_TYPE_LINELIST_S:
-    {
-      config_line_t *lastval = *(config_line_t**)lvalue;
-      if (lastval && lastval->fragile) {
-        if (c->command != CONFIG_LINE_APPEND) {
-          config_free_lines(lastval);
-          *(config_line_t**)lvalue = NULL;
-        } else {
-          lastval->fragile = 0;
-        }
-      }
-
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-    }
-    break;
-  case CONFIG_TYPE_OBSOLETE:
-    log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key);
-    break;
-  case CONFIG_TYPE_LINELIST_V:
-    tor_asprintf(msg,
-        "You may not provide a value for virtual option '%s'", c->key);
-    return -1;
-  default:
-    tor_assert(0);
-    break;
-  }
-  return 0;
-}
-
-/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
- * to it will replace old ones. */
-static void
-config_mark_lists_fragile(const config_format_t *fmt, void *options)
-{
-  int i;
-  tor_assert(fmt);
-  tor_assert(options);
-
-  for (i = 0; fmt->vars[i].name; ++i) {
-    const config_var_t *var = &fmt->vars[i];
-    config_line_t *list;
-    if (var->type != CONFIG_TYPE_LINELIST &&
-        var->type != CONFIG_TYPE_LINELIST_V)
-      continue;
-
-    list = *(config_line_t **)STRUCT_VAR_P(options, var->var_offset);
-    if (list)
-      list->fragile = 1;
-  }
-}
-
-void
-warn_deprecated_option(const char *what, const char *why)
-{
-  const char *space = (why && strlen(why)) ? " " : "";
-  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
-           "be removed in a future version of Tor.%s%s (If you think this is "
-           "a mistake, please let us know!)",
-           what, space, why);
-}
-
-/** If <b>c</b> is a syntactically valid configuration line, update
- * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
- * key, -2 for bad value.
- *
- * If <b>clear_first</b> is set, clear the value first. Then if
- * <b>use_defaults</b> is set, set the value to the default.
- *
- * Called from config_assign().
- */
-static int
-config_assign_line(const config_format_t *fmt, void *options,
-                   config_line_t *c, unsigned flags,
-                   bitarray_t *options_seen, char **msg)
-{
-  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
-  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
-  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  if (!var) {
-    if (fmt->extra) {
-      void *lvalue = STRUCT_VAR_P(options, fmt->extra->var_offset);
-      log_info(LD_CONFIG,
-               "Found unrecognized option '%s'; saving it.", c->key);
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-      return 0;
-    } else {
-      tor_asprintf(msg,
-                "Unknown option '%s'.  Failing.", c->key);
-      return -1;
-    }
-  }
-
-  /* Put keyword into canonical case. */
-  if (strcmp(var->name, c->key)) {
-    tor_free(c->key);
-    c->key = tor_strdup(var->name);
-  }
-
-  const char *deprecation_msg;
-  if (warn_deprecations &&
-      (deprecation_msg = config_find_deprecation(fmt, var->name))) {
-    warn_deprecated_option(var->name, deprecation_msg);
-  }
-
-  if (!strlen(c->value)) {
-    /* reset or clear it, then return */
-    if (!clear_first) {
-      if ((var->type == CONFIG_TYPE_LINELIST ||
-           var->type == CONFIG_TYPE_LINELIST_S) &&
-          c->command != CONFIG_LINE_CLEAR) {
-        /* We got an empty linelist from the torrc or command line.
-           As a special case, call this an error. Warn and ignore. */
-        log_warn(LD_CONFIG,
-                 "Linelist option '%s' has no value. Skipping.", c->key);
-      } else { /* not already cleared */
-        config_reset(fmt, options, var, use_defaults);
-      }
-    }
-    return 0;
-  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
-    config_reset(fmt, options, var, use_defaults);
-  }
-
-  if (options_seen && (var->type != CONFIG_TYPE_LINELIST &&
-                       var->type != CONFIG_TYPE_LINELIST_S)) {
-    /* We're tracking which options we've seen, and this option is not
-     * supposed to occur more than once. */
-    int var_index = (int)(var - fmt->vars);
-    if (bitarray_is_set(options_seen, var_index)) {
-      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
-               "value will be ignored.", var->name);
-    }
-    bitarray_set(options_seen, var_index);
-  }
-
-  if (config_assign_value(fmt, options, c, msg) < 0)
-    return -2;
-  return 0;
-}
-
-/** Restore the option named <b>key</b> in options to its default value.
- * Called from config_assign(). */
-static void
-config_reset_line(const config_format_t *fmt, void *options,
-                  const char *key, int use_defaults)
-{
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var)
-    return; /* give error on next pass. */
-
-  config_reset(fmt, options, var, use_defaults);
-}
-
-/** Return true iff value needs to be quoted and escaped to be used in
- * a configuration file. */
-static int
-config_value_needs_escape(const char *value)
-{
-  if (*value == '\"')
-    return 1;
-  while (*value) {
-    switch (*value)
-    {
-    case '\r':
-    case '\n':
-    case '#':
-      /* Note: quotes and backspaces need special handling when we are using
-       * quotes, not otherwise, so they don't trigger escaping on their
-       * own. */
-      return 1;
-    default:
-      if (!TOR_ISPRINT(*value))
-        return 1;
-    }
-    ++value;
-  }
-  return 0;
-}
-
-/** Return newly allocated line or lines corresponding to <b>key</b> in the
- * configuration <b>options</b>.  If <b>escape_val</b> is true and a
- * value needs to be quoted before it's put in a config file, quote and
- * escape that value. Return NULL if no such key exists. */
-config_line_t *
-config_get_assigned_option(const config_format_t *fmt, const void *options,
-                           const char *key, int escape_val)
-{
-  const config_var_t *var;
-  const void *value;
-  config_line_t *result;
-  tor_assert(options && key);
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var) {
-    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
-    return NULL;
-  }
-  value = STRUCT_VAR_P(options, var->var_offset);
-
-  result = tor_malloc_zero(sizeof(config_line_t));
-  result->key = tor_strdup(var->name);
-  switch (var->type)
-    {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      if (*(char**)value) {
-        result->value = tor_strdup(*(char**)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-        return NULL;
-      }
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      if (*(time_t*)value) {
-        result->value = tor_malloc(ISO_TIME_LEN+1);
-        format_iso_time(result->value, *(time_t*)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-      }
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_PORT:
-      if (*(int*)value == CFG_AUTO_PORT) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-      /* This means every or_options_t uint or bool element
-       * needs to be an int. Not, say, a uint16_t or char. */
-      tor_asprintf(&result->value, "%d", *(int*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_UINT64: FALLTHROUGH;
-    case CONFIG_TYPE_MEMUNIT:
-      tor_asprintf(&result->value, "%"PRIu64,
-                   (*(uint64_t*)value));
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      tor_asprintf(&result->value, "%f", *(double*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-
-    case CONFIG_TYPE_AUTOBOOL:
-      if (*(int*)value == -1) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_BOOL:
-      result->value = tor_strdup(*(int*)value ? "1" : "0");
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      result->value = routerset_to_string(*(routerset_t**)value);
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)value)
-        result->value =
-          smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
-      else
-        result->value = tor_strdup("");
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      log_fn(LOG_INFO, LD_CONFIG,
-             "You asked me for the value of an obsolete config option '%s'.",
-             key);
-      tor_free(result->key);
-      tor_free(result);
-      return NULL;
-    case CONFIG_TYPE_LINELIST_S:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup_and_filter(*(const config_line_t **)value,
-                                           key);
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_V:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup(*(const config_line_t**)value);
-      break;
-    default:
-      tor_free(result->key);
-      tor_free(result);
-      log_warn(LD_BUG,"Unknown type %d for known key '%s'",
-               var->type, key);
-      return NULL;
-    }
-
-  if (escape_val) {
-    config_line_t *line;
-    for (line = result; line; line = line->next) {
-      if (line->value && config_value_needs_escape(line->value)) {
-        char *newval = esc_for_log(line->value);
-        tor_free(line->value);
-        line->value = newval;
-      }
-    }
-  }
-
-  return result;
-}
-/** Iterate through the linked list of requested options <b>list</b>.
- * For each item, convert as appropriate and assign to <b>options</b>.
- * If an item is unrecognized, set *msg and return -1 immediately,
- * else return 0 for success.
- *
- * If <b>clear_first</b>, interpret config options as replacing (not
- * extending) their previous values. If <b>clear_first</b> is set,
- * then <b>use_defaults</b> to decide if you set to defaults after
- * clearing, or make the value 0 or NULL.
- *
- * Here are the use cases:
- * 1. A non-empty AllowInvalid line in your torrc. Appends to current
- *    if linelist, replaces current if csv.
- * 2. An empty AllowInvalid line in your torrc. Should clear it.
- * 3. "RESETCONF AllowInvalid" sets it to default.
- * 4. "SETCONF AllowInvalid" makes it NULL.
- * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
- *
- * Use_defaults   Clear_first
- *    0                0       "append"
- *    1                0       undefined, don't use
- *    0                1       "set to null first"
- *    1                1       "set to defaults first"
- * Return 0 on success, -1 on bad key, -2 on bad value.
- *
- * As an additional special case, if a LINELIST config option has
- * no value and clear_first is 0, then warn and ignore it.
- */
-
-/*
-There are three call cases for config_assign() currently.
-
-Case one: Torrc entry
-options_init_from_torrc() calls config_assign(0, 0)
-  calls config_assign_line(0, 0).
-    if value is empty, calls config_reset(0) and returns.
-    calls config_assign_value(), appends.
-
-Case two: setconf
-options_trial_assign() calls config_assign(0, 1)
-  calls config_reset_line(0)
-    calls config_reset(0)
-      calls option_clear().
-  calls config_assign_line(0, 1).
-    if value is empty, returns.
-    calls config_assign_value(), appends.
-
-Case three: resetconf
-options_trial_assign() calls config_assign(1, 1)
-  calls config_reset_line(1)
-    calls config_reset(1)
-      calls option_clear().
-      calls config_assign_value(default)
-  calls config_assign_line(1, 1).
-    returns.
-*/
-int
-config_assign(const config_format_t *fmt, void *options, config_line_t *list,
-              unsigned config_assign_flags, char **msg)
-{
-  config_line_t *p;
-  bitarray_t *options_seen;
-  const int n_options = config_count_options(fmt);
-  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
-  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
-
-  CONFIG_CHECK(fmt, options);
-
-  /* pass 1: normalize keys */
-  for (p = list; p; p = p->next) {
-    const char *full = config_expand_abbrev(fmt, p->key, 0, 1);
-    if (strcmp(full,p->key)) {
-      tor_free(p->key);
-      p->key = tor_strdup(full);
-    }
-  }
-
-  /* pass 2: if we're reading from a resetting source, clear all
-   * mentioned config options, and maybe set to their defaults. */
-  if (clear_first) {
-    for (p = list; p; p = p->next)
-      config_reset_line(fmt, options, p->key, use_defaults);
-  }
-
-  options_seen = bitarray_init_zero(n_options);
-  /* pass 3: assign. */
-  while (list) {
-    int r;
-    if ((r=config_assign_line(fmt, options, list, config_assign_flags,
-                              options_seen, msg))) {
-      bitarray_free(options_seen);
-      return r;
-    }
-    list = list->next;
-  }
-  bitarray_free(options_seen);
-
-  /** Now we're done assigning a group of options to the configuration.
-   * Subsequent group assignments should _replace_ linelists, not extend
-   * them. */
-  config_mark_lists_fragile(fmt, options);
-
-  return 0;
-}
-
-/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
- * Called from config_reset() and config_free(). */
-static void
-config_clear(const config_format_t *fmt, void *options,
-             const config_var_t *var)
-{
-  void *lvalue = STRUCT_VAR_P(options, var->var_offset);
-  (void)fmt; /* unused */
-  switch (var->type) {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      tor_free(*(char**)lvalue);
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      *(double*)lvalue = 0.0;
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      *(time_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-    case CONFIG_TYPE_PORT:
-    case CONFIG_TYPE_BOOL:
-      *(int*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_AUTOBOOL:
-      *(int*)lvalue = -1;
-      break;
-    case CONFIG_TYPE_UINT64:
-    case CONFIG_TYPE_MEMUNIT:
-      *(uint64_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      if (*(routerset_t**)lvalue) {
-        routerset_free(*(routerset_t**)lvalue);
-        *(routerset_t**)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)lvalue) {
-        SMARTLIST_FOREACH(*(smartlist_t **)lvalue, char *, cp, tor_free(cp));
-        smartlist_free(*(smartlist_t **)lvalue);
-        *(smartlist_t **)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_S:
-      config_free_lines(*(config_line_t **)lvalue);
-      *(config_line_t **)lvalue = NULL;
-      break;
-    case CONFIG_TYPE_LINELIST_V:
-      /* handled by linelist_s. */
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      break;
-  }
-}
-
-/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
- * <b>use_defaults</b>, set it to its default value.
- * Called by config_init() and option_reset_line() and option_assign_line(). */
-static void
-config_reset(const config_format_t *fmt, void *options,
-             const config_var_t *var, int use_defaults)
-{
-  config_line_t *c;
-  char *msg = NULL;
-  CONFIG_CHECK(fmt, options);
-  config_clear(fmt, options, var); /* clear it first */
-  if (!use_defaults)
-    return; /* all done */
-  if (var->initvalue) {
-    c = tor_malloc_zero(sizeof(config_line_t));
-    c->key = tor_strdup(var->name);
-    c->value = tor_strdup(var->initvalue);
-    if (config_assign_value(fmt, options, c, &msg) < 0) {
-      log_warn(LD_BUG, "Failed to assign default: %s", msg);
-      tor_free(msg); /* if this happens it's a bug */
-    }
-    config_free_lines(c);
-  }
-}
-
-/** Release storage held by <b>options</b>. */
-void
-config_free_(const config_format_t *fmt, void *options)
-{
-  int i;
-
-  if (!options)
-    return;
-
-  tor_assert(fmt);
-
-  for (i=0; fmt->vars[i].name; ++i)
-    config_clear(fmt, options, &(fmt->vars[i]));
-  if (fmt->extra) {
-    config_line_t **linep = STRUCT_VAR_P(options, fmt->extra->var_offset);
-    config_free_lines(*linep);
-    *linep = NULL;
-  }
-  tor_free(options);
-}
-
-/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
- * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
- */
-int
-config_is_same(const config_format_t *fmt,
-               const void *o1, const void *o2,
-               const char *name)
-{
-  config_line_t *c1, *c2;
-  int r = 1;
-  CONFIG_CHECK(fmt, o1);
-  CONFIG_CHECK(fmt, o2);
-
-  c1 = config_get_assigned_option(fmt, o1, name, 0);
-  c2 = config_get_assigned_option(fmt, o2, name, 0);
-  r = config_lines_eq(c1, c2);
-  config_free_lines(c1);
-  config_free_lines(c2);
-  return r;
-}
-
-/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
-void *
-config_dup(const config_format_t *fmt, const void *old)
-{
-  void *newopts;
-  int i;
-  config_line_t *line;
-
-  newopts = config_new(fmt);
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE)
-      continue;
-    line = config_get_assigned_option(fmt, old, fmt->vars[i].name, 0);
-    if (line) {
-      char *msg = NULL;
-      if (config_assign(fmt, newopts, line, 0, &msg) < 0) {
-        log_err(LD_BUG, "config_get_assigned_option() generated "
-                "something we couldn't config_assign(): %s", msg);
-        tor_free(msg);
-        tor_assert(0);
-      }
-    }
-    config_free_lines(line);
-  }
-  return newopts;
-}
-/** Set all vars in the configuration object <b>options</b> to their default
- * values. */
-void
-config_init(const config_format_t *fmt, void *options)
-{
-  int i;
-  const config_var_t *var;
-  CONFIG_CHECK(fmt, options);
-
-  for (i=0; fmt->vars[i].name; ++i) {
-    var = &fmt->vars[i];
-    if (!var->initvalue)
-      continue; /* defaults to NULL or 0 */
-    config_reset(fmt, options, var, 1);
-  }
-}
-
-/** Allocate and return a new string holding the written-out values of the vars
- * in 'options'.  If 'minimal', do not write out any default-valued vars.
- * Else, if comment_defaults, write default values as comments.
- */
-char *
-config_dump(const config_format_t *fmt, const void *default_options,
-            const void *options, int minimal,
-            int comment_defaults)
-{
-  smartlist_t *elements;
-  const void *defaults = default_options;
-  void *defaults_tmp = NULL;
-  config_line_t *line, *assigned;
-  char *result;
-  int i;
-  char *msg = NULL;
-
-  if (defaults == NULL) {
-    defaults = defaults_tmp = config_new(fmt);
-    config_init(fmt, defaults_tmp);
-  }
-
-  /* XXX use a 1 here so we don't add a new log line while dumping */
-  if (default_options == NULL) {
-    if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) {
-      log_err(LD_BUG, "Failed to validate default config: %s", msg);
-      tor_free(msg);
-      tor_assert(0);
-    }
-  }
-
-  elements = smartlist_new();
-  for (i=0; fmt->vars[i].name; ++i) {
-    int comment_option = 0;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE ||
-        fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    /* Don't save 'hidden' control variables. */
-    if (!strcmpstart(fmt->vars[i].name, "__"))
-      continue;
-    if (minimal && config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      continue;
-    else if (comment_defaults &&
-             config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      comment_option = 1;
-
-    line = assigned =
-      config_get_assigned_option(fmt, options, fmt->vars[i].name, 1);
-
-    for (; line; line = line->next) {
-      if (!strcmpstart(line->key, "__")) {
-        /* This check detects "hidden" variables inside LINELIST_V structures.
-         */
-        continue;
-      }
-      smartlist_add_asprintf(elements, "%s%s %s\n",
-                   comment_option ? "# " : "",
-                   line->key, line->value);
-    }
-    config_free_lines(assigned);
-  }
-
-  if (fmt->extra) {
-    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset);
-    for (; line; line = line->next) {
-      smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value);
-    }
-  }
-
-  result = smartlist_join_strings(elements, "", 0, NULL);
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-  if (defaults_tmp) {
-    fmt->free_fn(defaults_tmp);
-  }
-  return result;
-}
-
-/** Mapping from a unit name to a multiplier for converting that unit into a
- * base unit.  Used by config_parse_unit. */
-struct unit_table_t {
-  const char *unit; /**< The name of the unit */
-  uint64_t multiplier; /**< How many of the base unit appear in this unit */
-};
-
-/** Table to map the names of memory units to the number of bytes they
- * contain. */
-static struct unit_table_t memory_units[] = {
-  { "",          1 },
-  { "b",         1<< 0 },
-  { "byte",      1<< 0 },
-  { "bytes",     1<< 0 },
-  { "kb",        1<<10 },
-  { "kbyte",     1<<10 },
-  { "kbytes",    1<<10 },
-  { "kilobyte",  1<<10 },
-  { "kilobytes", 1<<10 },
-  { "kilobits",  1<<7  },
-  { "kilobit",   1<<7  },
-  { "kbits",     1<<7  },
-  { "kbit",      1<<7  },
-  { "m",         1<<20 },
-  { "mb",        1<<20 },
-  { "mbyte",     1<<20 },
-  { "mbytes",    1<<20 },
-  { "megabyte",  1<<20 },
-  { "megabytes", 1<<20 },
-  { "megabits",  1<<17 },
-  { "megabit",   1<<17 },
-  { "mbits",     1<<17 },
-  { "mbit",      1<<17 },
-  { "gb",        1<<30 },
-  { "gbyte",     1<<30 },
-  { "gbytes",    1<<30 },
-  { "gigabyte",  1<<30 },
-  { "gigabytes", 1<<30 },
-  { "gigabits",  1<<27 },
-  { "gigabit",   1<<27 },
-  { "gbits",     1<<27 },
-  { "gbit",      1<<27 },
-  { "tb",        UINT64_C(1)<<40 },
-  { "tbyte",     UINT64_C(1)<<40 },
-  { "tbytes",    UINT64_C(1)<<40 },
-  { "terabyte",  UINT64_C(1)<<40 },
-  { "terabytes", UINT64_C(1)<<40 },
-  { "terabits",  UINT64_C(1)<<37 },
-  { "terabit",   UINT64_C(1)<<37 },
-  { "tbits",     UINT64_C(1)<<37 },
-  { "tbit",      UINT64_C(1)<<37 },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of seconds they
- * contain. */
-static struct unit_table_t time_units[] = {
-  { "",         1 },
-  { "second",   1 },
-  { "seconds",  1 },
-  { "minute",   60 },
-  { "minutes",  60 },
-  { "hour",     60*60 },
-  { "hours",    60*60 },
-  { "day",      24*60*60 },
-  { "days",     24*60*60 },
-  { "week",     7*24*60*60 },
-  { "weeks",    7*24*60*60 },
-  { "month",    2629728, }, /* about 30.437 days */
-  { "months",   2629728, },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of milliseconds
- * they contain. */
-static struct unit_table_t time_msec_units[] = {
-  { "",         1 },
-  { "msec",     1 },
-  { "millisecond", 1 },
-  { "milliseconds", 1 },
-  { "second",   1000 },
-  { "seconds",  1000 },
-  { "minute",   60*1000 },
-  { "minutes",  60*1000 },
-  { "hour",     60*60*1000 },
-  { "hours",    60*60*1000 },
-  { "day",      24*60*60*1000 },
-  { "days",     24*60*60*1000 },
-  { "week",     7*24*60*60*1000 },
-  { "weeks",    7*24*60*60*1000 },
-  { NULL, 0 },
-};
-
-/** Parse a string <b>val</b> containing a number, zero or more
- * spaces, and an optional unit string.  If the unit appears in the
- * table <b>u</b>, then multiply the number by the unit multiplier.
- * On success, set *<b>ok</b> to 1 and return this product.
- * Otherwise, set *<b>ok</b> to 0.
- */
-static uint64_t
-config_parse_units(const char *val, struct unit_table_t *u, int *ok)
-{
-  uint64_t v = 0;
-  double d = 0;
-  int use_float = 0;
-  char *cp;
-
-  tor_assert(ok);
-
-  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
-  if (!*ok || (cp && *cp == '.')) {
-    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
-    if (!*ok)
-      goto done;
-    use_float = 1;
-  }
-
-  if (!cp) {
-    *ok = 1;
-    v = use_float ? ((uint64_t)d) :  v;
-    goto done;
-  }
-
-  cp = (char*) eat_whitespace(cp);
-
-  for ( ;u->unit;++u) {
-    if (!strcasecmp(u->unit, cp)) {
-      if (use_float)
-        v = (uint64_t)(u->multiplier * d);
-      else
-        v *= u->multiplier;
-      *ok = 1;
-      goto done;
-    }
-  }
-  log_warn(LD_CONFIG, "Unknown unit '%s'.", cp);
-  *ok = 0;
- done:
-
-  if (*ok)
-    return v;
-  else
-    return 0;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
- * and return the number of bytes specified.  Otherwise, set
- * *<b>ok</b> to false and return 0. */
-static uint64_t
-config_parse_memunit(const char *s, int *ok)
-{
-  uint64_t u = config_parse_units(s, memory_units, ok);
-  return u;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * time in milliseconds.  On success, set *<b>ok</b> to true and return
- * the number of milliseconds in the provided interval.  Otherwise, set
- * *<b>ok</b> to 0 and return -1. */
-static int
-config_parse_msec_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_msec_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of time.
- * On success, set *<b>ok</b> to true and return the number of seconds in
- * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
- */
-static int
-config_parse_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h
deleted file mode 100644
index 57f1ec1762..0000000000
--- a/src/app/config/confparse.h
+++ /dev/null
@@ -1,233 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.h
- *
- * \brief Header for confparse.c.
- */
-
-#ifndef TOR_CONFPARSE_H
-#define TOR_CONFPARSE_H
-
-/** Enumeration of types which option values can take */
-typedef enum config_type_t {
-  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
-  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
-  CONFIG_TYPE_UINT,         /**< A non-negative integer less than MAX_INT */
-  CONFIG_TYPE_INT,          /**< Any integer. */
-  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
-  CONFIG_TYPE_PORT,         /**< A port from 1...65535, 0 for "not set", or
-                             * "auto".  */
-  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
-  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
-                              * units */
-  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
-  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
-  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
-  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
-                             * 1 for true, and -1 for auto  */
-  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
-  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
-                              * optional whitespace. */
-  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
-                              * optional whitespace, representing intervals in
-                              * seconds, with optional units.  We allow
-                              * multiple values here for legacy reasons, but
-                              * ignore every value after the first. */
-  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
-  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
-                             * mixed with other keywords. */
-  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
-                             * context-sensitive config lines when fetching.
-                             */
-  CONFIG_TYPE_ROUTERSET,    /**< A list of router names, addrs, and fps,
-                             * parsed into a routerset_t. */
-  CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
-} config_type_t;
-
-#ifdef TOR_UNIT_TESTS
-/**
- * Union used when building in test mode typechecking the members of a type
- * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
- * it is used. */
-typedef union {
-  char **STRING;
-  char **FILENAME;
-  int *UINT; /* yes, really: Even though the confparse type is called
-              * "UINT", it still uses the C int type -- it just enforces that
-              * the values are in range [0,INT_MAX].
-              */
-  uint64_t *UINT64;
-  int *INT;
-  int *PORT;
-  int *INTERVAL;
-  int *MSEC_INTERVAL;
-  uint64_t *MEMUNIT;
-  double *DOUBLE;
-  int *BOOL;
-  int *AUTOBOOL;
-  time_t *ISOTIME;
-  smartlist_t **CSV;
-  int *CSV_INTERVAL;
-  struct config_line_t **LINELIST;
-  struct config_line_t **LINELIST_S;
-  struct config_line_t **LINELIST_V;
-  routerset_t **ROUTERSET;
-} confparse_dummy_values_t;
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** An abbreviation for a configuration option allowed on the command line. */
-typedef struct config_abbrev_t {
-  const char *abbreviated;
-  const char *full;
-  int commandline_only;
-  int warn;
-} config_abbrev_t;
-
-typedef struct config_deprecation_t {
-  const char *name;
-  const char *why_deprecated;
-} config_deprecation_t;
-
-/* Handy macro for declaring "In the config file or on the command line,
- * you can abbreviate <b>tok</b>s as <b>tok</b>". */
-#define PLURAL(tok) { #tok, #tok "s", 0, 0 }
-
-/** A variable allowed in the configuration file or on the command line. */
-typedef struct config_var_t {
-  const char *name; /**< The full keyword (case insensitive). */
-  config_type_t type; /**< How to interpret the type and turn it into a
-                       * value. */
-  off_t var_offset; /**< Offset of the corresponding member of or_options_t. */
-  const char *initvalue; /**< String (or null) describing initial value. */
-
-#ifdef TOR_UNIT_TESTS
-  /** Used for compiler-magic to typecheck the corresponding field in the
-   * corresponding struct. Only used in unit test mode, at compile-time. */
-  confparse_dummy_values_t var_ptr_dummy;
-#endif
-} config_var_t;
-
-/* Macros to define extra members inside config_var_t fields, and at the
- * end of a list of them.
- */
-#ifdef TOR_UNIT_TESTS
-/* This is a somewhat magic type-checking macro for users of confparse.c.
- * It initializes a union member "confparse_dummy_values_t.conftype" with
- * the address of a static member "tp_dummy.member".   This
- * will give a compiler warning unless the member field is of the correct
- * type.
- *
- * (This warning is mandatory, because a type mismatch here violates the type
- * compatibility constraint for simple assignment, and requires a diagnostic,
- * according to the C spec.)
- *
- * For example, suppose you say:
- *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
- * Then this macro will evaluate to:
- *     { .STRING = &or_options_t_dummy.Address }
- * And since confparse_dummy_values_t.STRING has type "char **", that
- * expression will create a warning unless or_options_t.Address also
- * has type "char *".
- */
-#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
-  { . conftype = &tp ## _dummy . member }
-#define CONF_TEST_MEMBERS(tp, conftype, member) \
-  , CONF_CHECK_VAR_TYPE(tp, conftype, member)
-#define END_OF_CONFIG_VARS                                      \
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } }
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  static tp tp ## _dummy
-#else /* !(defined(TOR_UNIT_TESTS)) */
-#define CONF_TEST_MEMBERS(tp, conftype, member)
-#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
-/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  struct tor_semicolon_eater
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Type of a callback to validate whether a given configuration is
- * well-formed and consistent. See options_trial_assign() for documentation
- * of arguments. */
-typedef int (*validate_fn_t)(void*,void*,void*,int,char**);
-
-/** Callback to free a configuration object. */
-typedef void (*free_cfg_fn_t)(void*);
-
-/** Information on the keys, value types, key-to-struct-member mappings,
- * variable descriptions, validation functions, and abbreviations for a
- * configuration or storage format. */
-typedef struct config_format_t {
-  size_t size; /**< Size of the struct that everything gets parsed into. */
-  uint32_t magic; /**< Required 'magic value' to make sure we have a struct
-                   * of the right type. */
-  off_t magic_offset; /**< Offset of the magic value within the struct. */
-  config_abbrev_t *abbrevs; /**< List of abbreviations that we expand when
-                             * parsing this format. */
-  const config_deprecation_t *deprecations; /** List of deprecated options */
-  config_var_t *vars; /**< List of variables we recognize, their default
-                       * values, and where we stick them in the structure. */
-  validate_fn_t validate_fn; /**< Function to validate config. */
-  free_cfg_fn_t free_fn; /**< Function to free the configuration. */
-  /** If present, extra is a LINELIST variable for unrecognized
-   * lines.  Otherwise, unrecognized lines are an error. */
-  config_var_t *extra;
-} config_format_t;
-
-/** Macro: assert that <b>cfg</b> has the right magic field for format
- * <b>fmt</b>. */
-#define CONFIG_CHECK(fmt, cfg) STMT_BEGIN                               \
-    tor_assert(fmt && cfg);                                             \
-    tor_assert((fmt)->magic ==                                          \
-               *(uint32_t*)STRUCT_VAR_P(cfg,fmt->magic_offset));        \
-  STMT_END
-
-#define CAL_USE_DEFAULTS      (1u<<0)
-#define CAL_CLEAR_FIRST       (1u<<1)
-#define CAL_WARN_DEPRECATIONS (1u<<2)
-
-void *config_new(const config_format_t *fmt);
-void config_free_(const config_format_t *fmt, void *options);
-#define config_free(fmt, options) do {                \
-    config_free_((fmt), (options));                   \
-    (options) = NULL;                                 \
-  } while (0)
-
-struct config_line_t *config_get_assigned_option(const config_format_t *fmt,
-                                          const void *options, const char *key,
-                                          int escape_val);
-int config_is_same(const config_format_t *fmt,
-                   const void *o1, const void *o2,
-                   const char *name);
-void config_init(const config_format_t *fmt, void *options);
-void *config_dup(const config_format_t *fmt, const void *old);
-char *config_dump(const config_format_t *fmt, const void *default_options,
-                  const void *options, int minimal,
-                  int comment_defaults);
-int config_assign(const config_format_t *fmt, void *options,
-                  struct config_line_t *list,
-                  unsigned flags, char **msg);
-config_var_t *config_find_option_mutable(config_format_t *fmt,
-                                         const char *key);
-const char *config_find_deprecation(const config_format_t *fmt,
-                                     const char *key);
-const config_var_t *config_find_option(const config_format_t *fmt,
-                                       const char *key);
-const char *config_expand_abbrev(const config_format_t *fmt,
-                                 const char *option,
-                                 int command_line, int warn_obsolete);
-void warn_deprecated_option(const char *what, const char *why);
-
-/* Helper macros to compare an option across two configuration objects */
-#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
-#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
-#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
-#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
-
-#endif /* !defined(TOR_CONFPARSE_H) */
diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h
index 74d2fefa16..e6be797017 100644
--- a/src/app/config/or_options_st.h
+++ b/src/app/config/or_options_st.h
@@ -18,6 +18,7 @@
 
 struct smartlist_t;
 struct config_line_t;
+struct config_suite_t;
 
 /** Enumeration of outbound address configuration types:
  * Exit-only, OR-only, or both */
@@ -72,6 +73,9 @@ struct or_options_t {
   routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as exits. */
+  routerset_t *MiddleNodes; /**< Structure containing nicknames, digests,
+                             * country codes and IP address patterns of ORs to
+                             * consider as middles. */
   routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as entry points. */
@@ -118,7 +122,6 @@ struct or_options_t {
   struct config_line_t *RecommendedVersions;
   struct config_line_t *RecommendedClientVersions;
   struct config_line_t *RecommendedServerVersions;
-  struct config_line_t *RecommendedPackages;
   /** Whether dirservers allow router descriptors with private IPs. */
   int DirAllowPrivateAddresses;
   /** Whether routers accept EXTEND cells to routers with private IPs. */
@@ -245,6 +248,17 @@ struct or_options_t {
    * pad to the server regardless of server support. */
   int ConnectionPadding;
 
+  /** Boolean: if true, then circuit padding will be negotiated by client
+   * and server, subject to consenus limits (default). If 0, it will be fully
+   * disabled. */
+  int CircuitPadding;
+
+  /** Boolean: if true, then this client will only use circuit padding
+   * algorithms that are known to use a low amount of overhead. If false,
+   * we will use all available circuit padding algorithms.
+   */
+  int ReducedCircuitPadding;
+
   /** To what authority types do we publish our descriptor? Choices are
    * "v1", "v2", "v3", "bridge", or "". */
   struct smartlist_t *PublishServerDescriptor;
@@ -666,6 +680,9 @@ struct or_options_t {
    * accessing this value directly.  */
   int ClientPreferIPv6DirPort;
 
+  /** If true, prefer an IPv4 or IPv6 OR port at random. */
+  int ClientAutoIPv6ORPort;
+
   /** The length of time that we think a consensus should be fresh. */
   int V3AuthVotingInterval;
   /** The length of time we think it will take to distribute votes. */
@@ -991,6 +1008,13 @@ struct or_options_t {
    */
   uint64_t MaxUnparseableDescSizeToLog;
 
+  /** Bool (default: 1): Under bandwidth pressure, if set to 1, the authority
+   * will always answer directory requests from relays but will start sending
+   * 503 error code for the other connections. If set to 0, all connections
+   * are considered the same and the authority will try to answer them all
+   * regardless of bandwidth pressure or not. */
+  int AuthDirRejectRequestsUnderLoad;
+
   /** Bool (default: 1): Switch for the shared random protocol. Only
    * relevant to a directory authority. If off, the authority won't
    * participate in the protocol. If on (default), a flag is added to the
@@ -1072,6 +1096,33 @@ struct or_options_t {
 
   /** Autobool: Do we refuse single hop client rendezvous? */
   int DoSRefuseSingleHopClientRendezvous;
+
+  /** Interval: how long without activity does it take for a client
+   * to become dormant?
+   **/
+  int DormantClientTimeout;
+
+  /** Boolean: true if having an idle stream is sufficient to prevent a client
+   * from becoming dormant.
+   **/
+  int DormantTimeoutDisabledByIdleStreams;
+
+  /** Boolean: true if Tor should be dormant the first time it starts with
+   * a datadirectory; false otherwise. */
+  int DormantOnFirstStartup;
+  /**
+   * Boolean: true if Tor should treat every startup event as cancelling
+   * a possible previous dormant state.
+   **/
+  int DormantCanceledByStartup;
+
+  /**
+   * Configuration objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant configuration object.
+   */
+  struct config_suite_t *subconfigs_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_OPTIONS_ST_H) */
diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h
index 5f8214d146..225003bb7e 100644
--- a/src/app/config/or_state_st.h
+++ b/src/app/config/or_state_st.h
@@ -15,6 +15,7 @@
 
 #include "lib/cc/torint.h"
 struct smartlist_t;
+struct config_suite_t;
 
 /** Persistent state for an onion router, as saved to disk. */
 struct or_state_t {
@@ -87,6 +88,21 @@ struct or_state_t {
 
   /** When did we last rotate our onion key?  "0" for 'no idea'. */
   time_t LastRotatedOnionKey;
+
+  /** Number of minutes since the last user-initiated request (as defined by
+   * the dormant net-status system.) Set to zero if we are dormant. */
+  int MinutesSinceUserActivity;
+  /** True if we were dormant when we last wrote the file; false if we
+   * weren't.  "auto" on initial startup. */
+  int Dormant;
+
+  /**
+   * State objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant state object if you must.
+   */
+  struct config_suite_t *substates_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_STATE_ST_H) */
diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c
index 89039a05b5..552bd2c443 100644
--- a/src/app/config/statefile.c
+++ b/src/app/config/statefile.c
@@ -32,10 +32,11 @@
 #include "core/or/or.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/stats/rephist.h"
@@ -45,6 +46,7 @@
 #include "app/config/statefile.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
@@ -68,16 +70,13 @@ static config_abbrev_t state_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_state_t);
 
-/*XXXX these next two are duplicates or near-duplicates from config.c */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member),       \
-      initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
 
 /** Array of "state" variables saved to the ~/.tor/state file. */
-static config_var_t state_vars_[] = {
+static const config_var_t state_vars_[] = {
   /* Remember to document these in state-contents.txt ! */
 
   V(AccountingBytesReadInInterval,    MEMUNIT,  NULL),
@@ -103,19 +102,19 @@ static config_var_t state_vars_[] = {
   V(HidServRevCounter,            LINELIST, NULL),
 
   V(BWHistoryReadEnds,                ISOTIME,  NULL),
-  V(BWHistoryReadInterval,            UINT,     "900"),
+  V(BWHistoryReadInterval,            POSINT,     "900"),
   V(BWHistoryReadValues,              CSV,      ""),
   V(BWHistoryReadMaxima,              CSV,      ""),
   V(BWHistoryWriteEnds,               ISOTIME,  NULL),
-  V(BWHistoryWriteInterval,           UINT,     "900"),
+  V(BWHistoryWriteInterval,           POSINT,     "900"),
   V(BWHistoryWriteValues,             CSV,      ""),
   V(BWHistoryWriteMaxima,             CSV,      ""),
   V(BWHistoryDirReadEnds,             ISOTIME,  NULL),
-  V(BWHistoryDirReadInterval,         UINT,     "900"),
+  V(BWHistoryDirReadInterval,         POSINT,     "900"),
   V(BWHistoryDirReadValues,           CSV,      ""),
   V(BWHistoryDirReadMaxima,           CSV,      ""),
   V(BWHistoryDirWriteEnds,            ISOTIME,  NULL),
-  V(BWHistoryDirWriteInterval,        UINT,     "900"),
+  V(BWHistoryDirWriteInterval,        POSINT,     "900"),
   V(BWHistoryDirWriteValues,          CSV,      ""),
   V(BWHistoryDirWriteMaxima,          CSV,      ""),
 
@@ -126,11 +125,14 @@ static config_var_t state_vars_[] = {
   V(LastRotatedOnionKey,              ISOTIME,  NULL),
   V(LastWritten,                      ISOTIME,  NULL),
 
-  V(TotalBuildTimes,                  UINT,     NULL),
-  V(CircuitBuildAbandonedCount,       UINT,     "0"),
+  V(TotalBuildTimes,                  POSINT,     NULL),
+  V(CircuitBuildAbandonedCount,       POSINT,     "0"),
   VAR("CircuitBuildTimeBin",          LINELIST_S, BuildtimeHistogram, NULL),
   VAR("BuildtimeHistogram",           LINELIST_V, BuildtimeHistogram, NULL),
 
+  V(MinutesSinceUserActivity,         POSINT,     NULL),
+  V(Dormant,                          AUTOBOOL, "auto"),
+
   END_OF_CONFIG_VARS
 };
 
@@ -143,31 +145,48 @@ static int or_state_validate_cb(void *old_options, void *options,
                                 void *default_options,
                                 int from_setconf, char **msg);
 
-static void or_state_free_cb(void *state);
-
 /** Magic value for or_state_t. */
 #define OR_STATE_MAGIC 0x57A73f57
 
 /** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines)
+static struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(or_state_t, ExtraLines),
 };
 
 /** Configuration format for or_state_t. */
 static const config_format_t state_format = {
   sizeof(or_state_t),
-  OR_STATE_MAGIC,
-  offsetof(or_state_t, magic_),
+  {
+   "or_state_t",
+   OR_STATE_MAGIC,
+   offsetof(or_state_t, magic_),
+  },
   state_abbrevs_,
   NULL,
   state_vars_,
   or_state_validate_cb,
-  or_state_free_cb,
+  NULL,
   &state_extra_var,
+  offsetof(or_state_t, substates_),
 };
 
+/* A global configuration manager for state-file objects */
+static config_mgr_t *state_mgr = NULL;
+
+/** Return the configuration manager for state-file objects. */
+static const config_mgr_t *
+get_state_mgr(void)
+{
+  if (PREDICT_UNLIKELY(state_mgr == NULL)) {
+    state_mgr = config_mgr_new(&state_format);
+    config_mgr_freeze(state_mgr);
+  }
+  return state_mgr;
+}
+
 /** Persistent serialized state. */
 static or_state_t *global_state = NULL;
 
@@ -262,12 +281,6 @@ or_state_validate_cb(void *old_state, void *state, void *default_state,
   return or_state_validate(state, msg);
 }
 
-static void
-or_state_free_cb(void *state)
-{
-  or_state_free_(state);
-}
-
 /** Return 0 if every setting in <b>state</b> is reasonable, and a
  * permissible transition from <b>old_state</b>.  Else warn and return -1.
  * Should have no side effects, except for normalizing the contents of
@@ -292,7 +305,7 @@ or_state_set(or_state_t *new_state)
   char *err = NULL;
   int ret = 0;
   tor_assert(new_state);
-  config_free(&state_format, global_state);
+  config_free(get_state_mgr(), global_state);
   global_state = new_state;
   if (entry_guards_parse_state(global_state, 1, &err)<0) {
     log_warn(LD_GENERAL,"%s",err);
@@ -308,6 +321,8 @@ or_state_set(or_state_t *new_state)
       get_circuit_build_times_mutable(),global_state) < 0) {
     ret = -1;
   }
+  netstatus_load_from_state(global_state, time(NULL));
+
   return ret;
 }
 
@@ -353,9 +368,8 @@ or_state_save_broken(char *fname)
 STATIC or_state_t *
 or_state_new(void)
 {
-  or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t));
-  new_state->magic_ = OR_STATE_MAGIC;
-  config_init(&state_format, new_state);
+  or_state_t *new_state = config_new(get_state_mgr());
+  config_init(get_state_mgr(), new_state);
 
   return new_state;
 }
@@ -396,7 +410,7 @@ or_state_load(void)
     int assign_retval;
     if (config_get_lines(contents, &lines, 0)<0)
       goto done;
-    assign_retval = config_assign(&state_format, new_state,
+    assign_retval = config_assign(get_state_mgr(), new_state,
                                   lines, 0, &errmsg);
     config_free_lines(lines);
     if (assign_retval<0)
@@ -423,7 +437,7 @@ or_state_load(void)
     or_state_save_broken(fname);
 
     tor_free(contents);
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
     new_state = or_state_new();
   } else if (contents) {
@@ -456,7 +470,7 @@ or_state_load(void)
   tor_free(fname);
   tor_free(contents);
   if (new_state)
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
   return r;
 }
@@ -499,6 +513,8 @@ or_state_save(time_t now)
   entry_guards_update_state(global_state);
   rep_hist_update_state(global_state);
   circuit_build_times_update_state(get_circuit_build_times(), global_state);
+  netstatus_flush_to_state(global_state, now);
+
   if (accounting_is_enabled(get_options()))
     accounting_run_housekeeping(now);
 
@@ -507,7 +523,7 @@ or_state_save(time_t now)
   tor_free(global_state->TorVersion);
   tor_asprintf(&global_state->TorVersion, "Tor %s", get_version());
 
-  state = config_dump(&state_format, NULL, global_state, 1, 0);
+  state = config_dump(get_state_mgr(), NULL, global_state, 1, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&contents,
                "# Tor state file last generated on %s local time\n"
@@ -717,7 +733,7 @@ or_state_free_(or_state_t *state)
   if (!state)
     return;
 
-  config_free(&state_format, state);
+  config_free(get_state_mgr(), state);
 }
 
 void
@@ -725,4 +741,5 @@ or_state_free_all(void)
 {
   or_state_free(global_state);
   global_state = NULL;
+  config_mgr_free(state_mgr);
 }
diff --git a/src/app/config/statefile.h b/src/app/config/statefile.h
index 1950078450..515c90a52f 100644
--- a/src/app/config/statefile.h
+++ b/src/app/config/statefile.h
@@ -31,6 +31,6 @@ STATIC struct config_line_t *get_transport_in_state_by_name(
 STATIC void or_state_free_(or_state_t *state);
 #define or_state_free(st) FREE_AND_NULL(or_state_t, or_state_free_, (st))
 STATIC or_state_t *or_state_new(void);
-#endif
+#endif /* defined(STATEFILE_PRIVATE) */
 
 #endif /* !defined(TOR_STATEFILE_H) */
diff --git a/src/app/config/testnet.inc b/src/app/config/testnet.inc
new file mode 100644
index 0000000000..0ed3c38627
--- /dev/null
+++ b/src/app/config/testnet.inc
@@ -0,0 +1,33 @@
+{ "DirAllowPrivateAddresses", "1" },
+{ "EnforceDistinctSubnets", "0" },
+{ "AssumeReachable", "1" },
+{ "AuthDirMaxServersPerAddr", "0" },
+{ "ClientBootstrapConsensusAuthorityDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusFallbackDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay", "0" },
+{ "ClientDNSRejectInternalAddresses", "0" },
+{ "ClientRejectInternalAddresses", "0" },
+{ "CountPrivateBandwidth", "1" },
+{ "ExitPolicyRejectPrivate", "0" },
+{ "ExtendAllowPrivateAddresses", "1" },
+{ "V3AuthVotingInterval", "5 minutes" },
+{ "V3AuthVoteDelay", "20 seconds" },
+{ "V3AuthDistDelay", "20 seconds" },
+{ "TestingV3AuthInitialVotingInterval", "150 seconds" },
+{ "TestingV3AuthInitialVoteDelay", "20 seconds" },
+{ "TestingV3AuthInitialDistDelay", "20 seconds" },
+{ "TestingAuthDirTimeToLearnReachability", "0 minutes" },
+{ "TestingEstimatedDescriptorPropagationTime", "0 minutes" },
+{ "MinUptimeHidServDirectoryV2", "0 minutes" },
+{ "TestingServerDownloadInitialDelay", "0" },
+{ "TestingClientDownloadInitialDelay", "0" },
+{ "TestingServerConsensusDownloadInitialDelay", "0" },
+{ "TestingClientConsensusDownloadInitialDelay", "0" },
+{ "TestingBridgeDownloadInitialDelay", "10" },
+{ "TestingBridgeBootstrapDownloadInitialDelay", "0" },
+{ "TestingClientMaxIntervalWithoutRequest", "5 seconds" },
+{ "TestingDirConnectionMaxStall", "30 seconds" },
+{ "TestingEnableConnBwEvent", "1" },
+{ "TestingEnableCellStatsEvent", "1" },
+{ "RendPostPeriod", "2 minutes" },
+{ "___UsingTestNetworkDefaults", "1" },
diff --git a/src/app/main/main.c b/src/app/main/main.c
index 67f2181cd5..1901479eba 100644
--- a/src/app/main/main.c
+++ b/src/app/main/main.c
@@ -15,63 +15,52 @@
 #include "app/config/statefile.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
-#include "core/or/channeltls.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitlist.h"
-#include "core/or/circuitmux_ewma.h"
 #include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/dos.h"
-#include "core/or/policies.h"
-#include "core/or/protover.h"
 #include "core/or/relay.h"
-#include "core/or/scheduler.h"
 #include "core/or/status.h"
 #include "feature/api/tor_api.h"
 #include "feature/api/tor_api_internal.h"
 #include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/entrynodes.h"
-#include "feature/client/transports.h"
 #include "feature/control/control.h"
-#include "feature/dirauth/bwauth.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
 #include "feature/dirparse/routerparse.h"
 #include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
-#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/relay/onion_queue.h"
 #include "feature/relay/routerkeys.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
-#include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/err/backtrace.h"
-#include "lib/geoip/geoip.h"
+#include "lib/net/resolve.h"
 
 #include "lib/process/waitpid.h"
+#include "lib/pubsub/pubsub_build.h"
 
 #include "lib/meminfo/meminfo.h"
 #include "lib/osinfo/uname.h"
@@ -83,10 +72,10 @@
 #include "lib/encoding/confline.h"
 #include "lib/evloop/timers.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
 
@@ -107,8 +96,6 @@
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
-void evdns_shutdown(int);
-
 #ifdef HAVE_RUST
 // helper function defined in Rust to output a log message indicating if tor is
 // running with Rust enabled. See src/rust/tor_util
@@ -301,6 +288,19 @@ process_signal(int sig)
       log_heartbeat(time(NULL));
       control_event_signal(sig);
       break;
+    case SIGACTIVE:
+      /* "SIGACTIVE" counts as ersatz user activity. */
+      note_user_activity(approx_time());
+      control_event_signal(sig);
+      break;
+    case SIGDORMANT:
+      /* "SIGDORMANT" means to ignore past user activity */
+      log_notice(LD_GENERAL, "Going dormant because of controller request.");
+      reset_user_activity(0);
+      set_network_participation(false);
+      schedule_rescan_periodic_events();
+      control_event_signal(sig);
+      break;
   }
 }
 
@@ -426,18 +426,6 @@ dumpstats(int severity)
   rend_service_dump_stats(severity);
 }
 
-/** Called by exit() as we shut down the process.
- */
-static void
-exit_function(void)
-{
-  /* NOTE: If we ever daemonize, this gets called immediately.  That's
-   * okay for now, because we only use this on Windows.  */
-#ifdef _WIN32
-  WSACleanup();
-#endif
-}
-
 #ifdef _WIN32
 #define UNIX_ONLY 0
 #else
@@ -482,6 +470,8 @@ static struct {
   { SIGNEWNYM, 0, NULL },
   { SIGCLEARDNSCACHE, 0, NULL },
   { SIGHEARTBEAT, 0, NULL },
+  { SIGACTIVE, 0, NULL },
+  { SIGDORMANT, 0, NULL },
   { -1, -1, NULL }
 };
 
@@ -546,18 +536,13 @@ tor_init(int argc, char *argv[])
   tor_snprintf(progname, sizeof(progname), "Tor %s", get_version());
   log_set_application_name(progname);
 
-  /* Set up the crypto nice and early */
-  if (crypto_early_init() < 0) {
-    log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!");
-    return -1;
-  }
-
   /* Initialize the history structures. */
   rep_hist_init();
   /* Initialize the service cache. */
   rend_cache_init();
   addressmap_init(); /* Init the client dns cache. Do it always, since it's
                       * cheap. */
+
   /* Initialize the HS subsystem. */
   hs_init();
 
@@ -632,12 +617,6 @@ tor_init(int argc, char *argv[])
   rust_log_welcome_string();
 #endif /* defined(HAVE_RUST) */
 
-  if (network_init()<0) {
-    log_err(LD_BUG,"Error initializing network; exiting.");
-    return -1;
-  }
-  atexit(exit_function);
-
   int init_rv = options_init_from_torrc(argc,argv);
   if (init_rv < 0) {
     log_err(LD_CONFIG,"Reading config failed--see warnings above.");
@@ -651,9 +630,17 @@ tor_init(int argc, char *argv[])
   /* The options are now initialised */
   const or_options_t *options = get_options();
 
-  /* Initialize channelpadding parameters to defaults until we get
-   * a consensus */
+  /* Initialize channelpadding and circpad parameters to defaults
+   * until we get a consensus */
   channelpadding_new_consensus_params(NULL);
+  circpad_new_consensus_params(NULL);
+
+  /* Initialize circuit padding to defaults+torrc until we get a consensus */
+  circpad_machines_init();
+
+  /* Initialize hidden service DoS subsystem. We need to do this once the
+   * configuration object has been set because it can be accessed. */
+  hs_dos_init();
 
   /* Initialize predicted ports list after loading options */
   predicted_ports_init();
@@ -670,10 +657,6 @@ tor_init(int argc, char *argv[])
     log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
     return -1;
   }
-  stream_choice_seed_weak_rng();
-  if (tor_init_libevent_rng() < 0) {
-    log_warn(LD_NET, "Problem initializing libevent RNG.");
-  }
 
   /* Scan/clean unparseable descriptors; after reading config */
   routerparse_init();
@@ -743,86 +726,6 @@ release_lockfile(void)
   }
 }
 
-/** Free all memory that we might have allocated somewhere.
- * If <b>postfork</b>, we are a worker process and we want to free
- * only the parts of memory that we won't touch. If !<b>postfork</b>,
- * Tor is shutting down and we should free everything.
- *
- * Helps us find the real leaks with sanitizers and the like. Also valgrind
- * should then report 0 reachable in its leak report (in an ideal world --
- * in practice libevent, SSL, libc etc never quite free everything). */
-void
-tor_free_all(int postfork)
-{
-  if (!postfork) {
-    evdns_shutdown(1);
-  }
-  geoip_free_all();
-  geoip_stats_free_all();
-  dirvote_free_all();
-  routerlist_free_all();
-  networkstatus_free_all();
-  addressmap_free_all();
-  dirserv_free_fingerprint_list();
-  dirserv_free_all();
-  dirserv_clear_measured_bw_cache();
-  rend_cache_free_all();
-  rend_service_authorization_free_all();
-  rep_hist_free_all();
-  dns_free_all();
-  clear_pending_onions();
-  circuit_free_all();
-  entry_guards_free_all();
-  pt_free_all();
-  channel_tls_free_all();
-  channel_free_all();
-  connection_free_all();
-  connection_edge_free_all();
-  scheduler_free_all();
-  nodelist_free_all();
-  microdesc_free_all();
-  routerparse_free_all();
-  ext_orport_free_all();
-  control_free_all();
-  tor_free_getaddrinfo_cache();
-  protover_free_all();
-  bridges_free_all();
-  consdiffmgr_free_all();
-  hs_free_all();
-  dos_free_all();
-  circuitmux_ewma_free_all();
-  accounting_free_all();
-
-  if (!postfork) {
-    config_free_all();
-    or_state_free_all();
-    router_free_all();
-    routerkeys_free_all();
-    policies_free_all();
-  }
-  if (!postfork) {
-    tor_tls_free_all();
-#ifndef _WIN32
-    tor_getpwnam(NULL);
-#endif
-  }
-  /* stuff in main.c */
-
-  tor_mainloop_free_all();
-
-  if (!postfork) {
-    release_lockfile();
-  }
-  tor_libevent_free_all();
-  /* Stuff in util.c and address.c*/
-  if (!postfork) {
-    escaped(NULL);
-    esc_router_info(NULL);
-    clean_up_backtrace_handler();
-    logs_free_all(); /* free log strings. do this last so logs keep working. */
-  }
-}
-
 /**
  * Remove the specified file, and log a warning if the operation fails for
  * any reason other than the file not existing. Ignores NULL filenames.
@@ -836,51 +739,6 @@ tor_remove_file(const char *filename)
   }
 }
 
-/** Do whatever cleanup is necessary before shutting Tor down. */
-void
-tor_cleanup(void)
-{
-  const or_options_t *options = get_options();
-  if (options->command == CMD_RUN_TOR) {
-    time_t now = time(NULL);
-    /* Remove our pid file. We don't care if there was an error when we
-     * unlink, nothing we could do about it anyways. */
-    tor_remove_file(options->PidFile);
-    /* Remove control port file */
-    tor_remove_file(options->ControlPortWriteToFile);
-    /* Remove cookie authentication file */
-    {
-      char *cookie_fname = get_controller_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    /* Remove Extended ORPort cookie authentication file */
-    {
-      char *cookie_fname = get_ext_or_auth_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    if (accounting_is_enabled(options))
-      accounting_record_bandwidth_usage(now, get_or_state());
-    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
-    or_state_save(now);
-    if (authdir_mode(options)) {
-      sr_save_and_cleanup();
-    }
-    if (authdir_mode_tests_reachability(options))
-      rep_hist_record_mtbf_data(now, 0);
-    keypin_close_journal();
-  }
-
-  timers_shutdown();
-
-  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
-                      later, if it makes shutdown unacceptably slow.  But for
-                      now, leave it here: it's helped us catch bugs in the
-                      past. */
-  crypto_global_cleanup();
-}
-
 /** Read/create keys as needed, and echo our fingerprint to stdout. */
 static int
 do_list_fingerprint(void)
@@ -1274,7 +1132,6 @@ int
 run_tor_main_loop(void)
 {
   handle_signals();
-  monotime_init();
   timers_initialize();
   initialize_mainloop_events();
 
@@ -1379,6 +1236,32 @@ run_tor_main_loop(void)
   return do_main_loop();
 }
 
+/** Install the publish/subscribe relationships for all the subsystems. */
+void
+pubsub_install(void)
+{
+    pubsub_builder_t *builder = pubsub_builder_new();
+    int r = subsystems_add_pubsub(builder);
+    tor_assert(r == 0);
+    r = tor_mainloop_connect_pubsub(builder); // consumes builder
+    tor_assert(r == 0);
+}
+
+/** Connect the mainloop to its publish/subscribe message delivery events if
+ * appropriate, and configure the global channels appropriately. */
+void
+pubsub_connect(void)
+{
+  if (get_options()->command == CMD_RUN_TOR) {
+    tor_mainloop_connect_pubsub_events();
+    /* XXXX For each pubsub channel, its delivery strategy should be set at
+     * this XXXX point, using tor_mainloop_set_delivery_strategy().
+     */
+    tor_mainloop_set_delivery_strategy("orconn", DELIV_IMMEDIATE);
+    tor_mainloop_set_delivery_strategy("ocirc", DELIV_IMMEDIATE);
+  }
+}
+
 /* Main entry point for the Tor process.  Called from tor_main(), and by
  * anybody embedding Tor. */
 int
@@ -1386,54 +1269,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
 {
   int result = 0;
 
-#ifdef _WIN32
-#ifndef HeapEnableTerminationOnCorruption
-#define HeapEnableTerminationOnCorruption 1
-#endif
-  /* On heap corruption, just give up; don't try to play along. */
-  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
-
-  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
-   * (On 64-bit Windows it always fails, and some compilers don't like the
-   * PSETDEP cast.)
-   * 32-bit Windows defines _WIN32.
-   * 64-bit Windows defines _WIN32 and _WIN64. */
-#ifndef _WIN64
-  /* Call SetProcessDEPPolicy to permanently enable DEP.
-     The function will not resolve on earlier versions of Windows,
-     and failure is not dangerous. */
-  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
-  if (hMod) {
-    typedef BOOL (WINAPI *PSETDEP)(DWORD);
-    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
-                           "SetProcessDEPPolicy");
-    if (setdeppolicy) {
-      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
-      setdeppolicy(3);
-    }
-  }
-#endif /* !defined(_WIN64) */
-#endif /* defined(_WIN32) */
-
-  {
-    int bt_err = configure_backtrace_handler(get_version());
-    if (bt_err < 0) {
-      log_warn(LD_BUG, "Unable to install backtrace handler: %s",
-               strerror(-bt_err));
-    }
-  }
-
 #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED
   event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_);
 #endif
 
-  init_protocol_warning_severity_level();
+  subsystems_init();
 
-  update_approx_time(time(NULL));
-  tor_threads_init();
-  tor_compress_init();
-  init_logging(0);
-  monotime_init();
+  init_protocol_warning_severity_level();
 
   int argc = tor_cfg->argc + tor_cfg->argc_owned;
   char **argv = tor_calloc(argc, sizeof(char*));
@@ -1451,6 +1293,9 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
      }
   }
 #endif /* defined(NT_SERVICE) */
+
+  pubsub_install();
+
   {
     int init_rv = tor_init(argc, argv);
     if (init_rv) {
@@ -1460,6 +1305,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     }
   }
 
+  pubsub_connect();
+
   if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
     sandbox_cfg_t* cfg = sandbox_init_filter();
 
@@ -1469,6 +1316,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
       tor_free_all(0);
       return -1;
     }
+    tor_make_getaddrinfo_cache_active();
 
     // registering libevent rng
 #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE
diff --git a/src/app/main/main.h b/src/app/main/main.h
index bbbbf984fb..76574a9071 100644
--- a/src/app/main/main.h
+++ b/src/app/main/main.h
@@ -21,11 +21,11 @@ void release_lockfile(void);
 
 void tor_remove_file(const char *filename);
 
-void tor_cleanup(void);
-void tor_free_all(int postfork);
-
 int tor_init(int argc, char **argv);
 
 int run_tor_main_loop(void);
 
+void pubsub_install(void);
+void pubsub_connect(void);
+
 #endif /* !defined(TOR_MAIN_H) */
diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c
index 05d203b0be..de82eeb1de 100644
--- a/src/app/main/ntmain.c
+++ b/src/app/main/ntmain.c
@@ -24,6 +24,7 @@
 #include "app/config/config.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
 #include "core/mainloop/mainloop.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/fs/winlib.h"
@@ -282,7 +283,9 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_install();
   r = tor_init(backup_argc, backup_argv);
+
   if (r) {
     /* Failed to start the Tor service */
     r = NT_SERVICE_ERROR_TORINIT_FAILED;
@@ -293,6 +296,8 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_connect();
+
   /* Set the service's status to SERVICE_RUNNING and start the main
    * event loop */
   service_status.dwCurrentState = SERVICE_RUNNING;
@@ -321,9 +326,12 @@ nt_service_main(void)
     errmsg = format_win32_error(result);
     printf("Service error %d : %s\n", (int) result, errmsg);
     tor_free(errmsg);
+
+    pubsub_install();
     if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
       if (tor_init(backup_argc, backup_argv))
         return;
+      pubsub_connect();
       switch (get_options()->command) {
       case CMD_RUN_TOR:
         run_tor_main_loop();
@@ -607,6 +615,7 @@ nt_service_install(int argc, char **argv)
                             &sidUse) == 0) {
     /* XXXX For some reason, the above test segfaults. Fix that. */
     printf("User \"%s\" doesn't seem to exist.\n", user_acct);
+    tor_free(command);
     return -1;
   } else {
     printf("Will try to install service as user \"%s\".\n", user_acct);
diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c
new file mode 100644
index 0000000000..93d6351d1b
--- /dev/null
+++ b/src/app/main/shutdown.c
@@ -0,0 +1,167 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file shutdown.c
+ * @brief Code to free global resources used by Tor.
+ *
+ * In the future, this should all be handled by the subsystem manager. */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop_pubsub.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/connection_edge.h"
+#include "core/or/dos.h"
+#include "core/or/scheduler.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendclient.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/geoip/geoip.h"
+
+void evdns_shutdown(int);
+
+/** Do whatever cleanup is necessary before shutting Tor down. */
+void
+tor_cleanup(void)
+{
+  const or_options_t *options = get_options();
+  if (options->command == CMD_RUN_TOR) {
+    time_t now = time(NULL);
+    /* Remove our pid file. We don't care if there was an error when we
+     * unlink, nothing we could do about it anyways. */
+    tor_remove_file(options->PidFile);
+    /* Remove control port file */
+    tor_remove_file(options->ControlPortWriteToFile);
+    /* Remove cookie authentication file */
+    {
+      char *cookie_fname = get_controller_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    /* Remove Extended ORPort cookie authentication file */
+    {
+      char *cookie_fname = get_ext_or_auth_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    if (accounting_is_enabled(options))
+      accounting_record_bandwidth_usage(now, get_or_state());
+    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
+    or_state_save(now);
+    if (authdir_mode(options)) {
+      sr_save_and_cleanup();
+    }
+    if (authdir_mode_tests_reachability(options))
+      rep_hist_record_mtbf_data(now, 0);
+  }
+
+  timers_shutdown();
+
+  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
+                      later, if it makes shutdown unacceptably slow.  But for
+                      now, leave it here: it's helped us catch bugs in the
+                      past. */
+}
+
+/** Free all memory that we might have allocated somewhere.
+ * If <b>postfork</b>, we are a worker process and we want to free
+ * only the parts of memory that we won't touch. If !<b>postfork</b>,
+ * Tor is shutting down and we should free everything.
+ *
+ * Helps us find the real leaks with sanitizers and the like. Also valgrind
+ * should then report 0 reachable in its leak report (in an ideal world --
+ * in practice libevent, SSL, libc etc never quite free everything). */
+void
+tor_free_all(int postfork)
+{
+  if (!postfork) {
+    evdns_shutdown(1);
+  }
+  geoip_free_all();
+  geoip_stats_free_all();
+  routerlist_free_all();
+  networkstatus_free_all();
+  addressmap_free_all();
+  dirserv_free_all();
+  rend_cache_free_all();
+  rend_service_authorization_free_all();
+  rep_hist_free_all();
+  circuit_free_all();
+  circpad_machines_free();
+  entry_guards_free_all();
+  pt_free_all();
+  channel_tls_free_all();
+  channel_free_all();
+  connection_free_all();
+  connection_edge_free_all();
+  scheduler_free_all();
+  nodelist_free_all();
+  microdesc_free_all();
+  routerparse_free_all();
+  control_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
+  hs_free_all();
+  dos_free_all();
+  circuitmux_ewma_free_all();
+  accounting_free_all();
+  circpad_free_all();
+
+  if (!postfork) {
+    config_free_all();
+    or_state_free_all();
+  }
+  if (!postfork) {
+#ifndef _WIN32
+    tor_getpwnam(NULL);
+#endif
+  }
+  /* stuff in main.c */
+
+  tor_mainloop_disconnect_pubsub();
+
+  if (!postfork) {
+    release_lockfile();
+  }
+
+  subsystems_shutdown();
+
+  /* Stuff in util.c and address.c*/
+  if (!postfork) {
+    esc_router_info(NULL);
+  }
+}
diff --git a/src/app/main/shutdown.h b/src/app/main/shutdown.h
new file mode 100644
index 0000000000..1bca96a0aa
--- /dev/null
+++ b/src/app/main/shutdown.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shutdown.h
+ * \brief Header file for shutdown.c.
+ **/
+
+#ifndef TOR_SHUTDOWN_H
+#define TOR_SHUTDOWN_H
+
+void tor_cleanup(void);
+void tor_free_all(int postfork);
+
+#endif /* !defined(TOR_SHUTDOWN_H) */
diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c
new file mode 100644
index 0000000000..5aa4fd76c9
--- /dev/null
+++ b/src/app/main/subsysmgr.c
@@ -0,0 +1,252 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "lib/malloc/malloc.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * True iff we have checked tor_subsystems for consistency.
+ **/
+static bool subsystem_array_validated = false;
+
+/**
+ * True if a given subsystem is initialized.  Expand this array if there
+ * are more than this number of subsystems.  (We'd rather not
+ * dynamically allocate in this module.)
+ **/
+static bool sys_initialized[128];
+
+/**
+ * Exit with a raw assertion if the subsystems list is inconsistent;
+ * initialize the subsystem_initialized array.
+ **/
+static void
+check_and_setup(void)
+{
+  if (subsystem_array_validated)
+    return;
+
+  raw_assert(ARRAY_LENGTH(sys_initialized) >= n_tor_subsystems);
+  memset(sys_initialized, 0, sizeof(sys_initialized));
+
+  int last_level = MIN_SUBSYS_LEVEL;
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) {
+      fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. "
+              "It is supposed to be between %d and %d (inclusive).\n",
+              sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    if (sys->level < last_level) {
+      fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. "
+              "Its level is %d; but the previous subsystem's level was %d.\n",
+              sys->name, i, sys->level, last_level);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    last_level = sys->level;
+  }
+
+  subsystem_array_validated = true;
+}
+
+/**
+ * Initialize all the subsystems; exit on failure.
+ **/
+int
+subsystems_init(void)
+{
+  return subsystems_init_upto(MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Initialize all the subsystems whose level is less than or equal to
+ * <b>target_level</b>; exit on failure.
+ **/
+int
+subsystems_init_upto(int target_level)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (sys_initialized[i])
+      continue;
+    int r = 0;
+    if (sys->initialize) {
+      // Note that the logging subsystem is designed so that it does no harm
+      // to log a message in an uninitialized state.  These messages will be
+      // discarded for now, however.
+      log_debug(LD_GENERAL, "Initializing %s", sys->name);
+      r = sys->initialize();
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n",
+              sys->name, i);
+      raw_assert_unreached_msg("A subsystem couldn't be initialized.");
+    }
+    sys_initialized[i] = true;
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems of level no more than <b>target_level</b>.
+ **/
+int
+subsystems_add_pubsub_upto(pubsub_builder_t *builder,
+                           int target_level)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (! sys_initialized[i])
+      continue;
+    int r = 0;
+    if (sys->add_pubsub) {
+      subsys_id_t sysid = get_subsys_id(sys->name);
+      raw_assert(sysid != ERROR_ID);
+      pubsub_connector_t *connector;
+      connector = pubsub_connector_for_subsystem(builder, sysid);
+      r = sys->add_pubsub(connector);
+      pubsub_connector_free(connector);
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) could not connect to "
+              "publish/subscribe system.", sys->name, sys->level);
+      raw_assert_unreached_msg("A subsystem couldn't be connected.");
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems.
+ **/
+int
+subsystems_add_pubsub(pubsub_builder_t *builder)
+{
+  return subsystems_add_pubsub_upto(builder, MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Shut down all the subsystems.
+ **/
+void
+subsystems_shutdown(void)
+{
+  subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1);
+}
+
+/**
+ * Shut down all the subsystems whose level is above <b>target_level</b>.
+ **/
+void
+subsystems_shutdown_downto(int target_level)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level <= target_level)
+      break;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->shutdown) {
+      log_debug(LD_GENERAL, "Shutting down %s", sys->name);
+      sys->shutdown();
+    }
+    sys_initialized[i] = false;
+  }
+}
+
+/**
+ * Run pre-fork code on all subsystems that declare any
+ **/
+void
+subsystems_prefork(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->prefork) {
+      log_debug(LD_GENERAL, "Pre-fork: %s", sys->name);
+      sys->prefork();
+    }
+  }
+}
+
+/**
+ * Run post-fork code on all subsystems that declare any
+ **/
+void
+subsystems_postfork(void)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->postfork) {
+      log_debug(LD_GENERAL, "Post-fork: %s", sys->name);
+      sys->postfork();
+    }
+  }
+}
+
+/**
+ * Run thread-cleanup code on all subsystems that declare any
+ **/
+void
+subsystems_thread_cleanup(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_initialized[i])
+      continue;
+    if (sys->thread_cleanup) {
+      log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name);
+      sys->thread_cleanup();
+    }
+  }
+}
diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h
new file mode 100644
index 0000000000..d4426614e3
--- /dev/null
+++ b/src/app/main/subsysmgr.h
@@ -0,0 +1,29 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SUBSYSMGR_T
+#define TOR_SUBSYSMGR_T
+
+#include "lib/subsys/subsys.h"
+
+extern const struct subsys_fns_t *tor_subsystems[];
+extern const unsigned n_tor_subsystems;
+
+int subsystems_init(void);
+int subsystems_init_upto(int level);
+
+struct pubsub_builder_t;
+int subsystems_add_pubsub_upto(struct pubsub_builder_t *builder,
+                               int target_level);
+int subsystems_add_pubsub(struct pubsub_builder_t *builder);
+
+void subsystems_shutdown(void);
+void subsystems_shutdown_downto(int level);
+
+void subsystems_prefork(void);
+void subsystems_postfork(void);
+void subsystems_thread_cleanup(void);
+
+#endif /* !defined(TOR_SUBSYSMGR_T) */
diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c
new file mode 100644
index 0000000000..1af9340c1a
--- /dev/null
+++ b/src/app/main/subsystem_list.c
@@ -0,0 +1,71 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+#include "core/mainloop/mainloop_sys.h"
+#include "core/or/ocirc_event_sys.h"
+#include "core/or/or_sys.h"
+#include "core/or/orconn_event_sys.h"
+#include "feature/control/btrack_sys.h"
+#include "feature/relay/relay_sys.h"
+#include "lib/compress/compress_sys.h"
+#include "lib/crypt_ops/crypto_sys.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/log/log_sys.h"
+#include "lib/net/network_sys.h"
+#include "lib/process/process_sys.h"
+#include "lib/process/winprocess_sys.h"
+#include "lib/thread/thread_sys.h"
+#include "lib/time/time_sys.h"
+#include "lib/tls/tortls_sys.h"
+#include "lib/wallclock/wallclock_sys.h"
+#include "lib/evloop/evloop_sys.h"
+
+#include "feature/dirauth/dirauth_sys.h"
+
+#include <stddef.h>
+
+/**
+ * Global list of the subsystems in Tor, in the order of their initialization.
+ * Want to know the exact level numbers?
+ * We'll implement a level dump command in #31614.
+ **/
+const subsys_fns_t *tor_subsystems[] = {
+  &sys_winprocess,
+  &sys_torerr,
+
+  &sys_wallclock,
+  &sys_threads,
+  &sys_logging,
+
+  &sys_time,
+  &sys_network,
+
+  &sys_compress,
+  &sys_crypto,
+  &sys_tortls,
+  &sys_process,
+
+  &sys_orconn_event,
+  &sys_ocirc_event,
+  &sys_btrack,
+
+  &sys_evloop,
+
+  &sys_mainloop,
+  &sys_or,
+
+  &sys_relay,
+
+#ifdef HAVE_MODULE_DIRAUTH
+  &sys_dirauth,
+#endif
+};
+
+const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems);
diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py
index 3a454a3fc1..b861e9433e 100644
--- a/src/config/mmdb-convert.py
+++ b/src/config/mmdb-convert.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/python
 
 #   This software has been dedicated to the public domain under the CC0
 #   public domain dedication.
@@ -77,7 +77,7 @@ def to_int32(s):
 
 def to_int28(s):
     "Parse a pair of big-endian 28-bit integers from bytestring s."
-    a, b = unpack("!LL", s + b'\x00')
+    a, b = struct.unpack("!LL", s + b'\x00')
     return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8)
 
 class Tree(object):
diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging
index cb3adca35c..90bad7f7cc 100644
--- a/src/config/torrc.minimal.in-staging
+++ b/src/config/torrc.minimal.in-staging
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in
index c2ae707e93..51e1c3af4b 100644
--- a/src/config/torrc.sample.in
+++ b/src/config/torrc.sample.in
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
@@ -174,13 +177,11 @@
 
 ## Uncomment this if you want your relay to be an exit, with the default
 ## exit policy (or whatever exit policy you set below).
-## (If ReducedExitPolicy or ExitPolicy are set, relays are exits.
-## If neither exit policy option is set, relays are non-exits.)
+## (If ReducedExitPolicy, ExitPolicy, or IPv6Exit are set, relays are exits.
+## If none of these options are set, relays are non-exits.)
 #ExitRelay 1
 
 ## Uncomment this if you want your relay to allow IPv6 exit traffic.
-## You must also set ExitRelay, ReducedExitPolicy, or ExitPolicy to make your
-## relay into an exit.
 ## (Relays do not allow any exit traffic by default.)
 #IPv6Exit 1
 
diff --git a/src/core/crypto/.may_include b/src/core/crypto/.may_include
new file mode 100644
index 0000000000..5782a36797
--- /dev/null
+++ b/src/core/crypto/.may_include
@@ -0,0 +1,10 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/cc/*.h
+lib/log/*.h
+
+core/crypto/*.h
diff --git a/src/core/crypto/hs_ntor.c b/src/core/crypto/hs_ntor.c
index c34073690e..add8a2b8f2 100644
--- a/src/core/crypto/hs_ntor.c
+++ b/src/core/crypto/hs_ntor.c
@@ -176,7 +176,6 @@ get_introduce1_key_material(const uint8_t *secret_input,
   uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN];
   uint8_t info_blob[INFO_BLOB_LEN];
   uint8_t kdf_input[KDF_INPUT_LEN];
-  crypto_xof_t *xof;
   uint8_t *ptr;
 
   /* Let's build info */
@@ -193,10 +192,8 @@ get_introduce1_key_material(const uint8_t *secret_input,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Now we need to run kdf_input over SHAKE-256 */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ;
-  crypto_xof_free(xof);
+  crypto_xof(keystream, sizeof(keystream),
+             kdf_input, sizeof(kdf_input));
 
   { /* Get the keys */
     memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN);
@@ -594,7 +591,6 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
 {
   uint8_t *ptr;
   uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN];
-  crypto_xof_t *xof;
 
   /* Sanity checks on lengths to make sure we are good */
   if (BUG(seed_len != DIGEST256_LEN)) {
@@ -611,10 +607,8 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Generate the keys */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN);
-  crypto_xof_free(xof);
+  crypto_xof(keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN,
+             kdf_input, sizeof(kdf_input));
 
   return 0;
 }
diff --git a/src/core/crypto/onion_crypto.h b/src/core/crypto/onion_crypto.h
index 1cddde3610..7abdd6538e 100644
--- a/src/core/crypto/onion_crypto.h
+++ b/src/core/crypto/onion_crypto.h
@@ -44,4 +44,4 @@ void server_onion_keys_free_(server_onion_keys_t *keys);
 #define server_onion_keys_free(keys) \
   FREE_AND_NULL(server_onion_keys_t, server_onion_keys_free_, (keys))
 
-#endif
+#endif /* !defined(TOR_ONION_CRYPTO_H) */
diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c
index 0b83b2d0a5..8a285131a8 100644
--- a/src/core/crypto/relay_crypto.c
+++ b/src/core/crypto/relay_crypto.c
@@ -6,12 +6,14 @@
 
 #include "core/or/or.h"
 #include "core/or/circuitlist.h"
+#include "core/or/crypt_path.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "core/crypto/hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
+#include "core/or/sendme.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
@@ -20,7 +22,7 @@
 /** Update digest from the payload of cell. Assign integrity part to
  * cell.
  */
-static void
+void
 relay_set_digest(crypto_digest_t *digest, cell_t *cell)
 {
   char integrity[4];
@@ -84,12 +86,39 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
  *
  * Note that we use the same operation for encrypting and for decrypting.
  */
-static void
+void
 relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in)
 {
   crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
 }
 
+/** Return the sendme_digest within the <b>crypto</b> object. */
+uint8_t *
+relay_crypto_get_sendme_digest(relay_crypto_t *crypto)
+{
+  tor_assert(crypto);
+  return crypto->sendme_digest;
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                  bool is_foward_digest)
+{
+  struct crypto_digest_t *digest;
+
+  tor_assert(crypto);
+
+  digest = crypto->b_digest;
+  if (is_foward_digest) {
+    digest = crypto->f_digest;
+  }
+
+  crypto_digest_get_digest(digest, (char *) crypto->sendme_digest,
+                           sizeof(crypto->sendme_digest));
+}
+
 /** Do the appropriate en/decryptions for <b>cell</b> arriving on
  * <b>circ</b> in direction <b>cell_direction</b>.
  *
@@ -134,12 +163,12 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell,
         tor_assert(thishop);
 
         /* decrypt one layer */
-        relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload);
+        cpath_crypt_cell(thishop, cell->payload, true);
 
         relay_header_unpack(&rh, cell->payload);
         if (rh.recognized == 0) {
           /* it's possibly recognized. have to check digest to be sure. */
-          if (relay_digest_matches(thishop->crypto.b_digest, cell)) {
+          if (relay_digest_matches(cpath_get_incoming_digest(thishop), cell)) {
             *recognized = 1;
             *layer_hint = thishop;
             return 0;
@@ -187,14 +216,17 @@ relay_encrypt_cell_outbound(cell_t *cell,
                             crypt_path_t *layer_hint)
 {
   crypt_path_t *thishop; /* counter for repeated crypts */
-  relay_set_digest(layer_hint->crypto.f_digest, cell);
+  cpath_set_cell_forward_digest(layer_hint, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(circ), layer_hint);
 
   thishop = layer_hint;
   /* moving from farthest to nearest hop */
   do {
     tor_assert(thishop);
     log_debug(LD_OR,"encrypting a layer of the relay cell.");
-    relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload);
+    cpath_crypt_cell(thishop, cell->payload, false);
 
     thishop = thishop->prev;
   } while (thishop != circ->cpath->prev);
@@ -212,6 +244,10 @@ relay_encrypt_cell_inbound(cell_t *cell,
                            or_circuit_t *or_circ)
 {
   relay_set_digest(or_circ->crypto.b_digest, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(or_circ), NULL);
+
   /* encrypt one layer */
   relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload);
 }
diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h
index 45a21d14ab..9478f8d359 100644
--- a/src/core/crypto/relay_crypto.h
+++ b/src/core/crypto/relay_crypto.h
@@ -27,5 +27,16 @@ void relay_crypto_clear(relay_crypto_t *crypto);
 
 void relay_crypto_assert_ok(const relay_crypto_t *crypto);
 
+uint8_t *relay_crypto_get_sendme_digest(relay_crypto_t *crypto);
+
+void relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                       bool is_foward_digest);
+
+void
+relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in);
+
+void
+relay_set_digest(crypto_digest_t *digest, cell_t *cell);
+
 #endif /* !defined(TOR_RELAY_CRYPTO_H) */
 
diff --git a/src/core/include.am b/src/core/include.am
index 1b8ef2ac58..9b4b251c81 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -6,11 +6,14 @@ noinst_LIBRARIES += \
 	src/core/libtor-app-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 LIBTOR_APP_A_SOURCES = 				\
 	src/app/config/config.c			\
-	src/app/config/confparse.c		\
 	src/app/config/statefile.c		\
 	src/app/main/main.c			\
+	src/app/main/shutdown.c			\
+	src/app/main/subsystem_list.c		\
+	src/app/main/subsysmgr.c		\
 	src/core/crypto/hs_ntor.c		\
 	src/core/crypto/onion_crypto.c		\
 	src/core/crypto/onion_fast.c		\
@@ -20,6 +23,8 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/mainloop/connection.c		\
 	src/core/mainloop/cpuworker.c		\
 	src/core/mainloop/mainloop.c		\
+	src/core/mainloop/mainloop_pubsub.c	\
+	src/core/mainloop/mainloop_sys.c	\
 	src/core/mainloop/netstatus.c		\
 	src/core/mainloop/periodic.c		\
 	src/core/or/address_set.c		\
@@ -30,13 +35,20 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/or/circuitlist.c		\
 	src/core/or/circuitmux.c		\
 	src/core/or/circuitmux_ewma.c		\
+	src/core/or/circuitpadding.c		\
+	src/core/or/circuitpadding_machines.c	\
 	src/core/or/circuitstats.c		\
 	src/core/or/circuituse.c		\
+	src/core/or/crypt_path.c		\
 	src/core/or/command.c			\
 	src/core/or/connection_edge.c		\
 	src/core/or/connection_or.c		\
 	src/core/or/dos.c			\
 	src/core/or/onion.c			\
+	src/core/or/ocirc_event.c		\
+	src/core/or/or_periodic.c		\
+	src/core/or/or_sys.c			\
+	src/core/or/orconn_event.c		\
 	src/core/or/policies.c			\
 	src/core/or/protover.c			\
 	src/core/or/protover_rust.c		\
@@ -45,6 +57,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/core/or/scheduler.c			\
 	src/core/or/scheduler_kist.c		\
 	src/core/or/scheduler_vanilla.c		\
+	src/core/or/sendme.c			\
 	src/core/or/status.c			\
 	src/core/or/versions.c			\
 	src/core/proto/proto_cell.c		\
@@ -59,10 +72,21 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/client/dnsserv.c		\
 	src/feature/client/entrynodes.c		\
 	src/feature/client/transports.c		\
+	src/feature/control/btrack.c		\
+	src/feature/control/btrack_circuit.c	\
+	src/feature/control/btrack_orconn.c	\
+	src/feature/control/btrack_orconn_cevent.c	\
+	src/feature/control/btrack_orconn_maps.c	\
 	src/feature/control/control.c		\
+	src/feature/control/control_auth.c	\
+	src/feature/control/control_bootstrap.c	\
+	src/feature/control/control_cmd.c	\
+	src/feature/control/control_events.c	\
+	src/feature/control/control_fmt.c	\
+	src/feature/control/control_getinfo.c	\
+	src/feature/control/control_proto.c	\
 	src/feature/control/fmt_serverstatus.c  \
 	src/feature/control/getinfo_geoip.c	\
-	src/feature/dirauth/keypin.c		\
 	src/feature/dircache/conscache.c	\
 	src/feature/dircache/consdiffmgr.c	\
 	src/feature/dircache/dircache.c		\
@@ -92,6 +116,7 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/hs/hs_config.c		\
 	src/feature/hs/hs_control.c		\
 	src/feature/hs/hs_descriptor.c		\
+	src/feature/hs/hs_dos.c			\
 	src/feature/hs/hs_ident.c		\
 	src/feature/hs/hs_intropoint.c		\
 	src/feature/hs/hs_service.c		\
@@ -99,13 +124,13 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/hs_common/replaycache.c	\
 	src/feature/hs_common/shared_random_client.c	\
 	src/feature/keymgt/loadkey.c		\
-	src/feature/dirauth/keypin.c		\
 	src/feature/nodelist/authcert.c		\
 	src/feature/nodelist/describe.c		\
 	src/feature/nodelist/dirlist.c		\
 	src/feature/nodelist/microdesc.c	\
 	src/feature/nodelist/networkstatus.c	\
 	src/feature/nodelist/nickname.c		\
+	src/feature/nodelist/nodefamily.c	\
 	src/feature/nodelist/nodelist.c		\
 	src/feature/nodelist/node_select.c	\
 	src/feature/nodelist/routerinfo.c	\
@@ -116,6 +141,8 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/relay/dns.c			\
 	src/feature/relay/ext_orport.c		\
 	src/feature/relay/onion_queue.c		\
+	src/feature/relay/relay_periodic.c	\
+	src/feature/relay/relay_sys.c		\
 	src/feature/relay/router.c		\
 	src/feature/relay/routerkeys.c		\
 	src/feature/relay/routermode.c		\
@@ -130,17 +157,6 @@ LIBTOR_APP_A_SOURCES = 				\
 	src/feature/stats/rephist.c		\
 	src/feature/stats/predict_ports.c
 
-# These should eventually move into module_dirauth_sources, but for now
-# the separation is only in the code location.
-LIBTOR_APP_A_SOURCES += 			\
-	src/feature/dirauth/bwauth.c		\
-	src/feature/dirauth/dsigs_parse.c	\
-	src/feature/dirauth/guardfraction.c	\
-	src/feature/dirauth/reachability.c	\
-	src/feature/dirauth/recommend_pkg.c	\
-	src/feature/dirauth/process_descs.c	\
-	src/feature/dirauth/voteflags.c
-
 if BUILD_NT_SERVICES
 LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
 endif
@@ -156,10 +172,21 @@ LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES)
 # The Directory Authority module.
 MODULE_DIRAUTH_SOURCES = 					\
 	src/feature/dirauth/authmode.c				\
+	src/feature/dirauth/bridgeauth.c			\
+	src/feature/dirauth/bwauth.c				\
+	src/feature/dirauth/dirauth_periodic.c			\
+	src/feature/dirauth/dirauth_sys.c			\
 	src/feature/dirauth/dircollate.c			\
 	src/feature/dirauth/dirvote.c				\
+	src/feature/dirauth/dsigs_parse.c			\
+	src/feature/dirauth/guardfraction.c			\
+	src/feature/dirauth/keypin.c				\
+	src/feature/dirauth/process_descs.c			\
+	src/feature/dirauth/reachability.c			\
+	src/feature/dirauth/recommend_pkg.c			\
 	src/feature/dirauth/shared_random.c			\
-	src/feature/dirauth/shared_random_state.c
+	src/feature/dirauth/shared_random_state.c		\
+	src/feature/dirauth/voteflags.c
 
 if BUILD_MODULE_DIRAUTH
 LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
@@ -183,14 +210,16 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\""		\
 src_core_libtor_app_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_core_libtor_app_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/app/config/config.h				\
-	src/app/config/confparse.h			\
 	src/app/config/or_options_st.h			\
 	src/app/config/or_state_st.h			\
 	src/app/config/statefile.h			\
 	src/app/main/main.h				\
 	src/app/main/ntmain.h				\
+	src/app/main/shutdown.h 			\
+	src/app/main/subsysmgr.h			\
 	src/core/crypto/hs_ntor.h			\
 	src/core/crypto/onion_crypto.h	        	\
 	src/core/crypto/onion_fast.h			\
@@ -200,6 +229,8 @@ noinst_HEADERS +=					\
 	src/core/mainloop/connection.h			\
 	src/core/mainloop/cpuworker.h			\
 	src/core/mainloop/mainloop.h			\
+	src/core/mainloop/mainloop_pubsub.h		\
+	src/core/mainloop/mainloop_sys.h		\
 	src/core/mainloop/netstatus.h			\
 	src/core/mainloop/periodic.h			\
 	src/core/or/addr_policy_st.h			\
@@ -215,28 +246,37 @@ noinst_HEADERS +=					\
 	src/core/or/circuitmux.h			\
 	src/core/or/circuitmux_ewma.h			\
 	src/core/or/circuitstats.h			\
+	src/core/or/circuitpadding.h			\
+	src/core/or/circuitpadding_machines.h		\
 	src/core/or/circuituse.h			\
 	src/core/or/command.h				\
 	src/core/or/connection_edge.h			\
 	src/core/or/connection_or.h			\
 	src/core/or/connection_st.h			\
+	src/core/or/crypt_path.h			\
 	src/core/or/cpath_build_state_st.h		\
 	src/core/or/crypt_path_reference_st.h		\
 	src/core/or/crypt_path_st.h			\
 	src/core/or/destroy_cell_queue_st.h		\
 	src/core/or/dos.h				\
 	src/core/or/edge_connection_st.h		\
-	src/core/or/half_edge_st.h		\
+	src/core/or/half_edge_st.h			\
 	src/core/or/entry_connection_st.h		\
 	src/core/or/entry_port_cfg_st.h			\
 	src/core/or/extend_info_st.h			\
 	src/core/or/listener_connection_st.h		\
 	src/core/or/onion.h				\
 	src/core/or/or.h				\
+	src/core/or/or_periodic.h			\
+	src/core/or/or_sys.h				\
+	src/core/or/orconn_event.h			\
+	src/core/or/orconn_event_sys.h			\
 	src/core/or/or_circuit_st.h			\
 	src/core/or/or_connection_st.h			\
 	src/core/or/or_handshake_certs_st.h		\
 	src/core/or/or_handshake_state_st.h		\
+	src/core/or/ocirc_event.h			\
+	src/core/or/ocirc_event_sys.h			\
 	src/core/or/origin_circuit_st.h			\
 	src/core/or/policies.h				\
 	src/core/or/port_cfg_st.h			\
@@ -245,6 +285,7 @@ noinst_HEADERS +=					\
 	src/core/or/relay.h				\
 	src/core/or/relay_crypto_st.h			\
 	src/core/or/scheduler.h				\
+	src/core/or/sendme.h				\
 	src/core/or/server_port_cfg_st.h		\
 	src/core/or/socks_request_st.h			\
 	src/core/or/status.h				\
@@ -263,12 +304,27 @@ noinst_HEADERS +=					\
 	src/feature/client/dnsserv.h			\
 	src/feature/client/entrynodes.h			\
 	src/feature/client/transports.h			\
+	src/feature/control/btrack_circuit.h		\
+	src/feature/control/btrack_orconn.h		\
+	src/feature/control/btrack_orconn_cevent.h	\
+	src/feature/control/btrack_orconn_maps.h	\
+	src/feature/control/btrack_sys.h		\
 	src/feature/control/control.h			\
+	src/feature/control/control_auth.h		\
+	src/feature/control/control_cmd.h		\
+	src/feature/control/control_cmd_args_st.h	\
 	src/feature/control/control_connection_st.h	\
+	src/feature/control/control_events.h		\
+	src/feature/control/control_fmt.h		\
+	src/feature/control/control_getinfo.h		\
+	src/feature/control/control_proto.h		\
 	src/feature/control/fmt_serverstatus.h		\
 	src/feature/control/getinfo_geoip.h		\
 	src/feature/dirauth/authmode.h			\
+        src/feature/dirauth/bridgeauth.h		\
 	src/feature/dirauth/bwauth.h			\
+	src/feature/dirauth/dirauth_periodic.h		\
+	src/feature/dirauth/dirauth_sys.h		\
 	src/feature/dirauth/dircollate.h		\
 	src/feature/dirauth/dirvote.h			\
 	src/feature/dirauth/dsigs_parse.h		\
@@ -317,6 +373,7 @@ noinst_HEADERS +=					\
 	src/feature/hs/hs_config.h			\
 	src/feature/hs/hs_control.h			\
 	src/feature/hs/hs_descriptor.h			\
+	src/feature/hs/hs_dos.h				\
 	src/feature/hs/hs_ident.h			\
 	src/feature/hs/hs_intropoint.h			\
 	src/feature/hs/hs_service.h			\
@@ -340,6 +397,8 @@ noinst_HEADERS +=					\
 	src/feature/nodelist/networkstatus_voter_info_st.h	\
 	src/feature/nodelist/nickname.h			\
 	src/feature/nodelist/node_st.h			\
+	src/feature/nodelist/nodefamily.h		\
+	src/feature/nodelist/nodefamily_st.h		\
 	src/feature/nodelist/nodelist.h			\
 	src/feature/nodelist/node_select.h		\
 	src/feature/nodelist/routerinfo.h		\
@@ -356,6 +415,8 @@ noinst_HEADERS +=					\
 	src/feature/relay/dns_structs.h			\
 	src/feature/relay/ext_orport.h			\
 	src/feature/relay/onion_queue.h			\
+	src/feature/relay/relay_periodic.h		\
+	src/feature/relay/relay_sys.h			\
 	src/feature/relay/router.h			\
 	src/feature/relay/routerkeys.h			\
 	src/feature/relay/routermode.h			\
@@ -374,9 +435,10 @@ noinst_HEADERS +=					\
 	src/feature/stats/rephist.h			\
 	src/feature/stats/predict_ports.h
 
-noinst_HEADERS +=			\
-	src/app/config/auth_dirs.inc	\
-	src/app/config/fallback_dirs.inc
+noinst_HEADERS +=				\
+	src/app/config/auth_dirs.inc		\
+	src/app/config/fallback_dirs.inc	\
+	src/app/config/testnet.inc
 
 # This may someday want to be an installed file?
 noinst_HEADERS += src/feature/api/tor_api.h
diff --git a/src/core/mainloop/.may_include b/src/core/mainloop/.may_include
new file mode 100644
index 0000000000..79d6a130a4
--- /dev/null
+++ b/src/core/mainloop/.may_include
@@ -0,0 +1,20 @@
+!advisory
+
+orconfig.h
+
+lib/container/*.h
+lib/dispatch/*.h
+lib/evloop/*.h
+lib/pubsub/*.h
+lib/subsys/*.h
+lib/buf/*.h
+lib/crypt_ops/*.h
+lib/err/*.h
+lib/tls/*.h
+lib/net/*.h
+lib/evloop/*.h
+lib/geoip/*.h
+lib/sandbox/*.h
+lib/compress/*.h
+
+core/mainloop/*.h
\ No newline at end of file
diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c
index 3595bba85c..50cd3810a4 100644
--- a/src/core/mainloop/connection.c
+++ b/src/core/mainloop/connection.c
@@ -57,7 +57,7 @@
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/err/backtrace.h"
 
@@ -82,12 +82,14 @@
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/dnsserv.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
@@ -696,6 +698,7 @@ connection_free_minimal(connection_t *conn)
     control_connection_t *control_conn = TO_CONTROL_CONN(conn);
     tor_free(control_conn->safecookie_client_hash);
     tor_free(control_conn->incoming_cmd);
+    tor_free(control_conn->current_cmd);
     if (control_conn->ephemeral_onion_services) {
       SMARTLIST_FOREACH(control_conn->ephemeral_onion_services, char *, cp, {
         memwipe(cp, 0, strlen(cp));
@@ -1195,7 +1198,7 @@ make_win32_socket_exclusive(tor_socket_t sock)
     return -1;
   }
   return 0;
-#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */
+#else /* !defined(SO_EXCLUSIVEADDRUSE) */
   (void) sock;
   return 0;
 #endif /* defined(SO_EXCLUSIVEADDRUSE) */
@@ -1466,6 +1469,20 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                tor_socket_strerror(tor_socket_errno(s)));
       goto err;
     }
+
+#ifndef __APPLE__
+    /* This code was introduced to help debug #28229. */
+    int value;
+    socklen_t len = sizeof(value);
+
+    if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) {
+      if (value == 0) {
+        log_err(LD_NET, "Could not listen on %s - "
+                        "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address);
+        goto err;
+      }
+    }
+#endif /* !defined(__APPLE__) */
 #endif /* defined(HAVE_SYS_UN_H) */
   } else {
     log_err(LD_BUG, "Got unexpected address family %d.",
@@ -1644,7 +1661,7 @@ check_sockaddr(const struct sockaddr *sa, int len, int level)
              len,(int)sizeof(struct sockaddr_in6));
       ok = 0;
     }
-    if (tor_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
+    if (fast_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
         sin6->sin6_port == 0) {
       log_fn(level, LD_NET,
              "Address for new connection has address/port equal to zero.");
@@ -1867,7 +1884,7 @@ connection_init_accepted_conn(connection_t *conn,
       /* Initiate Extended ORPort authentication. */
       return connection_ext_or_start_auth(TO_OR_CONN(conn));
     case CONN_TYPE_OR:
-      control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
+      connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
       rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1);
       if (rv < 0) {
         connection_or_close_for_error(TO_OR_CONN(conn), 0);
@@ -1880,6 +1897,9 @@ connection_init_accepted_conn(connection_t *conn,
       TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch();
       TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type;
 
+      /* Any incoming connection on an entry port counts as user activity. */
+      note_user_activity(approx_time());
+
       switch (TO_CONN(listener)->type) {
         case CONN_TYPE_AP_LISTENER:
           conn->state = AP_CONN_STATE_SOCKS_WAIT;
@@ -2079,6 +2099,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn)
     return;
   }
 
+  if (fascist_firewall_use_ipv6(options)) {
+    log_info(LD_NET, "Our outgoing connection is using IPv%d.",
+             tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4);
+  }
+
   /* Check if we couldn't satisfy an address family preference */
   if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6)
       || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) {
@@ -2837,7 +2862,7 @@ retry_listener_ports(smartlist_t *old_conns,
           SMARTLIST_DEL_CURRENT(old_conns, conn);
           break;
         }
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
       }
     } SMARTLIST_FOREACH_END(wanted);
 
@@ -2901,6 +2926,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
     retval = -1;
 
 #ifdef ENABLE_LISTENER_REBIND
+  if (smartlist_len(replacements))
+    log_debug(LD_NET, "%d replacements - starting rebinding loop.",
+              smartlist_len(replacements));
+
   SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) {
     int addr_in_use = 0;
     int skip = 0;
@@ -2912,8 +2941,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
       connection_listener_new_for_port(r->new_port, &skip, &addr_in_use);
     connection_t *old_conn = r->old_conn;
 
-    if (skip)
+    if (skip) {
+      log_debug(LD_NET, "Skipping creating new listener for %s:%d",
+                old_conn->address, old_conn->port);
       continue;
+    }
 
     connection_close_immediate(old_conn);
     connection_mark_for_close(old_conn);
@@ -2932,7 +2964,7 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
                conn_type_to_string(old_conn->type), old_conn->address,
                old_conn->port, new_conn->address, new_conn->port);
   } SMARTLIST_FOREACH_END(r);
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
 
   /* Any members that were still in 'listeners' don't correspond to
    * any configured port.  Kill 'em. */
@@ -3019,7 +3051,7 @@ connection_mark_all_noncontrol_connections(void)
  * uses pluggable transports, since we should then limit it even if it
  * comes from an internal IP address. */
 static int
-connection_is_rate_limited(connection_t *conn)
+connection_is_rate_limited(const connection_t *conn)
 {
   const or_options_t *options = get_options();
   if (conn->linked)
@@ -3154,14 +3186,14 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
                                      global_bucket_val, conn_bucket);
 }
 
-/** Return 1 if the global write buckets are low enough that we
+/** Return true iff the global write buckets are low enough that we
  * shouldn't send <b>attempt</b> bytes of low-priority directory stuff
- * out to <b>conn</b>. Else return 0.
-
- * Priority was 1 for v1 requests (directories and running-routers),
- * and 2 for v2 requests and later (statuses and descriptors).
+ * out to <b>conn</b>.
+ *
+ * If we are a directory authority, always answer dir requests thus true is
+ * always returned.
  *
- * There are a lot of parameters we could use here:
+ * Note: There are a lot of parameters we could use here:
  * - global_relayed_write_bucket. Low is bad.
  * - global_write_bucket. Low is bad.
  * - bandwidthrate. Low is bad.
@@ -3173,39 +3205,40 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
  *   mean is "total directory bytes added to outbufs recently", but
  *   that's harder to quantify and harder to keep track of.
  */
-int
-global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
+bool
+connection_dir_is_global_write_low(const connection_t *conn, size_t attempt)
 {
   size_t smaller_bucket =
     MIN(token_bucket_rw_get_write(&global_bucket),
         token_bucket_rw_get_write(&global_relayed_bucket));
-  if (authdir_mode(get_options()) && priority>1)
-    return 0; /* there's always room to answer v2 if we're an auth dir */
+
+  /* Special case for authorities (directory only). */
+  if (authdir_mode_v3(get_options())) {
+    /* Are we configured to possibly reject requests under load? */
+    if (!get_options()->AuthDirRejectRequestsUnderLoad) {
+      /* Answer request no matter what. */
+      return false;
+    }
+    /* Always answer requests from a known relay which includes the other
+     * authorities. The following looks up the addresses for relays that we
+     * have their descriptor _and_ any configured trusted directories. */
+    if (nodelist_probably_contains_address(&conn->addr)) {
+      return false;
+    }
+  }
 
   if (!connection_is_rate_limited(conn))
-    return 0; /* local conns don't get limited */
+    return false; /* local conns don't get limited */
 
   if (smaller_bucket < attempt)
-    return 1; /* not enough space no matter the priority */
+    return true; /* not enough space. */
 
   {
     const time_t diff = approx_time() - write_buckets_last_empty_at;
     if (diff <= 1)
-      return 1; /* we're already hitting our limits, no more please */
+      return true; /* we're already hitting our limits, no more please */
   }
-
-  if (priority == 1) { /* old-style v1 query */
-    /* Could we handle *two* of these requests within the next two seconds? */
-    const or_options_t *options = get_options();
-    size_t can_write = (size_t) (smaller_bucket
-      + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate :
-           options->BandwidthRate));
-    if (can_write < 2*attempt)
-      return 1;
-  } else { /* v2 query */
-    /* no further constraints yet */
-  }
-  return 0;
+  return false;
 }
 
 /** When did we last tell the accounting subsystem about transmitted
@@ -3931,9 +3964,9 @@ update_send_buffer_size(tor_socket_t sock)
       &isb, sizeof(isb), &bytesReturned, NULL, NULL)) {
     setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb));
   }
-#else
+#else /* !defined(_WIN32) */
   (void) sock;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Try to flush more bytes onto <b>conn</b>-\>s.
@@ -4331,6 +4364,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
   connection_write_to_buf_commit(conn, written);
 }
 
+/**
+ * Write a <b>string</b> (of size <b>len</b> to directory connection
+ * <b>dir_conn</b>. Apply compression if connection is configured to use
+ * it and finalize it if <b>done</b> is true.
+ */
+void
+connection_dir_buf_add(const char *string, size_t len,
+                       dir_connection_t *dir_conn, int done)
+{
+  if (dir_conn->compress_state != NULL) {
+    connection_buf_add_compress(string, len, dir_conn, done);
+    return;
+  }
+
+  connection_buf_add(string, len, TO_CONN(dir_conn));
+}
+
 void
 connection_buf_add_compress(const char *string, size_t len,
                             dir_connection_t *conn, int done)
@@ -4445,6 +4495,16 @@ connection_get_by_type_state(int type, int state)
   CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state);
 }
 
+/**
+ * Return a connection of type <b>type</b> that is not an internally linked
+ * connection, and is not marked for close.
+ **/
+MOCK_IMPL(connection_t *,
+connection_get_by_type_nonlinked,(int type))
+{
+  CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked);
+}
+
 /** Return a connection of type <b>type</b> that has rendquery equal
  * to <b>rendquery</b>, and that is not marked for close. If state
  * is non-zero, conn must be of that state too.
@@ -5287,7 +5347,7 @@ assert_connection_ok(connection_t *conn, time_t now)
         tor_assert(entry_conn->socks_request->has_finished);
         if (!conn->marked_for_close) {
           tor_assert(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
-          assert_cpath_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
+          cpath_assert_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
         }
       }
     }
@@ -5341,17 +5401,20 @@ assert_connection_ok(connection_t *conn, time_t now)
 }
 
 /** Fills <b>addr</b> and <b>port</b> with the details of the global
- *  proxy server we are using.
- *  <b>conn</b> contains the connection we are using the proxy for.
+ *  proxy server we are using. Store a 1 to the int pointed to by
+ *  <b>is_put_out</b> if the connection is using a pluggable
+ *  transport; store 0 otherwise. <b>conn</b> contains the connection
+ *  we are using the proxy for.
  *
  *  Return 0 on success, -1 on failure.
  */
 int
 get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                   const connection_t *conn)
+                   int *is_pt_out, const connection_t *conn)
 {
   const or_options_t *options = get_options();
 
+  *is_pt_out = 0;
   /* Client Transport Plugins can use another proxy, but that should be hidden
    * from the rest of tor (as the plugin is responsible for dealing with the
    * proxy), check it first, then check the rest of the proxy types to allow
@@ -5367,6 +5430,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
       tor_addr_copy(addr, &transport->addr);
       *port = transport->port;
       *proxy_type = transport->socks_version;
+      *is_pt_out = 1;
       return 0;
     }
 
@@ -5403,11 +5467,13 @@ log_failed_proxy_connection(connection_t *conn)
 {
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt;
 
-  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, conn) != 0)
+  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         conn) != 0)
     return; /* if we have no proxy set up, leave this function. */
 
+  (void)is_pt;
   log_warn(LD_NET,
            "The connection to the %s proxy server at %s just failed. "
            "Make sure that the proxy server is up and running.",
diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h
index 8ecdd6f06f..668c740042 100644
--- a/src/core/mainloop/connection.h
+++ b/src/core/mainloop/connection.h
@@ -187,7 +187,7 @@ int connection_proxy_connect(connection_t *conn, int type);
 int connection_read_proxy_handshake(connection_t *conn);
 void log_failed_proxy_connection(connection_t *conn);
 int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                       const connection_t *conn);
+                       int *is_pt_out, const connection_t *conn);
 
 int retry_all_listeners(smartlist_t *new_conns,
                         int close_all_noncontrol);
@@ -195,8 +195,9 @@ int retry_all_listeners(smartlist_t *new_conns,
 void connection_mark_all_noncontrol_listeners(void);
 void connection_mark_all_noncontrol_connections(void);
 
-ssize_t connection_bucket_write_limit(connection_t *conn, time_t now);
-int global_write_bucket_low(connection_t *conn, size_t attempt, int priority);
+ssize_t connection_bucket_write_limit(struct connection_t *conn, time_t now);
+bool connection_dir_is_global_write_low(const struct connection_t *conn,
+                                        size_t attempt);
 void connection_bucket_init(void);
 void connection_bucket_adjust(const or_options_t *options);
 void connection_bucket_refill_all(time_t now,
@@ -226,6 +227,8 @@ MOCK_DECL(void, connection_write_to_buf_impl_,
 /* DOCDOC connection_write_to_buf */
 static void connection_buf_add(const char *string, size_t len,
                                     connection_t *conn);
+void connection_dir_buf_add(const char *string, size_t len,
+                            dir_connection_t *dir_conn, int done);
 static inline void
 connection_buf_add(const char *string, size_t len, connection_t *conn)
 {
@@ -240,6 +243,7 @@ size_t connection_get_outbuf_len(connection_t *conn);
 connection_t *connection_get_by_global_id(uint64_t id);
 
 connection_t *connection_get_by_type(int type);
+MOCK_DECL(connection_t *,connection_get_by_type_nonlinked,(int type));
 MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type,
                                                   const tor_addr_t *addr,
                                                   uint16_t port, int purpose));
diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c
index e704d55642..436fcd28c3 100644
--- a/src/core/mainloop/cpuworker.c
+++ b/src/core/mainloop/cpuworker.c
@@ -34,7 +34,6 @@
 #include "core/crypto/onion_crypto.h"
 
 #include "core/or/or_circuit_st.h"
-#include "lib/intmath/weakrng.h"
 
 static void queue_pending_tasks(void);
 
@@ -74,8 +73,6 @@ worker_state_free_void(void *arg)
 static replyqueue_t *replyqueue = NULL;
 static threadpool_t *threadpool = NULL;
 
-static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT;
-
 static int total_pending_tasks = 0;
 static int max_pending_tasks = 128;
 
@@ -109,7 +106,6 @@ cpu_init(void)
 
   /* Total voodoo. Can we make this more sensible? */
   max_pending_tasks = get_num_cpus(get_options()) * 64;
-  crypto_seed_weak_rng(&request_sample_rng);
 }
 
 /** Magic numbers to make sure our cpuworker_requests don't grow any
@@ -235,9 +231,10 @@ should_time_request(uint16_t onionskin_type)
    * sample */
   if (onionskins_n_processed[onionskin_type] < 4096)
     return 1;
+
   /** Otherwise, measure with P=1/128.  We avoid doing this for every
    * handshake, since the measurement itself can take a little time. */
-  return tor_weak_random_one_in_n(&request_sample_rng, 128);
+  return crypto_fast_rng_one_in_n(get_thread_fast_rng(), 128);
 }
 
 /** Return an estimate of how many microseconds we will need for a single
diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c
index f0aa37e8da..c47e440774 100644
--- a/src/core/mainloop/mainloop.c
+++ b/src/core/mainloop/mainloop.c
@@ -73,8 +73,8 @@
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
-#include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
@@ -95,7 +95,7 @@
 #include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/err/backtrace.h"
 #include "lib/tls/buffers_tls.h"
@@ -105,9 +105,6 @@
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/authmode.h"
-
 #include "core/or/cell_st.h"
 #include "core/or/entry_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -200,12 +197,10 @@ static int can_complete_circuits = 0;
 #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60)
 
 static int conn_close_if_marked(int i);
-static int run_main_loop_until_done(void);
 static void connection_start_reading_from_linked_conn(connection_t *conn);
 static int connection_should_read_from_linked_conn(connection_t *conn);
 static void conn_read_callback(evutil_socket_t fd, short event, void *_conn);
 static void conn_write_callback(evutil_socket_t fd, short event, void *_conn);
-static void second_elapsed_callback(periodic_timer_t *timer, void *args);
 static void shutdown_did_not_work_callback(evutil_socket_t fd, short event,
                                            void *arg) ATTR_NORETURN;
 
@@ -759,7 +754,7 @@ tor_shutdown_event_loop_for_restart_cb(
   tor_event_free(tor_shutdown_event_loop_for_restart_event);
   tor_shutdown_event_loop_and_exit(0);
 }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
 /**
  * After finishing the current callback (if any), shut down the main loop,
@@ -1361,119 +1356,92 @@ static int periodic_events_initialized = 0;
 #define CALLBACK(name) \
   static int name ## _callback(time_t, const or_options_t *)
 CALLBACK(add_entropy);
-CALLBACK(check_authority_cert);
-CALLBACK(check_canonical_channels);
-CALLBACK(check_descriptor);
-CALLBACK(check_dns_honesty);
-CALLBACK(check_ed_keys);
 CALLBACK(check_expired_networkstatus);
-CALLBACK(check_for_reachability_bw);
-CALLBACK(check_onion_keys_expiry_time);
 CALLBACK(clean_caches);
 CALLBACK(clean_consdiffmgr);
-CALLBACK(dirvote);
-CALLBACK(downrate_stability);
-CALLBACK(expire_old_ciruits_serverside);
 CALLBACK(fetch_networkstatus);
 CALLBACK(heartbeat);
 CALLBACK(hs_service);
 CALLBACK(launch_descriptor_fetches);
-CALLBACK(launch_reachability_tests);
-CALLBACK(reachability_warnings);
+CALLBACK(prune_old_routers);
 CALLBACK(record_bridge_stats);
 CALLBACK(rend_cache_failure_clean);
 CALLBACK(reset_padding_counts);
-CALLBACK(retry_dns);
 CALLBACK(retry_listeners);
-CALLBACK(rotate_onion_key);
 CALLBACK(rotate_x509_certificate);
-CALLBACK(save_stability);
 CALLBACK(save_state);
-CALLBACK(write_bridge_ns);
 CALLBACK(write_stats_file);
+CALLBACK(control_per_second_events);
+CALLBACK(second_elapsed);
 
 #undef CALLBACK
 
 /* Now we declare an array of periodic_event_item_t for each periodic event */
-#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f)
-
-STATIC periodic_event_item_t periodic_events[] = {
-  /* Everyone needs to run those. */
-  CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0),
-
-  /* Routers (bridge and relay) only. */
-  CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0),
-
-  /* Authorities (bridge and directory) only. */
-  CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-  CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-
-  /* Directory authority only. */
-  CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0),
-  CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET),
-
-  /* Relay only. */
-  CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+#define CALLBACK(name, r, f) \
+  PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f)
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+
+STATIC periodic_event_item_t mainloop_periodic_events[] = {
+
+  /* Everyone needs to run these. They need to have very long timeouts for
+   * that to be safe. */
+  CALLBACK(add_entropy, ALL, 0),
+  CALLBACK(heartbeat, ALL, 0),
+  CALLBACK(reset_padding_counts, ALL, 0),
+
+  /* This is a legacy catch-all callback that runs once per second if
+   * we are online and active. */
+  CALLBACK(second_elapsed, NET_PARTICIPANT,
+           FL(RUN_ON_DISABLE)),
+
+  /* XXXX Do we have a reason to do this on a callback? Does it do any good at
+   * all?  For now, if we're dormant, we can let our listeners decay. */
+  CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)),
+
+  /* We need to do these if we're participating in the Tor network. */
+  CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)),
+  CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0),
+  CALLBACK(check_network_participation, NET_PARTICIPANT, 0),
+
+  /* We need to do these if we're participating in the Tor network, and
+   * immediately before we stop. */
+  CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Hidden Service service only. */
-  CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more
 
   /* Bridge only. */
-  CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0),
+  CALLBACK(record_bridge_stats, BRIDGE, 0),
 
   /* Client only. */
-  CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0),
-
-  /* Bridge Authority only. */
-  CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0),
+  /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */
+  CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Directory server only. */
-  CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0),
+  CALLBACK(clean_consdiffmgr, DIRSERVER, 0),
+
+  /* Controller with per-second events only. */
+  CALLBACK(control_per_second_events, CONTROLEV, 0),
 
   END_OF_PERIODIC_EVENTS
 };
 #undef CALLBACK
+#undef FL
 
 /* These are pointers to members of periodic_events[] that are used to
  * implement particular callbacks.  We keep them separate here so that we
  * can access them by name.  We also keep them inside periodic_events[]
  * so that we can implement "reset all timers" in a reasonable way. */
-static periodic_event_item_t *check_descriptor_event=NULL;
-static periodic_event_item_t *dirvote_event=NULL;
 static periodic_event_item_t *fetch_networkstatus_event=NULL;
 static periodic_event_item_t *launch_descriptor_fetches_event=NULL;
 static periodic_event_item_t *check_dns_honesty_event=NULL;
 static periodic_event_item_t *save_state_event=NULL;
+static periodic_event_item_t *prune_old_routers_event=NULL;
 
 /** Reset all the periodic events so we'll do all our actions again as if we
  * just started up.
@@ -1483,24 +1451,7 @@ static periodic_event_item_t *save_state_event=NULL;
 void
 reset_all_main_loop_timers(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_reschedule(&periodic_events[i]);
-  }
-}
-
-/** Return the member of periodic_events[] whose name is <b>name</b>.
- * Return NULL if no such event is found.
- */
-static periodic_event_item_t *
-find_periodic_event(const char *name)
-{
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    if (strcmp(name, periodic_events[i].name) == 0)
-      return &periodic_events[i];
-  }
-  return NULL;
+  periodic_events_reset_all();
 }
 
 /** Return a bitmask of the roles this tor instance is configured for using
@@ -1510,7 +1461,7 @@ get_my_roles(const or_options_t *options)
 {
   tor_assert(options);
 
-  int roles = 0;
+  int roles = PERIODIC_EVENT_ROLE_ALL;
   int is_bridge = options->BridgeRelay;
   int is_relay = server_mode(options);
   int is_dirauth = authdir_mode_v3(options);
@@ -1518,6 +1469,8 @@ get_my_roles(const or_options_t *options)
   int is_hidden_service = !!hs_service_get_num_services() ||
                           !!rend_num_services();
   int is_dirserver = dir_server_mode(options);
+  int sending_control_events = control_any_per_second_event_enabled();
+
   /* We also consider tor to have the role of a client if the ControlPort is
    * set because a lot of things can be done over the control port which
    * requires tor to have basic functionnalities. */
@@ -1525,6 +1478,9 @@ get_my_roles(const or_options_t *options)
                   options->ControlPort_set ||
                   options->OwningControllerFD != UINT64_MAX;
 
+  int is_net_participant = is_participating_on_network() ||
+    is_relay || is_hidden_service;
+
   if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE;
   if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT;
   if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY;
@@ -1532,6 +1488,8 @@ get_my_roles(const or_options_t *options)
   if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH;
   if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE;
   if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER;
+  if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV;
 
   return roles;
 }
@@ -1556,9 +1514,9 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data)
   rescan_periodic_events(get_options());
 }
 
-/** Set up all the members of periodic_events[], and configure them all to be
- * launched from a callback. */
-STATIC void
+/** Set up all the members of mainloop_periodic_events[], and configure them
+ * all to be launched from a callback. */
+void
 initialize_periodic_events(void)
 {
   if (periodic_events_initialized)
@@ -1566,39 +1524,58 @@ initialize_periodic_events(void)
 
   periodic_events_initialized = 1;
 
-  /* Set up all periodic events. We'll launch them by roles. */
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_setup(&periodic_events[i]);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_events_register(&mainloop_periodic_events[i]);
   }
 
+  /* Set up all periodic events. We'll launch them by roles. */
+
 #define NAMED_CALLBACK(name) \
-  STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END
+  STMT_BEGIN name ## _event = periodic_events_find( #name ); STMT_END
 
-  NAMED_CALLBACK(check_descriptor);
-  NAMED_CALLBACK(dirvote);
+  NAMED_CALLBACK(prune_old_routers);
   NAMED_CALLBACK(fetch_networkstatus);
   NAMED_CALLBACK(launch_descriptor_fetches);
   NAMED_CALLBACK(check_dns_honesty);
   NAMED_CALLBACK(save_state);
-
-  struct timeval one_second = { 1, 0 };
-  initialize_periodic_events_event = tor_evtimer_new(
-                  tor_libevent_get_base(),
-                  initialize_periodic_events_cb, NULL);
-  event_add(initialize_periodic_events_event, &one_second);
 }
 
 STATIC void
 teardown_periodic_events(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_destroy(&periodic_events[i]);
-  }
+  periodic_events_disconnect_all();
+  fetch_networkstatus_event = NULL;
+  launch_descriptor_fetches_event = NULL;
+  check_dns_honesty_event = NULL;
+  save_state_event = NULL;
+  prune_old_routers_event = NULL;
   periodic_events_initialized = 0;
 }
 
+static mainloop_event_t *rescan_periodic_events_ev = NULL;
+
+/** Callback: rescan the periodic event list. */
+static void
+rescan_periodic_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+  rescan_periodic_events(get_options());
+}
+
+/**
+ * Schedule an event that will rescan which periodic events should run.
+ **/
+MOCK_IMPL(void,
+schedule_rescan_periodic_events,(void))
+{
+  if (!rescan_periodic_events_ev) {
+    rescan_periodic_events_ev =
+      mainloop_event_new(rescan_periodic_events_cb, NULL);
+  }
+  mainloop_event_activate(rescan_periodic_events_ev);
+}
+
 /** Do a pass at all our periodic events, disable those we don't need anymore
  * and enable those we need now using the given options. */
 void
@@ -1606,36 +1583,7 @@ rescan_periodic_events(const or_options_t *options)
 {
   tor_assert(options);
 
-  /* Avoid scanning the event list if we haven't initialized it yet. This is
-   * particularly useful for unit tests in order to avoid initializing main
-   * loop events everytime. */
-  if (!periodic_events_initialized) {
-    return;
-  }
-
-  int roles = get_my_roles(options);
-
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-
-    int enable = !!(item->roles & roles);
-
-    /* Handle the event flags. */
-    if (net_is_disabled() &&
-        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
-      enable = 0;
-    }
-
-    /* Enable the event if needed. It is safe to enable an event that was
-     * already enabled. Same goes for disabling it. */
-    if (enable) {
-      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
-      periodic_event_enable(item);
-    } else {
-      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
-      periodic_event_disable(item);
-    }
-  }
+  periodic_events_rescan_by_roles(get_my_roles(options), net_is_disabled());
 }
 
 /* We just got new options globally set, see if we need to enabled or disable
@@ -1643,26 +1591,7 @@ rescan_periodic_events(const or_options_t *options)
 void
 periodic_events_on_new_options(const or_options_t *options)
 {
-  /* Only if we've already initialized the events, rescan the list which will
-   * enable or disable events depending on our roles. This will be called at
-   * bootup and we don't want this function to initialize the events because
-   * they aren't set up at this stage. */
-  if (periodic_events_initialized) {
-    rescan_periodic_events(options);
-  }
-}
-
-/**
- * Update our schedule so that we'll check whether we need to update our
- * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
- * seconds.
- */
-void
-reschedule_descriptor_update_check(void)
-{
-  if (check_descriptor_event) {
-    periodic_event_reschedule(check_descriptor_event);
-  }
+  rescan_periodic_events(options);
 }
 
 /**
@@ -1708,40 +1637,41 @@ mainloop_schedule_postloop_cleanup(void)
   mainloop_event_activate(postloop_cleanup_ev);
 }
 
-#define LONGEST_TIMER_PERIOD (30 * 86400)
-/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
- * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */
-static inline int
-safe_timer_diff(time_t now, time_t next)
-{
-  if (next > now) {
-    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
-     * and nothing that could run Tor. It's a bug if 'next' is around then.
-     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
-     * Bang. We cannot extrapolate past a singularity, but there was probably
-     * nothing that could run Tor then, either.
-     **/
-    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
-
-    if (next - LONGEST_TIMER_PERIOD > now)
-      return LONGEST_TIMER_PERIOD;
-    return (int)(next - now);
-  } else {
-    return 1;
+/** Event to run 'scheduled_shutdown_cb' */
+static mainloop_event_t *scheduled_shutdown_ev=NULL;
+
+/** Callback: run a scheduled shutdown */
+static void
+scheduled_shutdown_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+  tor_shutdown_event_loop_and_exit(0);
+}
+
+/** Schedule the mainloop to exit after <b>delay_sec</b> seconds. */
+void
+mainloop_schedule_shutdown(int delay_sec)
+{
+  const struct timeval delay_tv = { delay_sec, 0 };
+  if (! scheduled_shutdown_ev) {
+    scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL);
   }
+  mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv);
 }
 
 /** Perform regular maintenance tasks.  This function gets run once per
- * second by second_elapsed_callback().
+ * second.
  */
-static void
-run_scheduled_events(time_t now)
+static int
+second_elapsed_callback(time_t now, const or_options_t *options)
 {
-  const or_options_t *options = get_options();
-
-  /* 0. See if we've been asked to shut down and our timeout has
-   * expired; or if our bandwidth limits are exhausted and we
-   * should hibernate; or if it's time to wake up from hibernation.
+  /* 0. See if our bandwidth limits are exhausted and we should hibernate
+   *
+   * Note: we have redundant mechanisms to handle the case where it's
+   * time to wake up from hibernation; or where we have a scheduled
+   * shutdown and it's time to run it, but this will also handle those.
    */
   consider_hibernation(now);
 
@@ -1751,10 +1681,13 @@ run_scheduled_events(time_t now)
   if (options->UseBridges && !net_is_disabled()) {
     /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches()
      * -- the latter is only for fetching consensus-derived directory info. */
+    // TODO: client
+    //     Also, schedule this rather than probing 1x / sec
     fetch_bridge_descriptors(options, now);
   }
 
   if (accounting_is_enabled(options)) {
+    // TODO: refactor or rewrite?
     accounting_run_housekeeping(now);
   }
 
@@ -1765,6 +1698,7 @@ run_scheduled_events(time_t now)
    */
   /* (If our circuit build timeout can ever become lower than a second (which
    * it can't, currently), we should do this more often.) */
+  // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE
   circuit_expire_building();
   circuit_expire_waiting_for_better_guard();
 
@@ -1798,80 +1732,8 @@ run_scheduled_events(time_t now)
     run_connection_housekeeping(i, now);
   }
 
-  /* 11b. check pending unconfigured managed proxies */
-  if (!net_is_disabled() && pt_proxies_configuration_pending())
-    pt_configure_remaining_proxies();
-}
-
-/* Periodic callback: rotate the onion keys after the period defined by the
- * "onion-key-rotation-days" consensus parameter, shut down and restart all
- * cpuworkers, and update our descriptor if necessary.
- */
-static int
-rotate_onion_key_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_lifetime = get_onion_key_lifetime();
-    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
-    if (rotation_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL,"Rotating onion key.");
-    rotate_onion_key();
-    cpuworkers_rotate_keyinfo();
-    if (router_rebuild_descriptor(1)<0) {
-      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
-    }
-    if (advertised_server_mode() && !net_is_disabled())
-      router_upload_dir_desc_to_dirservers(0);
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Period callback: Check if our old onion keys are still valid after the
- * period of time defined by the consensus parameter
- * "onion-key-grace-period-days", otherwise expire them by setting them to
- * NULL.
- */
-static int
-check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_grace_period = get_onion_key_grace_period();
-    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
-    if (expiry_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL, "Expiring old onion keys.");
-    expire_old_onion_keys();
-    cpuworkers_rotate_keyinfo();
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Periodic callback: Every 30 seconds, check whether it's time to make new
- * Ed25519 subkeys.
- */
-static int
-check_ed_keys_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    if (should_make_new_ed_keys(options, now)) {
-      int new_signing_key = load_ed_keys(options, now);
-      if (new_signing_key < 0 ||
-          generate_ed_link_cert(options, now, new_signing_key > 0)) {
-        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
-        tor_shutdown_event_loop_and_exit(1);
-      }
-    }
-    return 30;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
+  /* Run again in a second. */
+  return 1;
 }
 
 /**
@@ -1947,100 +1809,53 @@ add_entropy_callback(time_t now, const or_options_t *options)
   return ENTROPY_INTERVAL;
 }
 
-/**
- * Periodic callback: if we're an authority, make sure we test
- * the routers on the network for reachability.
- */
-static int
-launch_reachability_tests_callback(time_t now, const or_options_t *options)
+/** Periodic callback: if there has been no network usage in a while,
+ * enter a dormant state. */
+STATIC int
+check_network_participation_callback(time_t now, const or_options_t *options)
 {
-  if (authdir_mode_tests_reachability(options) &&
-      !net_is_disabled()) {
-    /* try to determine reachability of the other Tor relays */
-    dirserv_test_reachability(now);
+  /* If we're a server, we can't become dormant. */
+  if (server_mode(options)) {
+    goto found_activity;
   }
-  return REACHABILITY_TEST_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, discount the stability
- * information (and other rephist information) that's older.
- */
-static int
-downrate_stability_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* 1d. Periodically, we discount older stability information so that new
-   * stability info counts more, and save the stability information to disk as
-   * appropriate. */
-  time_t next = rep_hist_downrate_old_runs(now);
-  return safe_timer_diff(now, next);
-}
+  /* If we're running an onion service, we can't become dormant. */
+  /* XXXX this would be nice to change, so that we can be dormant with a
+   * service. */
+  if (hs_service_get_num_services() || rend_num_services()) {
+    goto found_activity;
+  }
 
-/**
- * Periodic callback: if we're an authority, record our measured stability
- * information from rephist in an mtbf file.
- */
-static int
-save_stability_callback(time_t now, const or_options_t *options)
-{
-  if (authdir_mode_tests_reachability(options)) {
-    if (rep_hist_record_mtbf_data(now, 1)<0) {
-      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+  /* If we have any currently open entry streams other than "linked"
+   * connections used for directory requests, those count as user activity.
+   */
+  if (options->DormantTimeoutDisabledByIdleStreams) {
+    if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) {
+      goto found_activity;
     }
   }
-#define SAVE_STABILITY_INTERVAL (30*60)
-  return SAVE_STABILITY_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, check on our authority
- * certificate (the one that authenticates our authority signing key).
- */
-static int
-check_authority_cert_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  (void)options;
-  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
-   * close to expiring and warn the admin if it is. */
-  v3_authority_check_key_expiry();
-#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
-  return CHECK_V3_CERTIFICATE_INTERVAL;
-}
+  /* XXXX Make this configurable? */
+/** How often do we check whether we have had network activity? */
+#define CHECK_PARTICIPATION_INTERVAL (5*60)
 
-/**
- * Scheduled callback: Run directory-authority voting functionality.
- *
- * The schedule is a bit complicated here, so dirvote_act() manages the
- * schedule itself.
- **/
-static int
-dirvote_callback(time_t now, const or_options_t *options)
-{
-  if (!authdir_mode_v3(options)) {
-    tor_assert_nonfatal_unreached();
-    return 3600;
+  /* Become dormant if there has been no user activity in a long time.
+   * (The funny checks below are in order to prevent overflow.) */
+  time_t time_since_last_activity = 0;
+  if (get_last_user_activity_time() < now)
+    time_since_last_activity = now - get_last_user_activity_time();
+  if (time_since_last_activity >= options->DormantClientTimeout) {
+    log_notice(LD_GENERAL, "No user activity in a long time: becoming"
+               " dormant.");
+    set_network_participation(false);
+    rescan_periodic_events(options);
   }
 
-  time_t next = dirvote_act(options, now);
-  if (BUG(next == TIME_MAX)) {
-    /* This shouldn't be returned unless we called dirvote_act() without
-     * being an authority.  If it happens, maybe our configuration will
-     * fix itself in an hour or so? */
-    return 3600;
-  }
-  return safe_timer_diff(now, next);
-}
+  return CHECK_PARTICIPATION_INTERVAL;
 
-/** Reschedule the directory-authority voting event.  Run this whenever the
- * schedule has changed. */
-void
-reschedule_dirvote(const or_options_t *options)
-{
-  if (periodic_events_initialized && authdir_mode_v3(options)) {
-    periodic_event_reschedule(dirvote_event);
-  }
+ found_activity:
+  note_user_activity(now);
+  return CHECK_PARTICIPATION_INTERVAL;
 }
 
 /**
@@ -2053,11 +1868,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   (void)options;
   /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
-  /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
-   * networkstatus_get_reasonably_live_consensus(), but that value is way
-   * way too high.  Arma: is the bridge issue there resolved yet? -NM */
-#define NS_EXPIRY_SLOP (24*60*60)
-  if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) &&
+  /* Use reasonably live consensuses until they are no longer reasonably live.
+   */
+  if (ns && !networkstatus_consensus_reasonably_live(ns, now) &&
       router_have_minimum_dir_info()) {
     router_dir_info_changed();
   }
@@ -2143,17 +1956,6 @@ write_stats_file_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next_time_to_write_stats_files);
 }
 
-#define CHANNEL_CHECK_INTERVAL (60*60)
-static int
-check_canonical_channels_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  if (public_server_mode(options))
-    channel_check_for_duplicates();
-
-  return CHANNEL_CHECK_INTERVAL;
-}
-
 static int
 reset_padding_counts_callback(time_t now, const or_options_t *options)
 {
@@ -2228,87 +2030,24 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
 }
 
 /**
- * Periodic callback: If we're a server and initializing dns failed, retry.
+ * Periodic callback: prune routerlist of old information about Tor network.
  */
 static int
-retry_dns_callback(time_t now, const or_options_t *options)
+prune_old_routers_callback(time_t now, const or_options_t *options)
 {
+#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour.
   (void)now;
-#define RETRY_DNS_INTERVAL (10*60)
-  if (server_mode(options) && has_dns_init_failed())
-    dns_init();
-  return RETRY_DNS_INTERVAL;
-}
-
-/** Periodic callback: consider rebuilding or and re-uploading our descriptor
- * (if we've passed our internal checks). */
-static int
-check_descriptor_callback(time_t now, const or_options_t *options)
-{
-/** How often do we check whether part of our router info has changed in a
- * way that would require an upload? That includes checking whether our IP
- * address has changed. */
-#define CHECK_DESCRIPTOR_INTERVAL (60)
-
   (void)options;
 
-  /* 2b. Once per minute, regenerate and upload the descriptor if the old
-   * one is inaccurate. */
   if (!net_is_disabled()) {
-    check_descriptor_bandwidth_changed(now);
-    check_descriptor_ipaddress_changed(now);
-    mark_my_descriptor_dirty_if_too_old(now);
-    consider_publishable_server(0);
     /* If any networkstatus documents are no longer recent, we need to
      * update all the descriptors' running status. */
     /* Remove dead routers. */
-    /* XXXX This doesn't belong here, but it was here in the pre-
-     * XXXX refactoring code. */
+    log_debug(LD_GENERAL, "Pruning routerlist...");
     routerlist_remove_old_routers();
   }
 
-  return CHECK_DESCRIPTOR_INTERVAL;
-}
-
-/**
- * Periodic callback: check whether we're reachable (as a relay), and
- * whether our bandwidth has changed enough that we need to
- * publish a new descriptor.
- */
-static int
-check_for_reachability_bw_callback(time_t now, const or_options_t *options)
-{
-  /* XXXX This whole thing was stuck in the middle of what is now
-   * XXXX check_descriptor_callback.  I'm not sure it's right. */
-
-  static int dirport_reachability_count = 0;
-  /* also, check religiously for reachability, if it's within the first
-   * 20 minutes of our uptime. */
-  if (server_mode(options) &&
-      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
-      !net_is_disabled()) {
-    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-      router_do_reachability_checks(1, dirport_reachability_count==0);
-      if (++dirport_reachability_count > 5)
-        dirport_reachability_count = 0;
-      return 1;
-    } else {
-      /* If we haven't checked for 12 hours and our bandwidth estimate is
-       * low, do another bandwidth test. This is especially important for
-       * bridges, since they might go long periods without much use. */
-      const routerinfo_t *me = router_get_my_routerinfo();
-      static int first_time = 1;
-      if (!first_time && me &&
-          me->bandwidthcapacity < me->bandwidthrate &&
-          me->bandwidthcapacity < 51200) {
-        reset_bandwidth_test();
-      }
-      first_time = 0;
-#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
-      return BANDWIDTH_RECHECK_INTERVAL;
-    }
-  }
-  return CHECK_DESCRIPTOR_INTERVAL;
+  return ROUTERLIST_PRUNING_INTERVAL;
 }
 
 /**
@@ -2353,109 +2092,6 @@ retry_listeners_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
-/**
- * Periodic callback: as a server, see if we have any old unused circuits
- * that should be expired */
-static int
-expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* every 11 seconds, so not usually the same second as other such events */
-  circuit_expire_old_circuits_serverside(now);
-  return 11;
-}
-
-/**
- * Callback: Send warnings if Tor doesn't find its ports reachable.
- */
-static int
-reachability_warnings_callback(time_t now, const or_options_t *options)
-{
-  (void) now;
-
-  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
-  }
-
-  if (server_mode(options) &&
-      !net_is_disabled() &&
-      have_completed_a_circuit()) {
-    /* every 20 minutes, check and complain if necessary */
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (me && !check_whether_orport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
-               "its ORPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->or_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
-                                  address, me->or_port);
-      tor_free(address);
-    }
-
-    if (me && !check_whether_dirport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,
-               "Your server (%s:%d) has not managed to confirm that its "
-               "DirPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->dir_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
-                                  address, me->dir_port);
-      tor_free(address);
-    }
-  }
-
-  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
-}
-
-static int dns_honesty_first_time = 1;
-
-/**
- * Periodic event: if we're an exit, see if our DNS server is telling us
- * obvious lies.
- */
-static int
-check_dns_honesty_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  /* 9. and if we're an exit node, check whether our DNS is telling stories
-   * to us. */
-  if (net_is_disabled() ||
-      ! public_server_mode(options) ||
-      router_my_exit_policy_is_reject_star())
-    return PERIODIC_EVENT_NO_UPDATE;
-
-  if (dns_honesty_first_time) {
-    /* Don't launch right when we start */
-    dns_honesty_first_time = 0;
-    return crypto_rand_int_range(60, 180);
-  }
-
-  dns_launch_correctness_checks();
-  return 12*3600 + crypto_rand_int(12*3600);
-}
-
-/**
- * Periodic callback: if we're the bridge authority, write a networkstatus
- * file to disk.
- */
-static int
-write_bridge_ns_callback(time_t now, const or_options_t *options)
-{
-  /* 10. write bridge networkstatus file to disk */
-  if (options->BridgeAuthoritativeDir) {
-    networkstatus_dump_bridge_status_to_file(now);
-#define BRIDGE_STATUSFILE_INTERVAL (30*60)
-     return BRIDGE_STATUSFILE_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
 static int heartbeat_callback_first_time = 1;
 
 /**
@@ -2522,36 +2158,19 @@ hs_service_callback(time_t now, const or_options_t *options)
   return 1;
 }
 
-/** Timer: used to invoke second_elapsed_callback() once per second. */
-static periodic_timer_t *second_timer = NULL;
-
-/**
- * Enable or disable the per-second timer as appropriate, creating it if
- * necessary.
+/*
+ * Periodic callback: Send once-per-second events to the controller(s).
+ * This is called every second.
  */
-void
-reschedule_per_second_timer(void)
+static int
+control_per_second_events_callback(time_t now, const or_options_t *options)
 {
-  struct timeval one_second;
-  one_second.tv_sec = 1;
-  one_second.tv_usec = 0;
-
-  if (! second_timer) {
-    second_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &one_second,
-                                      second_elapsed_callback,
-                                      NULL);
-    tor_assert(second_timer);
-  }
+  (void) options;
+  (void) now;
 
-  const bool run_per_second_events =
-    control_any_per_second_event_enabled() || ! net_is_completely_disabled();
+  control_per_second_events();
 
-  if (run_per_second_events) {
-    periodic_timer_launch(second_timer, &one_second);
-  } else {
-    periodic_timer_disable(second_timer);
-  }
+  return 1;
 }
 
 /** Last time that update_current_time was called. */
@@ -2581,6 +2200,17 @@ update_current_time(time_t now)
   memcpy(&last_updated, &current_second_last_changed, sizeof(last_updated));
   monotime_coarse_get(&current_second_last_changed);
 
+  /** How much clock jumping means that we should adjust our idea of when
+   * to go dormant? */
+#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20
+
+  /* Don't go dormant early or late just because we jumped in time. */
+  if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) {
+    if (is_participating_on_network()) {
+      netstatus_note_clock_jumped(seconds_elapsed);
+    }
+  }
+
   /** How much clock jumping do we tolerate? */
 #define NUM_JUMPED_SECONDS_BEFORE_WARN 100
 
@@ -2590,6 +2220,7 @@ update_current_time(time_t now)
   if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) {
     // moving back in time is always a bad sign.
     circuit_note_clock_jumped(seconds_elapsed, false);
+
   } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
     /* Compare the monotonic clock to the result of time(). */
     const int32_t monotime_msec_passed =
@@ -2619,31 +2250,6 @@ update_current_time(time_t now)
   current_second = now;
 }
 
-/** Libevent callback: invoked once every second. */
-static void
-second_elapsed_callback(periodic_timer_t *timer, void *arg)
-{
-  /* XXXX This could be sensibly refactored into multiple callbacks, and we
-   * could use Libevent's timers for this rather than checking the current
-   * time against a bunch of timeouts every second. */
-  time_t now;
-  (void)timer;
-  (void)arg;
-
-  now = time(NULL);
-
-  /* We don't need to do this once-per-second any more: time-updating is
-   * only in this callback _because it is a callback_. It should be fine
-   * to disable this callback, and the time will still get updated.
-   */
-  update_current_time(now);
-
-  /* Maybe some controller events are ready to fire */
-  control_per_second_events();
-
-  run_scheduled_events(now);
-}
-
 #ifdef HAVE_SYSTEMD_209
 static periodic_timer_t *systemd_watchdog_timer = NULL;
 
@@ -2701,8 +2307,7 @@ dns_servers_relaunch_checks(void)
 {
   if (server_mode(get_options())) {
     dns_reset_correctness_checks();
-    if (periodic_events_initialized) {
-      tor_assert(check_dns_honesty_event);
+    if (check_dns_honesty_event) {
       periodic_event_reschedule(check_dns_honesty_event);
     }
   }
@@ -2712,8 +2317,6 @@ dns_servers_relaunch_checks(void)
 void
 initialize_mainloop_events(void)
 {
-  initialize_periodic_events();
-
   if (!schedule_active_linked_connections_event) {
     schedule_active_linked_connections_event =
       mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL);
@@ -2731,11 +2334,16 @@ do_main_loop(void)
   /* initialize the periodic events first, so that code that depends on the
    * events being present does not assert.
    */
-  initialize_periodic_events();
+  tor_assert(periodic_events_initialized);
   initialize_mainloop_events();
 
-  /* set up once-a-second callback. */
-  reschedule_per_second_timer();
+  periodic_events_connect_all();
+
+  struct timeval one_second = { 1, 0 };
+  initialize_periodic_events_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  initialize_periodic_events_cb, NULL);
+  event_add(initialize_periodic_events_event, &one_second);
 
 #ifdef HAVE_SYSTEMD_209
   uint64_t watchdog_delay;
@@ -2758,10 +2366,6 @@ do_main_loop(void)
     }
   }
 #endif /* defined(HAVE_SYSTEMD_209) */
-
-  main_loop_should_exit = 0;
-  main_loop_exit_value = 0;
-
 #ifdef ENABLE_RESTART_DEBUGGING
   {
     static int first_time = 1;
@@ -2785,7 +2389,7 @@ do_main_loop(void)
       event_add(tor_shutdown_event_loop_for_restart_event, &restart_after);
     }
   }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
   return run_main_loop_until_done();
 }
@@ -2887,10 +2491,14 @@ run_main_loop_once(void)
  *
  * Shadow won't invoke this function, so don't fill it up with things.
  */
-static int
+STATIC int
 run_main_loop_until_done(void)
 {
   int loop_result = 1;
+
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
+
   do {
     loop_result = run_main_loop_once();
   } while (loop_result == 1);
@@ -2921,7 +2529,6 @@ tor_mainloop_free_all(void)
   smartlist_free(connection_array);
   smartlist_free(closeable_connection_lst);
   smartlist_free(active_linked_connection_lst);
-  periodic_timer_free(second_timer);
   teardown_periodic_events();
   tor_event_free(shutdown_did_not_work_event);
   tor_event_free(initialize_periodic_events_event);
@@ -2929,6 +2536,8 @@ tor_mainloop_free_all(void)
   mainloop_event_free(schedule_active_linked_connections_event);
   mainloop_event_free(postloop_cleanup_ev);
   mainloop_event_free(handle_deferred_signewnym_ev);
+  mainloop_event_free(scheduled_shutdown_ev);
+  mainloop_event_free(rescan_periodic_events_ev);
 
 #ifdef HAVE_SYSTEMD_209
   periodic_timer_free(systemd_watchdog_timer);
@@ -2948,7 +2557,6 @@ tor_mainloop_free_all(void)
   can_complete_circuits = 0;
   quiet_level = 0;
   should_init_bridge_stats = 1;
-  dns_honesty_first_time = 1;
   heartbeat_callback_first_time = 1;
   current_second = 0;
   memset(&current_second_last_changed, 0,
diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h
index c5669fc4e0..caef736c15 100644
--- a/src/core/mainloop/mainloop.h
+++ b/src/core/mainloop/mainloop.h
@@ -59,12 +59,11 @@ void directory_info_has_arrived(time_t now, int from_cache, int suppress_logs);
 void ip_address_changed(int at_interface);
 void dns_servers_relaunch_checks(void);
 void reset_all_main_loop_timers(void);
-void reschedule_descriptor_update_check(void);
 void reschedule_directory_downloads(void);
 void reschedule_or_state_save(void);
-void reschedule_dirvote(const or_options_t *options);
 void mainloop_schedule_postloop_cleanup(void);
 void rescan_periodic_events(const or_options_t *options);
+MOCK_DECL(void, schedule_rescan_periodic_events,(void));
 
 void update_current_time(time_t now);
 
@@ -81,13 +80,15 @@ uint64_t get_main_loop_error_count(void);
 uint64_t get_main_loop_idle_count(void);
 
 void periodic_events_on_new_options(const or_options_t *options);
-void reschedule_per_second_timer(void);
 
 void do_signewnym(time_t);
 time_t get_last_signewnym_time(void);
 
+void mainloop_schedule_shutdown(int delay_sec);
+
 void tor_init_connection_lists(void);
 void initialize_mainloop_events(void);
+void initialize_periodic_events(void);
 void tor_mainloop_free_all(void);
 
 struct token_bucket_rw_t;
@@ -98,17 +99,20 @@ extern struct token_bucket_rw_t global_bucket;
 extern struct token_bucket_rw_t global_relayed_bucket;
 
 #ifdef MAINLOOP_PRIVATE
+STATIC int run_main_loop_until_done(void);
 STATIC void close_closeable_connections(void);
-STATIC void initialize_periodic_events(void);
 STATIC void teardown_periodic_events(void);
 STATIC int get_my_roles(const or_options_t *);
+STATIC int check_network_participation_callback(time_t now,
+                                                const or_options_t *options);
+
 #ifdef TOR_UNIT_TESTS
 extern smartlist_t *connection_array;
 
 /* We need the periodic_event_item_t definition. */
 #include "core/mainloop/periodic.h"
-extern periodic_event_item_t periodic_events[];
-#endif
-#endif /* defined(MAIN_PRIVATE) */
+extern periodic_event_item_t mainloop_periodic_events[];
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(MAINLOOP_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_MAINLOOP_H) */
diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c
new file mode 100644
index 0000000000..53275d8119
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.c
@@ -0,0 +1,170 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/pubsub/pubsub_build.h"
+
+/**
+ * Dispatcher to use for delivering messages.
+ **/
+static dispatch_t *the_dispatcher = NULL;
+static pubsub_items_t *the_pubsub_items = NULL;
+/**
+ * A list of mainloop_event_t, indexed by channel ID, to flush the messages
+ * on a channel.
+ **/
+static smartlist_t *alert_events = NULL;
+
+/**
+ * Mainloop event callback: flush all the messages in a channel.
+ *
+ * The channel is encoded as a pointer, and passed via arg.
+ **/
+static void
+flush_channel_event(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  if (!the_dispatcher)
+    return;
+
+  channel_id_t chan = (channel_id_t)(uintptr_t)(arg);
+  dispatch_flush(the_dispatcher, chan, INT_MAX);
+}
+
+/**
+ * Construct our global pubsub object from <b>builder</b>. Return 0 on
+ * success, -1 on failure. */
+int
+tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder)
+{
+  int rv = -1;
+  tor_mainloop_disconnect_pubsub();
+
+  the_dispatcher = pubsub_builder_finalize(builder, &the_pubsub_items);
+  if (! the_dispatcher)
+    goto err;
+
+  rv = 0;
+  goto done;
+ err:
+  tor_mainloop_disconnect_pubsub();
+ done:
+  return rv;
+}
+
+/**
+ * Install libevent events for all of the pubsub channels.
+ *
+ * Invoke this after tor_mainloop_connect_pubsub, and after libevent has been
+ * initialized.
+ */
+void
+tor_mainloop_connect_pubsub_events(void)
+{
+  tor_assert(the_dispatcher);
+  tor_assert(! alert_events);
+
+  const size_t num_channels = get_num_channel_ids();
+  alert_events = smartlist_new();
+  for (size_t i = 0; i < num_channels; ++i) {
+    smartlist_add(alert_events,
+                  mainloop_event_postloop_new(flush_channel_event,
+                                              (void*)(uintptr_t)(i)));
+  }
+}
+
+/**
+ * Dispatch alertfn callback: do nothing. Implements DELIV_NEVER.
+ **/
+static void
+alertfn_never(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  (void)arg;
+}
+
+/**
+ * Dispatch alertfn callback: activate a mainloop event. Implements
+ * DELIV_PROMPT.
+ **/
+static void
+alertfn_prompt(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  mainloop_event_t *event = arg;
+  mainloop_event_activate(event);
+}
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Set the strategy to be used for delivering messages on the named channel.
+ *
+ * This function needs to be called once globally for each channel, to
+ * set up how messages are delivered.
+ **/
+int
+tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                   deliv_strategy_t strategy)
+{
+  channel_id_t chan = get_channel_id(msg_channel_name);
+  if (BUG(chan == ERROR_ID) ||
+      BUG(chan >= smartlist_len(alert_events)))
+    return -1;
+
+  switch (strategy) {
+    case DELIV_NEVER:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_never, NULL);
+      break;
+    case DELIV_PROMPT:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_prompt,
+                            smartlist_get(alert_events, chan));
+      break;
+    case DELIV_IMMEDIATE:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_immediate, NULL);
+      break;
+  }
+  return 0;
+}
+
+/**
+ * Remove all pubsub dispatchers and events from the mainloop.
+ **/
+void
+tor_mainloop_disconnect_pubsub(void)
+{
+  if (the_pubsub_items) {
+    pubsub_items_clear_bindings(the_pubsub_items);
+    pubsub_items_free(the_pubsub_items);
+  }
+  if (alert_events) {
+    SMARTLIST_FOREACH(alert_events, mainloop_event_t *, ev,
+                      mainloop_event_free(ev));
+    smartlist_free(alert_events);
+  }
+  dispatch_free(the_dispatcher);
+}
diff --git a/src/core/mainloop/mainloop_pubsub.h b/src/core/mainloop/mainloop_pubsub.h
new file mode 100644
index 0000000000..365a3dd565
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_MAINLOOP_PUBSUB_H
+#define TOR_MAINLOOP_PUBSUB_H
+
+struct pubsub_builder_t;
+
+typedef enum {
+   DELIV_NEVER=0,
+   DELIV_PROMPT,
+   DELIV_IMMEDIATE,
+} deliv_strategy_t;
+
+int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder);
+void tor_mainloop_connect_pubsub_events(void);
+int tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                        deliv_strategy_t strategy);
+void tor_mainloop_disconnect_pubsub(void);
+
+#endif /* !defined(TOR_MAINLOOP_PUBSUB_H) */
diff --git a/src/core/mainloop/mainloop_sys.c b/src/core/mainloop/mainloop_sys.c
new file mode 100644
index 0000000000..fbd5a40327
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.c
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop_sys.h"
+#include "core/mainloop/mainloop.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_mainloop_initialize(void)
+{
+  initialize_periodic_events();
+  return 0;
+}
+
+static void
+subsys_mainloop_shutdown(void)
+{
+  tor_mainloop_free_all();
+}
+
+const struct subsys_fns_t sys_mainloop = {
+  .name = "mainloop",
+  .supported = true,
+  .level = 5,
+  .initialize = subsys_mainloop_initialize,
+  .shutdown = subsys_mainloop_shutdown,
+};
diff --git a/src/core/mainloop/mainloop_sys.h b/src/core/mainloop/mainloop_sys.h
new file mode 100644
index 0000000000..fa74fe5d4b
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef MAINLOOP_SYS_H
+#define MAINLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_mainloop;
+
+#endif /* !defined(MAINLOOP_SYS_H) */
diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c
index 1444ca5db2..4924888598 100644
--- a/src/core/mainloop/netstatus.c
+++ b/src/core/mainloop/netstatus.c
@@ -6,9 +6,12 @@
 
 #include "core/or/or.h"
 #include "core/mainloop/netstatus.h"
+#include "core/mainloop/mainloop.h"
 #include "app/config/config.h"
 #include "feature/hibernate/hibernate.h"
 
+#include "app/config/or_state_st.h"
+
 /** Return true iff our network is in some sense disabled or shutting down:
  * either we're hibernating, entering hibernation, or the network is turned
  * off with DisableNetwork. */
@@ -26,3 +29,136 @@ net_is_completely_disabled(void)
 {
   return get_options()->DisableNetwork || we_are_fully_hibernating();
 }
+
+/**
+ * The time at which we've last seen "user activity" -- that is, any activity
+ * that should keep us as a participant on the network.
+ *
+ * This is not actually the true time.  We will adjust this forward if
+ * our clock jumps, or if Tor is shut down for a while, so that the time
+ * since our last activity remains as it was before the jump or shutdown.
+ */
+static time_t last_user_activity_seen = 0;
+
+/**
+ * True iff we are currently a "network participant" -- that is, we
+ * are building circuits, fetching directory information, and so on.
+ **/
+static bool participating_on_network = false;
+
+/**
+ * Record the fact that we have seen "user activity" at the time now.  Move
+ * "last activity seen" time forwards, but never backwards.
+ *
+ * If we were previously not participating on the network, set our
+ * participation status to true, and launch periodic events as appropriate.
+ **/
+void
+note_user_activity(time_t now)
+{
+  last_user_activity_seen = MAX(now, last_user_activity_seen);
+
+  if (! participating_on_network) {
+    log_notice(LD_GENERAL, "Tor is no longer dormant.");
+    set_network_participation(true);
+    schedule_rescan_periodic_events();
+  }
+}
+
+/**
+ * Change the time at which "user activitiy" was last seen to <b>now</b>.
+ *
+ * Unlike note_user_actity, this function sets the time without checking
+ * whether it is in the past, and without causing any rescan of periodic events
+ * or change in participation status.
+ */
+void
+reset_user_activity(time_t now)
+{
+  last_user_activity_seen = now;
+}
+
+/**
+ * Return the most recent time at which we recorded "user activity".
+ **/
+time_t
+get_last_user_activity_time(void)
+{
+  return last_user_activity_seen;
+}
+
+/**
+ * Set the field that remembers whether we are currently participating on the
+ * network.  Does not schedule or un-schedule periodic events.
+ **/
+void
+set_network_participation(bool participation)
+{
+  participating_on_network = participation;
+}
+
+/**
+ * Return true iff we are currently participating on the network.
+ **/
+bool
+is_participating_on_network(void)
+{
+  return participating_on_network;
+}
+
+/**
+ * Update 'state' with the last time at which we were active on the network.
+ **/
+void
+netstatus_flush_to_state(or_state_t *state, time_t now)
+{
+  state->Dormant = ! participating_on_network;
+  if (participating_on_network) {
+    time_t sec_since_activity = MAX(0, now - last_user_activity_seen);
+    state->MinutesSinceUserActivity = (int)(sec_since_activity / 60);
+  } else {
+    state->MinutesSinceUserActivity = 0;
+  }
+}
+
+/**
+ * Update our current view of network participation from an or_state_t object.
+ **/
+void
+netstatus_load_from_state(const or_state_t *state, time_t now)
+{
+  time_t last_activity;
+  if (state->Dormant == -1) { // Initial setup.
+    if (get_options()->DormantOnFirstStartup) {
+      last_activity = 0;
+      participating_on_network = false;
+    } else {
+      // Start up as active, treat activity as happening now.
+      last_activity = now;
+      participating_on_network = true;
+    }
+  } else if (state->Dormant) {
+    last_activity = 0;
+    participating_on_network = false;
+  } else {
+    last_activity = now - 60 * state->MinutesSinceUserActivity;
+    participating_on_network = true;
+  }
+  if (get_options()->DormantCanceledByStartup) {
+    last_activity = now;
+    participating_on_network = true;
+  }
+  reset_user_activity(last_activity);
+}
+
+/**
+ * Adjust the time at which the user was last active by <b>seconds_diff</b>
+ * in response to a clock jump.
+ */
+void
+netstatus_note_clock_jumped(time_t seconds_diff)
+{
+  time_t last_active = get_last_user_activity_time();
+  if (last_active)
+    reset_user_activity(last_active + seconds_diff);
+}
diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h
index ae8547b30d..e8469ff558 100644
--- a/src/core/mainloop/netstatus.h
+++ b/src/core/mainloop/netstatus.h
@@ -10,4 +10,15 @@
 int net_is_disabled(void);
 int net_is_completely_disabled(void);
 
-#endif
+void note_user_activity(time_t now);
+void reset_user_activity(time_t now);
+time_t get_last_user_activity_time(void);
+
+void set_network_participation(bool participation);
+bool is_participating_on_network(void);
+
+void netstatus_flush_to_state(or_state_t *state, time_t now);
+void netstatus_load_from_state(const or_state_t *state, time_t now);
+void netstatus_note_clock_jumped(time_t seconds_diff);
+
+#endif /* !defined(TOR_NETSTATUS_H) */
diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c
index 2651bbbc89..5c2f6f2b36 100644
--- a/src/core/mainloop/periodic.c
+++ b/src/core/mainloop/periodic.c
@@ -6,9 +6,22 @@
  *
  * \brief Generic backend for handling periodic events.
  *
- * The events in this module are used by main.c to track items that need
+ * The events in this module are used to track items that need
  * to fire once every N seconds, possibly picking a new interval each time
- * that they fire.  See periodic_events[] in main.c for examples.
+ * that they fire.  See periodic_events[] in mainloop.c for examples.
+ *
+ * This module manages a global list of periodic_event_item_t objects,
+ * each corresponding to a single event.  To register an event, pass it to
+ * periodic_events_register() when initializing your subsystem.
+ *
+ * Registering an event makes the periodic event subsystem know about it, but
+ * doesn't cause the event to get created immediately.  Before the event can
+ * be started, periodic_event_connect_all() must be called by mainloop.c to
+ * connect all the events to Libevent.
+ *
+ * We expect that periodic_event_item_t objects will be statically allocated;
+ * we set them up and tear them down here, but we don't take ownership of
+ * them.
  */
 
 #include "core/or/or.h"
@@ -24,6 +37,12 @@
  */
 static const int MAX_INTERVAL = 10 * 365 * 86400;
 
+/**
+ * Global list of periodic events that have been registered with
+ * <b>periodic_event_register</a>.
+ **/
+static smartlist_t *the_periodic_events = NULL;
+
 /** Set the event <b>event</b> to run in <b>next_interval</b> seconds from
  * now. */
 static void
@@ -45,10 +64,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   periodic_event_item_t *event = data;
   tor_assert(ev == event->ev);
 
-  if (BUG(!periodic_event_is_enabled(event))) {
-    return;
-  }
-
   time_t now = time(NULL);
   update_current_time(now);
   const or_options_t *options = get_options();
@@ -57,7 +72,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   int next_interval = 0;
 
   if (!periodic_event_is_enabled(event)) {
-    /* The event got disabled from inside its callback; no need to
+    /* The event got disabled from inside its callback, or before: no need to
      * reschedule. */
     return;
   }
@@ -91,15 +106,16 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
 void
 periodic_event_reschedule(periodic_event_item_t *event)
 {
-  /* Don't reschedule a disabled event. */
-  if (periodic_event_is_enabled(event)) {
+  /* Don't reschedule a disabled or uninitialized event. */
+  if (event->ev && periodic_event_is_enabled(event)) {
     periodic_event_set_interval(event, 1);
   }
 }
 
-/** Initializes the libevent backend for a periodic event. */
+/** Connects a periodic event to the Libevent backend.  Does not launch the
+ * event immediately. */
 void
-periodic_event_setup(periodic_event_item_t *event)
+periodic_event_connect(periodic_event_item_t *event)
 {
   if (event->ev) { /* Already setup? This is a bug */
     log_err(LD_BUG, "Initial dispatch should only be done once.");
@@ -117,7 +133,7 @@ void
 periodic_event_launch(periodic_event_item_t *event)
 {
   if (! event->ev) { /* Not setup? This is a bug */
-    log_err(LD_BUG, "periodic_event_launch without periodic_event_setup");
+    log_err(LD_BUG, "periodic_event_launch without periodic_event_connect");
     tor_assert(0);
   }
   /* Event already enabled? This is a bug */
@@ -131,9 +147,9 @@ periodic_event_launch(periodic_event_item_t *event)
   periodic_event_dispatch(event->ev, event);
 }
 
-/** Release all storage associated with <b>event</b> */
-void
-periodic_event_destroy(periodic_event_item_t *event)
+/** Disconnect and unregister the periodic event in <b>event</b> */
+static void
+periodic_event_disconnect(periodic_event_item_t *event)
 {
   if (!event)
     return;
@@ -177,3 +193,177 @@ periodic_event_disable(periodic_event_item_t *event)
   mainloop_event_cancel(event->ev);
   event->enabled = 0;
 }
+
+/**
+ * Disable an event, then schedule it to run once.
+ * Do nothing if the event was already disabled.
+ */
+void
+periodic_event_schedule_and_disable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  if (!periodic_event_is_enabled(event))
+    return;
+
+  periodic_event_disable(event);
+
+  mainloop_event_activate(event->ev);
+}
+
+/**
+ * Add <b>item</b> to the list of periodic events.
+ *
+ * Note that <b>item</b> should be statically allocated: we do not
+ * take ownership of it.
+ **/
+void
+periodic_events_register(periodic_event_item_t *item)
+{
+  if (!the_periodic_events)
+    the_periodic_events = smartlist_new();
+
+  if (BUG(smartlist_contains(the_periodic_events, item)))
+    return;
+
+  smartlist_add(the_periodic_events, item);
+}
+
+/**
+ * Make all registered periodic events connect to the libevent backend.
+ */
+void
+periodic_events_connect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (item->ev)
+      continue;
+    periodic_event_connect(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Reset all the registered periodic events so we'll do all our actions again
+ * as if we just started up.
+ *
+ * Useful if our clock just moved back a long time from the future,
+ * so we don't wait until that future arrives again before acting.
+ */
+void
+periodic_events_reset_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    periodic_event_reschedule(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Return the registered periodic event whose name is <b>name</b>.
+ * Return NULL if no such event is found.
+ */
+periodic_event_item_t *
+periodic_events_find(const char *name)
+{
+  if (! the_periodic_events)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (strcmp(name, item->name) == 0)
+      return item;
+  } SMARTLIST_FOREACH_END(item);
+  return NULL;
+}
+
+/**
+ * Start or stop registered periodic events, depending on our current set of
+ * roles.
+ *
+ * Invoked when our list of roles, or the net_disabled flag has changed.
+ **/
+void
+periodic_events_rescan_by_roles(int roles, bool net_disabled)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    int enable = !!(item->roles & roles);
+
+    /* Handle the event flags. */
+    if (net_disabled &&
+        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
+      enable = 0;
+    }
+
+    /* Enable the event if needed. It is safe to enable an event that was
+     * already enabled. Same goes for disabling it. */
+    if (enable) {
+      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
+      periodic_event_enable(item);
+    } else {
+      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
+      if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) {
+        periodic_event_schedule_and_disable(item);
+      } else {
+        periodic_event_disable(item);
+      }
+    }
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Invoked at shutdown: disconnect and unregister all periodic events.
+ *
+ * Does not free the periodic_event_item_t object themselves, because we do
+ * not own them.
+ */
+void
+periodic_events_disconnect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    periodic_event_disconnect(item);
+  } SMARTLIST_FOREACH_END(item);
+
+  smartlist_free(the_periodic_events);
+}
+
+#define LONGEST_TIMER_PERIOD (30 * 86400)
+/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
+ * clipped to the range [1 second, LONGEST_TIMER_PERIOD].
+ *
+ * We use this to answer the question, "how many seconds is it from now until
+ * next" in periodic timer callbacks.  Don't use it for other purposes
+ **/
+int
+safe_timer_diff(time_t now, time_t next)
+{
+  if (next > now) {
+    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
+     * and nothing that could run Tor. It's a bug if 'next' is around then.
+     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
+     * Bang. We cannot extrapolate past a singularity, but there was probably
+     * nothing that could run Tor then, either.
+     **/
+    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
+
+    if (next - LONGEST_TIMER_PERIOD > now)
+      return LONGEST_TIMER_PERIOD;
+    return (int)(next - now);
+  } else {
+    return 1;
+  }
+}
diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h
index e49fafd174..a9aa461969 100644
--- a/src/core/mainloop/periodic.h
+++ b/src/core/mainloop/periodic.h
@@ -15,6 +15,10 @@
 #define PERIODIC_EVENT_ROLE_BRIDGEAUTH  (1U << 4)
 #define PERIODIC_EVENT_ROLE_HS_SERVICE  (1U << 5)
 #define PERIODIC_EVENT_ROLE_DIRSERVER   (1U << 6)
+#define PERIODIC_EVENT_ROLE_CONTROLEV   (1U << 7)
+
+#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8)
+#define PERIODIC_EVENT_ROLE_ALL         (1U << 9)
 
 /* Helper macro to make it a bit less annoying to defined groups of roles that
  * are often used. */
@@ -25,10 +29,6 @@
 /* Authorities that is both bridge and directory. */
 #define PERIODIC_EVENT_ROLE_AUTHORITIES \
   (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH)
-/* All roles. */
-#define PERIODIC_EVENT_ROLE_ALL \
-  (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \
-   PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER)
 
 /*
  * Event flags which can change the behavior of an event.
@@ -39,6 +39,11 @@
  * the net_is_disabled() check. */
 #define PERIODIC_EVENT_FLAG_NEED_NET  (1U << 0)
 
+/* Indicate that if the event is enabled, it needs to be run once before
+ * it becomes disabled.
+ */
+#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE  (1U << 1)
+
 /** Callback function for a periodic event to take action.  The return value
 * influences the next time the function will get called.  Return
 * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled
@@ -78,11 +83,20 @@ periodic_event_is_enabled(const periodic_event_item_t *item)
 }
 
 void periodic_event_launch(periodic_event_item_t *event);
-void periodic_event_setup(periodic_event_item_t *event);
-void periodic_event_destroy(periodic_event_item_t *event);
+void periodic_event_connect(periodic_event_item_t *event);
+//void periodic_event_disconnect(periodic_event_item_t *event);
 void periodic_event_reschedule(periodic_event_item_t *event);
 void periodic_event_enable(periodic_event_item_t *event);
 void periodic_event_disable(periodic_event_item_t *event);
+void periodic_event_schedule_and_disable(periodic_event_item_t *event);
 
-#endif /* !defined(TOR_PERIODIC_H) */
+void periodic_events_register(periodic_event_item_t *item);
+void periodic_events_connect_all(void);
+void periodic_events_reset_all(void);
+periodic_event_item_t *periodic_events_find(const char *name);
+void periodic_events_rescan_by_roles(int roles, bool net_disabled);
+void periodic_events_disconnect_all(void);
+
+int safe_timer_diff(time_t now, time_t next);
 
+#endif /* !defined(TOR_PERIODIC_H) */
diff --git a/src/core/or/.may_include b/src/core/or/.may_include
new file mode 100644
index 0000000000..5173e8a2b6
--- /dev/null
+++ b/src/core/or/.may_include
@@ -0,0 +1,38 @@
+!advisory
+
+orconfig.h
+
+lib/arch/*.h
+lib/buf/*.h
+lib/cc/*.h
+lib/compress/*.h
+lib/container/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/defs/*.h
+lib/encoding/*.h
+lib/err/*.h
+lib/evloop/*.h
+lib/fs/*.h
+lib/geoip/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/math/*.h
+lib/net/*.h
+lib/pubsub/*.h
+lib/string/*.h
+lib/subsys/*.h
+lib/test/*.h
+lib/testsupport/*.h
+lib/thread/*.h
+lib/time/*.h
+lib/tls/*.h
+lib/wallclock/*.h
+
+trunnel/*.h
+
+core/mainloop/*.h
+core/proto/*.h
+core/crypto/*.h
+core/or/*.h
\ No newline at end of file
diff --git a/src/core/or/addr_policy_st.h b/src/core/or/addr_policy_st.h
index a75f1a731d..11442d29b4 100644
--- a/src/core/or/addr_policy_st.h
+++ b/src/core/or/addr_policy_st.h
@@ -43,4 +43,4 @@ struct addr_policy_t {
   uint16_t prt_max; /**< Highest port number to accept/reject. */
 };
 
-#endif
+#endif /* !defined(TOR_ADDR_POLICY_ST_H) */
diff --git a/src/core/or/address_set.h b/src/core/or/address_set.h
index 7a9e71628e..95608a9a53 100644
--- a/src/core/or/address_set.h
+++ b/src/core/or/address_set.h
@@ -28,4 +28,4 @@ void address_set_add_ipv4h(address_set_t *set, uint32_t addr);
 int address_set_probably_contains(const address_set_t *set,
                                   const struct tor_addr_t *addr);
 
-#endif
+#endif /* !defined(TOR_ADDRESS_SET_H) */
diff --git a/src/core/or/cell_queue_st.h b/src/core/or/cell_queue_st.h
index 130b95a011..7ba339b965 100644
--- a/src/core/or/cell_queue_st.h
+++ b/src/core/or/cell_queue_st.h
@@ -26,4 +26,4 @@ struct cell_queue_t {
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
+#endif /* !defined(PACKED_CELL_ST_H) */
diff --git a/src/core/or/cell_st.h b/src/core/or/cell_st.h
index 7ab7eceb50..c4eec4f4b5 100644
--- a/src/core/or/cell_st.h
+++ b/src/core/or/cell_st.h
@@ -16,5 +16,5 @@ struct cell_t {
   uint8_t payload[CELL_PAYLOAD_SIZE]; /**< Cell body. */
 };
 
-#endif
+#endif /* !defined(CELL_ST_H) */
 
diff --git a/src/core/or/channel.c b/src/core/or/channel.c
index 3886906875..596932b008 100644
--- a/src/core/or/channel.c
+++ b/src/core/or/channel.c
@@ -1418,6 +1418,7 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
   int ret = -1;
   size_t cell_bytes;
+  uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids);
 
   tor_assert(chan);
   tor_assert(cell);
@@ -1452,6 +1453,16 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
   /* Successfully sent the cell. */
   ret = 0;
 
+  /* Update padding statistics for the packed codepath.. */
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
+  if (command == CELL_PADDING)
+    rep_hist_padding_count_write(PADDING_TYPE_CELL);
+  if (chan->padding_enabled) {
+    rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
+    if (command == CELL_PADDING)
+      rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
+  }
+
  done:
   return ret;
 }
diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c
index f874e39946..d6c9e21631 100644
--- a/src/core/or/channeltls.c
+++ b/src/core/or/channeltls.c
@@ -59,6 +59,7 @@
 #include "feature/nodelist/torcert.h"
 #include "feature/nodelist/networkstatus.h"
 #include "trunnel/channelpadding_negotiation.h"
+#include "trunnel/netinfo.h"
 #include "core/or/channelpadding.h"
 
 #include "core/or/cell_st.h"
@@ -949,7 +950,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
 void
 channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                           or_connection_t *conn,
-                                          uint8_t old_state,
                                           uint8_t state)
 {
   channel_t *base_chan;
@@ -958,8 +958,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
   tor_assert(conn);
   tor_assert(conn->chan == chan);
   tor_assert(chan->conn == conn);
-  /* Shut the compiler up without triggering -Wtautological-compare */
-  (void)old_state;
 
   base_chan = TLS_CHAN_TO_BASE(chan);
 
@@ -1029,6 +1027,16 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
 }
 #endif /* defined(KEEP_TIMING_STATS) */
 
+#ifdef KEEP_TIMING_STATS
+#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
+    ++num ## tp;                                                \
+    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
+                             channel_tls_process_ ## tp ## _cell);  \
+    } STMT_END
+#else /* !defined(KEEP_TIMING_STATS) */
+#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
+#endif /* defined(KEEP_TIMING_STATS) */
+
 /**
  * Handle an incoming cell on a channel_tls_t.
  *
@@ -1048,16 +1056,6 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   channel_tls_t *chan;
   int handshaking;
 
-#ifdef KEEP_TIMING_STATS
-#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
-    ++num ## tp;                                                \
-    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
-                             channel_tls_process_ ## tp ## _cell);  \
-    } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
-#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
-#endif /* defined(KEEP_TIMING_STATS) */
-
   tor_assert(cell);
   tor_assert(conn);
 
@@ -1075,7 +1073,8 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
     return;
 
   /* Reject all but VERSIONS and NETINFO when handshaking. */
-  /* (VERSIONS should actually be impossible; it's variable-length.) */
+  /* (VERSIONS actually indicates a protocol warning: it's variable-length,
+   * so if it reaches this function, we're on a v1 connection.) */
   if (handshaking && cell->command != CELL_VERSIONS &&
       cell->command != CELL_NETINFO) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
@@ -1096,13 +1095,13 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   entry_guards_note_internet_connectivity(get_guard_selection_info());
   rep_hist_padding_count_read(PADDING_TYPE_TOTAL);
 
-  if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+  if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
     rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL);
 
   switch (cell->command) {
     case CELL_PADDING:
       rep_hist_padding_count_read(PADDING_TYPE_CELL);
-      if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+      if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
         rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL);
       ++stats_n_padding_cells_processed;
       /* do nothing */
@@ -1330,6 +1329,8 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
   }
 }
 
+#undef PROCESS_CELL
+
 /**
  * Update channel marks after connection_or.c has changed an address.
  *
@@ -1645,6 +1646,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan)
 }
 
 /**
+ * Convert <b>netinfo_addr</b> into corresponding <b>tor_addr</b>.
+ * Return 0 on success; on failure, return -1 and log a warning.
+ */
+static int
+tor_addr_from_netinfo_addr(tor_addr_t *tor_addr,
+                           const netinfo_addr_t *netinfo_addr) {
+  tor_assert(tor_addr);
+  tor_assert(netinfo_addr);
+
+  uint8_t type = netinfo_addr_get_addr_type(netinfo_addr);
+  uint8_t len = netinfo_addr_get_len(netinfo_addr);
+
+  if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4)  {
+    uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr);
+    tor_addr_from_ipv4h(tor_addr, ipv4);
+  } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) {
+    const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6(
+                                  netinfo_addr);
+    tor_addr_from_ipv6_bytes(tor_addr, (const char *)ipv6_bytes);
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO "
+                                     "- wrong type/length.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/**
  * Helper: compute the absolute value of a time_t.
  *
  * (we need this because labs() doesn't always work for time_t, since
@@ -1668,8 +1698,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   time_t timestamp;
   uint8_t my_addr_type;
   uint8_t my_addr_len;
-  const uint8_t *my_addr_ptr;
-  const uint8_t *cp, *end;
   uint8_t n_other_addrs;
   time_t now = time(NULL);
   const routerinfo_t *me = router_get_my_routerinfo();
@@ -1717,7 +1745,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
         tor_assert(tor_digest_is_zero(
                   (const char*)(chan->conn->handshake_state->
                       authenticated_rsa_peer_id)));
-        tor_assert(tor_mem_is_zero(
+        tor_assert(fast_mem_is_zero(
                   (const char*)(chan->conn->handshake_state->
                                 authenticated_ed25519_peer_id.pubkey), 32));
         /* If the client never authenticated, it's a tor client or bridge
@@ -1740,38 +1768,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   }
 
   /* Decode the cell. */
-  timestamp = ntohl(get_uint32(cell->payload));
-  const time_t sent_versions_at =
-    chan->conn->handshake_state->sent_versions_at;
-  if (now > sent_versions_at && (now - sent_versions_at) < 180) {
-    /* If we have gotten the NETINFO cell reasonably soon after having
-     * sent our VERSIONS cell, maybe we can learn skew information from it. */
-    apparent_skew = now - timestamp;
+  netinfo_cell_t *netinfo_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload,
+                                      CELL_PAYLOAD_SIZE);
+
+  if (parsed < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+           "Failed to parse NETINFO cell - closing connection.");
+    connection_or_close_for_error(chan->conn, 0);
+    return;
   }
 
-  my_addr_type = (uint8_t) cell->payload[4];
-  my_addr_len = (uint8_t) cell->payload[5];
-  my_addr_ptr = (uint8_t*) cell->payload + 6;
-  end = cell->payload + CELL_PAYLOAD_SIZE;
-  cp = cell->payload + 6 + my_addr_len;
+  timestamp = netinfo_cell_get_timestamp(netinfo_cell);
+
+  const netinfo_addr_t *my_addr =
+    netinfo_cell_getconst_other_addr(netinfo_cell);
 
+  my_addr_type = netinfo_addr_get_addr_type(my_addr);
+  my_addr_len = netinfo_addr_get_len(my_addr);
+
+  if ((now - chan->conn->handshake_state->sent_versions_at) < 180) {
+    apparent_skew = now - timestamp;
+  }
   /* We used to check:
    *    if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) {
    *
    * This is actually never going to happen, since my_addr_len is at most 255,
    * and CELL_PAYLOAD_LEN - 6 is 503.  So we know that cp is < end. */
 
-  if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) {
-    tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr));
+  if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) {
+    connection_or_close_for_error(chan->conn, 0);
+    netinfo_cell_free(netinfo_cell);
+    return;
+  }
 
+  if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) {
     if (!get_options()->BridgeRelay && me &&
-        get_uint32(my_addr_ptr) == htonl(me->addr)) {
+        tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) {
       TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
     }
-
-  } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) {
-    tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr);
-
+  } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 &&
+             my_addr_len == 16) {
     if (!get_options()->BridgeRelay && me &&
         !tor_addr_is_null(&me->ipv6_addr) &&
         tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) {
@@ -1779,18 +1817,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
     }
   }
 
-  n_other_addrs = (uint8_t) *cp++;
-  while (n_other_addrs && cp < end-2) {
+  n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell);
+  for (uint8_t i = 0; i < n_other_addrs; i++) {
     /* Consider all the other addresses; if any matches, this connection is
      * "canonical." */
+
+    const netinfo_addr_t *netinfo_addr =
+      netinfo_cell_getconst_my_addrs(netinfo_cell, i);
+
     tor_addr_t addr;
-    const uint8_t *next =
-      decode_address_from_payload(&addr, cp, (int)(end-cp));
-    if (next == NULL) {
+
+    if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) {
       log_fn(LOG_PROTOCOL_WARN,  LD_OR,
-             "Bad address in netinfo cell; closing connection.");
-      connection_or_close_for_error(chan->conn, 0);
-      return;
+             "Bad address in netinfo cell; Skipping.");
+      continue;
     }
     /* A relay can connect from anywhere and be canonical, so
      * long as it tells you from where it came. This may sound a bit
@@ -1803,10 +1843,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       connection_or_set_canonical(chan->conn, 1);
       break;
     }
-    cp = next;
-    --n_other_addrs;
   }
 
+  netinfo_cell_free(netinfo_cell);
+
   if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer &&
       channel_is_canonical(TLS_CHAN_TO_BASE(chan))) {
     const char *descr =
diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h
index a2648ff537..634a2a00e9 100644
--- a/src/core/or/channeltls.h
+++ b/src/core/or/channeltls.h
@@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan);
 void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn);
 void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                                or_connection_t *conn,
-                                               uint8_t old_state,
                                                uint8_t state);
 void channel_tls_handle_var_cell(var_cell_t *var_cell,
                                  or_connection_t *conn);
diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h
index d4339ff50d..eae3c908d5 100644
--- a/src/core/or/circuit_st.h
+++ b/src/core/or/circuit_st.h
@@ -12,6 +12,11 @@
 #include "core/or/cell_queue_st.h"
 
 struct hs_token_t;
+struct circpad_machine_spec_t;
+struct circpad_machine_runtime_t;
+
+/** Number of padding state machines on a circuit. */
+#define CIRCPAD_MAX_MACHINES (2)
 
 /** "magic" value for an origin_circuit_t */
 #define ORIGIN_CIRCUIT_MAGIC 0x35315243u
@@ -61,12 +66,6 @@ struct circuit_t {
    */
   circid_t n_circ_id;
 
-  /**
-   * Circuit mux associated with n_chan to which this circuit is attached;
-   * NULL if we have no n_chan.
-   */
-  circuitmux_t *n_mux;
-
   /** Queue of cells waiting to be transmitted on n_chan */
   cell_queue_t n_chan_cells;
 
@@ -93,6 +92,10 @@ struct circuit_t {
   /** True iff this circuit has received a DESTROY cell in either direction */
   unsigned int received_destroy : 1;
 
+  /** True iff we have sent a sufficiently random data cell since last
+   * we reset send_randomness_after_n_cells. */
+  unsigned int have_sent_sufficiently_random_cell : 1;
+
   uint8_t state; /**< Current status of this circuit. */
   uint8_t purpose; /**< Why are we creating this circuit? */
 
@@ -105,6 +108,32 @@ struct circuit_t {
    * circuit-level sendme cells to indicate that we're willing to accept
    * more. */
   int deliver_window;
+  /**
+   * How many cells do we have until we need to send one that contains
+   * sufficient randomness?  Used to ensure that authenticated SENDME cells
+   * will reflect some unpredictable information.
+   **/
+  uint16_t send_randomness_after_n_cells;
+
+  /** FIFO containing the digest of the cells that are just before a SENDME is
+   * sent by the client. It is done at the last cell before our package_window
+   * goes down to 0 which is when we expect a SENDME.
+   *
+   * Our current circuit package window is capped to 1000
+   * (CIRCWINDOW_START_MAX) which is also the start value. The increment is
+   * set to 100 (CIRCWINDOW_INCREMENT) which means we don't allow more than
+   * 1000/100 = 10 outstanding SENDME cells worth of data. Meaning that this
+   * list can not contain more than 10 digests of DIGEST_LEN bytes (20).
+   *
+   * At position i in the list, the digest corresponds to the
+   * (CIRCWINDOW_INCREMENT * i)-nth cell received since we expect a SENDME to
+   * be received containing that cell digest.
+   *
+   * For example, position 2 (starting at 0) means that we've received 300
+   * cells so the 300th cell digest is kept at index 2.
+   *
+   * At maximum, this list contains 200 bytes plus the smartlist overhead. */
+  smartlist_t *sendme_last_digests;
 
   /** Temporary field used during circuits_handle_oom. */
   uint32_t age_tmp;
@@ -177,6 +206,27 @@ struct circuit_t {
   /** Hashtable node: used to look up the circuit by its HS token using the HS
       circuitmap. */
   HT_ENTRY(circuit_t) hs_circuitmap_node;
+
+  /** Adaptive Padding state machines: these are immutable. The state machines
+   *  that come from the consensus are saved to a global structure, to avoid
+   *  per-circuit allocations. This merely points to the global copy in
+   *  origin_padding_machines or relay_padding_machines that should never
+   *  change or get deallocated.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES];
+
+  /** Adaptive Padding machine runtime info for above machines. This is
+   *  the per-circuit mutable information, such as the current state and
+   *  histogram token counts. Some of it is optional (aka NULL).
+   *  If a machine is being shut down, these indexes can be NULL
+   *  without the corresponding padding_machine being NULL, while we
+   *  wait for the other end to respond to our shutdown request.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  struct circpad_machine_runtime_t *padding_info[CIRCPAD_MAX_MACHINES];
 };
 
-#endif
+#endif /* !defined(CIRCUIT_ST_H) */
diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c
index f3a5791d6c..16bb7e7d79 100644
--- a/src/core/or/circuitbuild.c
+++ b/src/core/or/circuitbuild.c
@@ -26,10 +26,11 @@
  **/
 
 #define CIRCUITBUILD_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/crypto/hs_ntor.h"
 #include "core/crypto/onion_crypto.h"
 #include "core/crypto/onion_fast.h"
@@ -42,17 +43,20 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/command.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/onion.h"
+#include "core/or/ocirc_event.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
@@ -87,8 +91,6 @@ static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
 static int circuit_deliver_create_cell(circuit_t *circ,
                                        const create_cell_t *create_cell,
                                        int relayed);
-static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
-STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 static int circuit_send_first_onion_skin(origin_circuit_t *circ);
 static int circuit_build_no_more_hops(origin_circuit_t *circ);
 static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
@@ -492,7 +494,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
     return NULL;
   }
 
-  control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
+  circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
 
   if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
     circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
@@ -508,6 +510,27 @@ origin_circuit_get_guard_state(origin_circuit_t *circ)
   return circ->guard_state;
 }
 
+/**
+ * Helper function to publish a channel association message
+ *
+ * circuit_handle_first_hop() calls this to notify subscribers about a
+ * channel launch event, which associates a circuit with a channel.
+ * This doesn't always correspond to an assignment of the circuit's
+ * n_chan field, because that seems to be only for fully-open
+ * channels.
+ **/
+static void
+circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = circ->global_identifier;
+  msg->chan = chan->global_identifier;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_chan_publish(msg);
+}
+
 /** Start establishing the first hop of our circuit. Figure out what
  * OR we should connect to, and if necessary start the connection to
  * it. If we're already connected, then send the 'create' cell.
@@ -522,7 +545,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   int should_launch = 0;
   const or_options_t *options = get_options();
 
-  firsthop = onion_next_hop_in_cpath(circ->cpath);
+  firsthop = cpath_get_next_non_open_hop(circ->cpath);
   tor_assert(firsthop);
   tor_assert(firsthop->extend_info);
 
@@ -559,8 +582,6 @@ circuit_handle_first_hop(origin_circuit_t *circ)
     circ->base_.n_hop = extend_info_dup(firsthop->extend_info);
 
     if (should_launch) {
-      if (circ->build_state->onehop_tunnel)
-        control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
       n_chan = channel_connect_for_circuit(
           &firsthop->extend_info->addr,
           firsthop->extend_info->port,
@@ -570,6 +591,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
       }
+      circuit_chan_publish(circ, n_chan);
     }
 
     log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
@@ -581,6 +603,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   } else { /* it's already open. use it. */
     tor_assert(!circ->base_.n_hop);
     circ->base_.n_chan = n_chan;
+    circuit_chan_publish(circ, n_chan);
     log_debug(LD_CIRC,"Conn open. Delivering first onion skin.");
     if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
       log_info(LD_CIRC,"circuit_send_next_onion_skin failed.");
@@ -923,15 +946,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
   tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
   tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
 
-  crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath);
+  crypt_path_t *hop = cpath_get_next_non_open_hop(circ->cpath);
   circuit_build_times_handle_completed_hop(circ);
 
+  circpad_machine_event_circ_added_hop(circ);
+
   if (hop) {
     /* Case two: we're on a hop after the first. */
     return circuit_send_intermediate_onion_skin(circ, hop);
   }
 
   /* Case three: the circuit is finished. Do housekeeping tasks on it. */
+  circpad_machine_event_circ_built(circ);
   return circuit_build_no_more_hops(circ);
 }
 
@@ -1332,34 +1358,6 @@ circuit_extend(cell_t *cell, circuit_t *circ)
   return 0;
 }
 
-/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
- *
- * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
- * service circuits and <b>key_data</b> must be at least
- * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
- *
- * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
- * bytes, which are used as follows:
- *   - 20 to initialize f_digest
- *   - 20 to initialize b_digest
- *   - 16 to key f_crypto
- *   - 16 to key b_crypto
- *
- * (If 'reverse' is true, then f_XX and b_XX are swapped.)
- *
- * Return 0 if init was successful, else -1 if it failed.
- */
-int
-circuit_init_cpath_crypto(crypt_path_t *cpath,
-                          const char *key_data, size_t key_data_len,
-                          int reverse, int is_hs_v3)
-{
-
-  tor_assert(cpath);
-  return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse,
-                           is_hs_v3);
-}
-
 /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>.
  * (The body of <b>reply</b> varies depending on what sort of handshake
  * this is.)
@@ -1385,7 +1383,7 @@ circuit_finish_handshake(origin_circuit_t *circ,
   if (circ->cpath->state == CPATH_STATE_AWAITING_KEYS) {
     hop = circ->cpath;
   } else {
-    hop = onion_next_hop_in_cpath(circ->cpath);
+    hop = cpath_get_next_non_open_hop(circ->cpath);
     if (!hop) { /* got an extended when we're all done? */
       log_warn(LD_PROTOCOL,"got extended when circ already built? Closing.");
       return - END_CIRC_REASON_TORPROTOCOL;
@@ -1409,14 +1407,14 @@ circuit_finish_handshake(origin_circuit_t *circ,
 
   onion_handshake_state_release(&hop->handshake_state);
 
-  if (circuit_init_cpath_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
+  if (cpath_init_circuit_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
     return -END_CIRC_REASON_TORPROTOCOL;
   }
 
   hop->state = CPATH_STATE_OPEN;
   log_info(LD_CIRC,"Finished building circuit hop:");
   circuit_log_path(LOG_INFO,LD_CIRC,circ);
-  control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0);
+  circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0);
 
   return 0;
 }
@@ -1461,7 +1459,7 @@ circuit_truncated(origin_circuit_t *circ, int reason)
     }
 
     layer->next = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
   log_info(LD_CIRC, "finished");
@@ -1655,24 +1653,28 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei)
  * to handle the desired path length, return -1.
  */
 STATIC int
-new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
+new_route_len(uint8_t purpose, extend_info_t *exit_ei,
+              const smartlist_t *nodes)
 {
-  int num_acceptable_routers;
   int routelen;
 
   tor_assert(nodes);
 
   routelen = route_len_for_purpose(purpose, exit_ei);
 
-  num_acceptable_routers = count_acceptable_nodes(nodes);
+  int num_acceptable_direct = count_acceptable_nodes(nodes, 1);
+  int num_acceptable_indirect = count_acceptable_nodes(nodes, 0);
 
-  log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).",
-            routelen, num_acceptable_routers, smartlist_len(nodes));
+  log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect "
+             "routers suitable).", routelen, num_acceptable_direct,
+             num_acceptable_indirect);
 
-  if (num_acceptable_routers < routelen) {
+  if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) {
     log_info(LD_CIRC,
-             "Not enough acceptable routers (%d/%d). Discarding this circuit.",
-             num_acceptable_routers, routelen);
+             "Not enough acceptable routers (%d/%d direct and %d/%d "
+             "indirect routers suitable). Discarding this circuit.",
+             num_acceptable_direct, routelen,
+             num_acceptable_indirect, routelen);
     return -1;
   }
 
@@ -2276,7 +2278,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
   state->chosen_exit = extend_info_dup(exit_ei);
 
   ++circ->build_state->desired_path_len;
-  onion_append_hop(&circ->cpath, exit_ei);
+  cpath_append_hop(&circ->cpath, exit_ei);
   return 0;
 }
 
@@ -2314,7 +2316,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
  * particular router. See bug #25885.)
  */
 MOCK_IMPL(STATIC int,
-count_acceptable_nodes, (smartlist_t *nodes))
+count_acceptable_nodes, (const smartlist_t *nodes, int direct))
 {
   int num=0;
 
@@ -2328,7 +2330,7 @@ count_acceptable_nodes, (smartlist_t *nodes))
     if (! node->is_valid)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i);
       continue;
-    if (! node_has_any_descriptor(node))
+    if (! node_has_preferred_descriptor(node, direct))
       continue;
     /* The node has a descriptor, so we can just check the ntor key directly */
     if (!node_has_curve25519_onion_key(node))
@@ -2341,47 +2343,6 @@ count_acceptable_nodes, (smartlist_t *nodes))
   return num;
 }
 
-/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
- * This function is used to extend cpath by another hop.
- */
-void
-onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop)
-{
-  if (*head_ptr) {
-    new_hop->next = (*head_ptr);
-    new_hop->prev = (*head_ptr)->prev;
-    (*head_ptr)->prev->next = new_hop;
-    (*head_ptr)->prev = new_hop;
-  } else {
-    *head_ptr = new_hop;
-    new_hop->prev = new_hop->next = new_hop;
-  }
-}
-
-#ifdef TOR_UNIT_TESTS
-
-/** Unittest helper function: Count number of hops in cpath linked list. */
-unsigned int
-cpath_get_n_hops(crypt_path_t **head_ptr)
-{
-  unsigned int n_hops = 0;
-  crypt_path_t *tmp;
-
-  if (!*head_ptr) {
-    return 0;
-  }
-
-  tmp = *head_ptr;
-  do {
-    n_hops++;
-    tmp = tmp->next;
-  } while (tmp != *head_ptr);
-
-  return n_hops;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 /**
  * Build the exclude list for vanguard circuits.
  *
@@ -2579,7 +2540,24 @@ choose_good_middle_server(uint8_t purpose,
     return choice;
   }
 
-  choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  if (options->MiddleNodes) {
+    smartlist_t *sl = smartlist_new();
+    routerset_get_all_nodes(sl, options->MiddleNodes,
+                            options->ExcludeNodes, 1);
+
+    smartlist_subtract(sl, excluded);
+
+    choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID);
+    smartlist_free(sl);
+    if (choice) {
+      log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s",
+          hex_str(choice->identity, DIGEST_LEN));
+    } else {
+      log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available");
+    }
+  } else {
+    choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  }
   smartlist_free(excluded);
   return choice;
 }
@@ -2639,20 +2617,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
   return choice;
 }
 
-/** Return the first non-open hop in cpath, or return NULL if all
- * hops are open. */
-static crypt_path_t *
-onion_next_hop_in_cpath(crypt_path_t *cpath)
-{
-  crypt_path_t *hop = cpath;
-  do {
-    if (hop->state != CPATH_STATE_OPEN)
-      return hop;
-    hop = hop->next;
-  } while (hop != cpath);
-  return NULL;
-}
-
 /** Choose a suitable next hop for the circuit <b>circ</b>.
  * Append the hop info to circ->cpath.
  *
@@ -2709,33 +2673,11 @@ onion_extend_cpath(origin_circuit_t *circ)
             extend_info_describe(info),
             cur_len+1, build_state_get_exit_nickname(state));
 
-  onion_append_hop(&circ->cpath, info);
+  cpath_append_hop(&circ->cpath, info);
   extend_info_free(info);
   return 0;
 }
 
-/** Create a new hop, annotate it with information about its
- * corresponding router <b>choice</b>, and append it to the
- * end of the cpath <b>head_ptr</b>. */
-STATIC int
-onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
-{
-  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
-
-  /* link hop into the cpath, at the end. */
-  onion_append_to_cpath(head_ptr, hop);
-
-  hop->magic = CRYPT_PATH_MAGIC;
-  hop->state = CPATH_STATE_CLOSED;
-
-  hop->extend_info = extend_info_dup(choice);
-
-  hop->package_window = circuit_initial_package_window();
-  hop->deliver_window = CIRCWINDOW_START;
-
-  return 0;
-}
-
 /** Allocate a new extend_info object based on the various arguments. */
 extend_info_t *
 extend_info_new(const char *nickname,
@@ -2940,7 +2882,7 @@ extend_info_supports_ntor(const extend_info_t* ei)
 {
   tor_assert(ei);
   /* Valid ntor keys have at least one non-zero byte */
-  return !tor_mem_is_zero(
+  return !fast_mem_is_zero(
                           (const char*)ei->curve25519_onion_key.public_key,
                           CURVE25519_PUBKEY_LEN);
 }
diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h
index 969f5b5cc3..ad7d032cd4 100644
--- a/src/core/or/circuitbuild.h
+++ b/src/core/or/circuitbuild.h
@@ -34,9 +34,6 @@ int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);
 void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle);
 int circuit_extend(cell_t *cell, circuit_t *circ);
-int circuit_init_cpath_crypto(crypt_path_t *cpath,
-                              const char *key_data, size_t key_data_len,
-                              int reverse, int is_hs_v3);
 struct created_cell_t;
 int circuit_finish_handshake(origin_circuit_t *circ,
                              const struct created_cell_t *created_cell);
@@ -51,7 +48,6 @@ MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now,
 
 int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info);
 int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info);
-void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop);
 extend_info_t *extend_info_new(const char *nickname,
                                const char *rsa_id_digest,
                                const struct ed25519_public_key_t *ed_id,
@@ -83,8 +79,9 @@ void circuit_upgrade_circuits_from_guard_wait(void);
 #ifdef CIRCUITBUILD_PRIVATE
 STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
 STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
-                         smartlist_t *nodes);
-MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
+                         const smartlist_t *nodes);
+MOCK_DECL(STATIC int, count_acceptable_nodes, (const smartlist_t *nodes,
+                                               int direct));
 
 STATIC int onion_extend_cpath(origin_circuit_t *circ);
 
@@ -92,11 +89,6 @@ STATIC int
 onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
                       int is_hs_v3_rp_circuit);
 
-#if defined(TOR_UNIT_TESTS)
-unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 #endif /* defined(CIRCUITBUILD_PRIVATE) */
 
 #endif /* !defined(TOR_CIRCUITBUILD_H) */
diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c
index 03d495cdfd..4102cf3d4c 100644
--- a/src/core/or/circuitlist.c
+++ b/src/core/or/circuitlist.c
@@ -51,6 +51,7 @@
  * logic, which was originally circuit-focused.
  **/
 #define CIRCUITLIST_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 #include "lib/cc/torint.h"  /* TOR_PRIuSZ */
 
 #include "core/or/or.h"
@@ -61,11 +62,13 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/circuitstats.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/crypt_path.h"
 #include "core/mainloop/connection.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/crypt_ops/crypto_dh.h"
@@ -94,7 +97,10 @@
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
+
+#define OCIRC_EVENT_PRIVATE
+#include "core/or/ocirc_event.h"
 
 #include "ht.h"
 
@@ -127,7 +133,6 @@ static smartlist_t *circuits_pending_other_guards = NULL;
  * circuit_mark_for_close and which are waiting for circuit_about_to_free. */
 static smartlist_t *circuits_pending_close = NULL;
 
-static void circuit_free_cpath_node(crypt_path_t *victim);
 static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
 static void circuit_about_to_free_atexit(circuit_t *circ);
 static void circuit_about_to_free(circuit_t *circ);
@@ -481,6 +486,54 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
   }
 }
 
+/**
+ * Helper function to publish a message about events on an origin circuit
+ *
+ * Publishes a message to subscribers of origin circuit events, and
+ * sends the control event.
+ **/
+int
+circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                     int reason_code)
+{
+  ocirc_cevent_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  tor_assert(circ);
+
+  msg->gid = circ->global_identifier;
+  msg->evtype = tp;
+  msg->reason = reason_code;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_cevent_publish(msg);
+  return control_event_circuit_status(circ, tp, reason_code);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * circuit_set_state() calls this to notify subscribers about a change
+ * of the state of an origin circuit.  @a circ must be an origin
+ * circuit.
+ **/
+static void
+circuit_state_publish(const circuit_t *circ)
+{
+  ocirc_state_msg_t *msg = tor_malloc(sizeof(*msg));
+  const origin_circuit_t *ocirc;
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  ocirc = CONST_TO_ORIGIN_CIRCUIT(circ);
+  /* Only inbound OR circuits can be in this state, not origin circuits. */
+  tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING);
+
+  msg->gid = ocirc->global_identifier;
+  msg->state = circ->state;
+  msg->onehop = ocirc->build_state->onehop_tunnel;
+
+  ocirc_state_publish(msg);
+}
+
 /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing
  * it from lists as appropriate. */
 void
@@ -510,6 +563,8 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN)
     tor_assert(!circ->n_chan_create_cell);
   circ->state = state;
+  if (CIRCUIT_IS_ORIGIN(circ))
+    circuit_state_publish(circ);
 }
 
 /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for
@@ -767,6 +822,8 @@ circuit_purpose_to_controller_string(uint8_t purpose)
       return "PATH_BIAS_TESTING";
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "HS_VANGUARDS";
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "CIRCUIT_PADDING";
 
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
@@ -796,6 +853,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_CONTROLLER:
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
       return NULL;
 
     case CIRCUIT_PURPOSE_INTRO_POINT:
@@ -896,6 +954,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "Hidden service: Pre-built vanguard circuit";
 
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "Circuit kept open for padding";
+
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
       return buf;
@@ -931,6 +992,7 @@ init_circuit_base(circuit_t *circ)
 
   circ->package_window = circuit_initial_package_window();
   circ->deliver_window = CIRCWINDOW_START;
+  circuit_reset_sendme_randomness(circ);
   cell_queue_init(&circ->n_chan_cells);
 
   smartlist_add(circuit_get_global_list(), circ);
@@ -1092,7 +1154,7 @@ circuit_free_(circuit_t *circ)
 
     if (ocirc->build_state) {
         extend_info_free(ocirc->build_state->chosen_exit);
-        circuit_free_cpath_node(ocirc->build_state->pending_final_cpath);
+        cpath_free(ocirc->build_state->pending_final_cpath);
         cpath_ref_decref(ocirc->build_state->service_pending_final_cpath_ref);
     }
     tor_free(ocirc->build_state);
@@ -1171,11 +1233,20 @@ circuit_free_(circuit_t *circ)
    * "active" checks will be violated. */
   cell_queue_clear(&circ->n_chan_cells);
 
+  /* Cleanup possible SENDME state. */
+  if (circ->sendme_last_digests) {
+    SMARTLIST_FOREACH(circ->sendme_last_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(circ->sendme_last_digests);
+  }
+
   log_info(LD_CIRC, "Circuit %u (id: %" PRIu32 ") has been freed.",
            n_circ_id,
            CIRCUIT_IS_ORIGIN(circ) ?
               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
 
+  /* Free any circuit padding structures */
+  circpad_circuit_free_all_machineinfos(circ);
+
   if (should_free) {
     memwipe(mem, 0xAA, memlen); /* poison memory */
     tor_free(mem);
@@ -1207,10 +1278,10 @@ circuit_clear_cpath(origin_circuit_t *circ)
   while (cpath->next && cpath->next != head) {
     victim = cpath;
     cpath = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
-  circuit_free_cpath_node(cpath);
+  cpath_free(cpath);
 
   circ->cpath = NULL;
 }
@@ -1267,29 +1338,13 @@ circuit_free_all(void)
   HT_CLEAR(chan_circid_map, &chan_circid_map);
 }
 
-/** Deallocate space associated with the cpath node <b>victim</b>. */
-static void
-circuit_free_cpath_node(crypt_path_t *victim)
-{
-  if (!victim)
-    return;
-
-  relay_crypto_clear(&victim->crypto);
-  onion_handshake_state_release(&victim->handshake_state);
-  crypto_dh_free(victim->rend_dh_handshake_state);
-  extend_info_free(victim->extend_info);
-
-  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
-  tor_free(victim);
-}
-
 /** Release a crypt_path_reference_t*, which may be NULL. */
 static void
 cpath_ref_decref(crypt_path_reference_t *cpath_ref)
 {
   if (cpath_ref != NULL) {
     if (--(cpath_ref->refcount) == 0) {
-      circuit_free_cpath_node(cpath_ref->cpath);
+      cpath_free(cpath_ref->cpath);
       tor_free(cpath_ref);
     }
   }
@@ -2140,6 +2195,11 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
   tor_assert(line);
   tor_assert(file);
 
+  /* Check whether the circuitpadding subsystem wants to block this close */
+  if (circpad_marked_circuit_for_padding(circ, reason)) {
+    return;
+  }
+
   if (circ->marked_for_close) {
     log_warn(LD_BUG,
         "Duplicate call to circuit_mark_for_close at %s:%d"
@@ -2270,7 +2330,7 @@ circuit_about_to_free(circuit_t *circ)
     smartlist_remove(circuits_pending_other_guards, circ);
   }
   if (CIRCUIT_IS_ORIGIN(circ)) {
-    control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
+    circuit_event_status(TO_ORIGIN_CIRCUIT(circ),
      (circ->state == CIRCUIT_STATE_OPEN ||
       circ->state == CIRCUIT_STATE_GUARD_WAIT) ?
                                  CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
@@ -2374,13 +2434,9 @@ marked_circuit_free_cells(circuit_t *circ)
     return;
   }
   cell_queue_clear(&circ->n_chan_cells);
-  if (circ->n_mux)
-    circuitmux_clear_num_cells(circ->n_mux, circ);
   if (! CIRCUIT_IS_ORIGIN(circ)) {
     or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
     cell_queue_clear(&orcirc->p_chan_cells);
-    if (orcirc->p_mux)
-      circuitmux_clear_num_cells(orcirc->p_mux, circ);
   }
 }
 
@@ -2724,59 +2780,6 @@ circuits_handle_oom(size_t current_allocation)
              n_dirconns_killed);
 }
 
-/** Verify that cpath layer <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-void
-assert_cpath_layer_ok(const crypt_path_t *cp)
-{
-//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
-//  tor_assert(cp->port);
-  tor_assert(cp);
-  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
-  switch (cp->state)
-    {
-    case CPATH_STATE_OPEN:
-      relay_crypto_assert_ok(&cp->crypto);
-      FALLTHROUGH;
-    case CPATH_STATE_CLOSED:
-      /*XXXX Assert that there's no handshake_state either. */
-      tor_assert(!cp->rend_dh_handshake_state);
-      break;
-    case CPATH_STATE_AWAITING_KEYS:
-      /* tor_assert(cp->dh_handshake_state); */
-      break;
-    default:
-      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
-      tor_assert(0);
-    }
-  tor_assert(cp->package_window >= 0);
-  tor_assert(cp->deliver_window >= 0);
-}
-
-/** Verify that cpath <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-static void
-assert_cpath_ok(const crypt_path_t *cp)
-{
-  const crypt_path_t *start = cp;
-
-  do {
-    assert_cpath_layer_ok(cp);
-    /* layers must be in sequence of: "open* awaiting? closed*" */
-    if (cp != start) {
-      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      } else if (cp->state == CPATH_STATE_OPEN) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      }
-    }
-    cp = cp->next;
-    tor_assert(cp);
-  } while (cp != start);
-}
-
 /** Verify that circuit <b>c</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */
@@ -2838,7 +2841,7 @@ assert_circuit_ok,(const circuit_t *c))
                !smartlist_contains(circuits_pending_chans, c));
   }
   if (origin_circ && origin_circ->cpath) {
-    assert_cpath_ok(origin_circ->cpath);
+    cpath_assert_ok(origin_circ->cpath);
   }
   if (c->purpose == CIRCUIT_PURPOSE_REND_ESTABLISHED) {
     tor_assert(or_circ);
diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h
index b87c6a3667..80c1f7ac4e 100644
--- a/src/core/or/circuitlist.h
+++ b/src/core/or/circuitlist.h
@@ -14,6 +14,7 @@
 
 #include "lib/testsupport/testsupport.h"
 #include "feature/hs/hs_ident.h"
+#include "core/or/ocirc_event.h"
 
 /** Circuit state: I'm the origin, still haven't done all my handshakes. */
 #define CIRCUIT_STATE_BUILDING 0
@@ -91,31 +92,33 @@
 #define CIRCUIT_PURPOSE_C_HS_MAX_ 13
 /** This circuit is used for build time measurement only */
 #define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 14
-#define CIRCUIT_PURPOSE_C_MAX_ 14
+/** This circuit is being held open by circuit padding */
+#define CIRCUIT_PURPOSE_C_CIRCUIT_PADDING 15
+#define CIRCUIT_PURPOSE_C_MAX_ 15
 
-#define CIRCUIT_PURPOSE_S_HS_MIN_ 15
+#define CIRCUIT_PURPOSE_S_HS_MIN_ 16
 /** Hidden-service-side circuit purpose: at the service, waiting for
  * introductions. */
-#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 15
+#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 16
 /** Hidden-service-side circuit purpose: at the service, successfully
  * established intro. */
-#define CIRCUIT_PURPOSE_S_INTRO 16
+#define CIRCUIT_PURPOSE_S_INTRO 17
 /** Hidden-service-side circuit purpose: at the service, connecting to rend
  * point. */
-#define CIRCUIT_PURPOSE_S_CONNECT_REND 17
+#define CIRCUIT_PURPOSE_S_CONNECT_REND 18
 /** Hidden-service-side circuit purpose: at the service, rendezvous
  * established. */
-#define CIRCUIT_PURPOSE_S_REND_JOINED 18
+#define CIRCUIT_PURPOSE_S_REND_JOINED 19
 /** This circuit is used for uploading hsdirs */
-#define CIRCUIT_PURPOSE_S_HSDIR_POST 19
-#define CIRCUIT_PURPOSE_S_HS_MAX_ 19
+#define CIRCUIT_PURPOSE_S_HSDIR_POST 20
+#define CIRCUIT_PURPOSE_S_HS_MAX_ 20
 
 /** A testing circuit; not meant to be used for actual traffic. */
-#define CIRCUIT_PURPOSE_TESTING 20
+#define CIRCUIT_PURPOSE_TESTING 21
 /** A controller made this circuit and Tor should not use it. */
-#define CIRCUIT_PURPOSE_CONTROLLER 21
+#define CIRCUIT_PURPOSE_CONTROLLER 22
 /** This circuit is used for path bias probing only */
-#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 22
+#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 23
 
 /** This circuit is used for vanguards/restricted paths.
  *
@@ -123,9 +126,9 @@
  *  on-demand. When an HS operation needs to take place (e.g. connect to an
  *  intro point), these circuits are then cannibalized and repurposed to the
  *  actual needed HS purpose. */
-#define CIRCUIT_PURPOSE_HS_VANGUARDS 23
+#define CIRCUIT_PURPOSE_HS_VANGUARDS 24
 
-#define CIRCUIT_PURPOSE_MAX_ 23
+#define CIRCUIT_PURPOSE_MAX_ 24
 /** A catch-all for unrecognized purposes. Currently we don't expect
  * to make or see any circuits with this purpose. */
 #define CIRCUIT_PURPOSE_UNKNOWN 255
@@ -184,6 +187,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id);
 void channel_mark_circid_usable(channel_t *chan, circid_t id);
 time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id,
                                                   channel_t *chan);
+int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                         int reason_code);
 void circuit_set_state(circuit_t *circ, uint8_t state);
 void circuit_close_all_marked(void);
 int32_t circuit_initial_package_window(void);
@@ -213,7 +218,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void);
 void circuit_synchronize_written_or_bandwidth(const circuit_t *c,
                                               circuit_channel_direction_t dir);
 MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason,
-                                          int line, const char *file));
+                                          int line, const char *cfile));
 int circuit_get_cpath_len(origin_circuit_t *circ);
 int circuit_get_cpath_opened_len(const origin_circuit_t *);
 void circuit_clear_cpath(origin_circuit_t *circ);
@@ -225,7 +230,6 @@ int circuit_count_pending_on_channel(channel_t *chan);
 #define circuit_mark_for_close(c, reason)                               \
   circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__)
 
-void assert_cpath_layer_ok(const crypt_path_t *cp);
 MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c));
 void circuit_free_all(void);
 void circuits_handle_oom(size_t current_allocation);
diff --git a/src/core/or/circuitmux.c b/src/core/or/circuitmux.c
index 88f9ac7923..4b19a12e3c 100644
--- a/src/core/or/circuitmux.c
+++ b/src/core/or/circuitmux.c
@@ -79,6 +79,8 @@
 #include "core/or/destroy_cell_queue_st.h"
 #include "core/or/or_circuit_st.h"
 
+#include "lib/crypt_ops/crypto_util.h"
+
 /*
  * Private typedefs for circuitmux.c
  */
@@ -294,9 +296,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /* Clear n_mux */
-            circ->n_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else if (circ->magic == OR_CIRCUIT_MAGIC) {
@@ -309,12 +308,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /*
-             * It has a sensible p_chan and direction == CELL_DIRECTION_IN,
-             * so clear p_mux.
-             */
-            TO_OR_CIRCUIT(circ)->p_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else {
@@ -836,18 +829,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      */
     log_info(LD_CIRC,
              "Circuit %u on channel %"PRIu64 " was already attached to "
-             "cmux %p (trying to attach to %p)",
+             "(trying to attach to %p)",
              (unsigned)circ_id, (channel_id),
-             ((direction == CELL_DIRECTION_OUT) ?
-                circ->n_mux : TO_OR_CIRCUIT(circ)->p_mux),
              cmux);
 
     /*
      * The mux pointer on this circuit and the direction in result should
      * match; otherwise assert.
      */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == cmux);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
     tor_assert(hashent->muxinfo.direction == direction);
 
     /*
@@ -872,13 +861,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
              "Attaching circuit %u on channel %"PRIu64 " to cmux %p",
               (unsigned)circ_id, (channel_id), cmux);
 
-    /*
-     * Assert that the circuit doesn't already have a mux for this
-     * direction.
-     */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == NULL);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == NULL);
-
     /* Insert it in the map */
     hashent = tor_malloc_zero(sizeof(*hashent));
     hashent->chan_id = channel_id;
@@ -902,10 +884,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     HT_INSERT(chanid_circid_muxinfo_map, cmux->chanid_circid_map,
               hashent);
 
-    /* Set the circuit's mux for this direction */
-    if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux;
-    else TO_OR_CIRCUIT(circ)->p_mux = cmux;
-
     /* Update counters */
     ++(cmux->n_circuits);
     if (cell_count > 0) {
@@ -993,14 +971,14 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
 
     /* Consistency check: the direction must match the direction searched */
     tor_assert(last_searched_direction == hashent->muxinfo.direction);
-    /* Clear the circuit's mux for this direction */
-    if (last_searched_direction == CELL_DIRECTION_OUT) circ->n_mux = NULL;
-    else TO_OR_CIRCUIT(circ)->p_mux = NULL;
 
     /* Now remove it from the map */
     HT_REMOVE(chanid_circid_muxinfo_map, cmux->chanid_circid_map, hashent);
 
-    /* Free the hash entry */
+    /* Wipe and free the hash entry */
+    // This isn't sensitive, but we want to be sure to know if we're accessing
+    // this accidentally.
+    memwipe(hashent, 0xef, sizeof(*hashent));
     tor_free(hashent);
   }
 }
@@ -1361,4 +1339,3 @@ circuitmux_compare_muxes, (circuitmux_t *cmux_1, circuitmux_t *cmux_2))
     return 0;
   }
 }
-
diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c
index 3f83c3fd5a..07f83af0f1 100644
--- a/src/core/or/circuitmux_ewma.c
+++ b/src/core/or/circuitmux_ewma.c
@@ -38,6 +38,7 @@
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
 #include "app/config/or_options_st.h"
 
@@ -147,7 +148,9 @@ TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol)
 {
   if (!pol) return NULL;
   else {
-    tor_assert(pol->magic == EWMA_POL_DATA_MAGIC);
+    tor_assertf(pol->magic == EWMA_POL_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_DATA_MAGIC);
     return DOWNCAST(ewma_policy_data_t, pol);
   }
 }
@@ -162,7 +165,9 @@ TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
 {
   if (!pol) return NULL;
   else {
-    tor_assert(pol->magic == EWMA_POL_CIRC_DATA_MAGIC);
+    tor_assertf(pol->magic == EWMA_POL_CIRC_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_CIRC_DATA_MAGIC);
     return DOWNCAST(ewma_policy_circ_data_t, pol);
   }
 }
@@ -295,6 +300,7 @@ ewma_free_cmux_data(circuitmux_t *cmux,
   pol = TO_EWMA_POL_DATA(pol_data);
 
   smartlist_free(pol->active_circuit_pqueue);
+  memwipe(pol, 0xda, sizeof(ewma_policy_data_t));
   tor_free(pol);
 }
 
@@ -361,7 +367,7 @@ ewma_free_circ_data(circuitmux_t *cmux,
   if (!pol_circ_data) return;
 
   cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
-
+  memwipe(cdata, 0xdc, sizeof(ewma_policy_circ_data_t));
   tor_free(cdata);
 }
 
diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c
new file mode 100644
index 0000000000..7f761fed6b
--- /dev/null
+++ b/src/core/or/circuitpadding.c
@@ -0,0 +1,3099 @@
+/* Copyright (c) 2017 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.c
+ * \brief Circuit-level padding implementation
+ *
+ * \details
+ *
+ * This file implements Tor proposal 254 "Padding Negotiation" which is heavily
+ * inspired by the paper "Toward an Efficient Website Fingerprinting Defense"
+ * by M. Juarez, M. Imani, M. Perry, C. Diaz, M. Wright.
+ *
+ * In particular the code in this file describes mechanisms for clients to
+ * negotiate various types of circuit-level padding from relays.
+ *
+ * Each padding type is described by a state machine (circpad_machine_spec_t),
+ * which is also referred as a "padding machine" in this file.  Currently,
+ * these state machines are hardcoded in the source code (e.g. see
+ * circpad_machines_init()), but in the future we will be able to
+ * serialize them in the torrc or the consensus.
+ *
+ * As specified by prop#254, clients can negotiate padding with relays by using
+ * PADDING_NEGOTIATE cells. After successful padding negotiation, padding
+ * machines are assigned to the circuit in their mutable form as a
+ * circpad_machine_runtime_t.
+ *
+ * Each state of a padding state machine can be either:
+ * - A histogram that specifies inter-arrival padding delays.
+ * - Or a parametrized probability distribution that specifies inter-arrival
+ *   delays (see circpad_distribution_type_t).
+ *
+ * Padding machines start from the START state and finish with the END
+ * state. They can transition between states using the events in
+ * circpad_event_t.
+ *
+ * When a padding machine reaches the END state, it gets wiped from the circuit
+ * so that other padding machines can take over if needed (see
+ * circpad_machine_spec_transitioned_to_end()).
+ *
+ ****************************
+ * General notes:
+ *
+ * All used machines should be heap allocated and placed into
+ * origin_padding_machines/relay_padding_machines so that they get correctly
+ * cleaned up by the circpad_free_all() function.
+ **/
+
+#define CIRCUITPADDING_PRIVATE
+
+#include <math.h>
+#include "lib/math/fp.h"
+#include "lib/math/prob_distr.h"
+#include "core/or/or.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/relay.h"
+#include "feature/stats/rephist.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/channel.h"
+
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/crypto/relay_crypto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "app/config/config.h"
+
+static inline circpad_circuit_state_t circpad_circuit_state(
+                                        origin_circuit_t *circ);
+static void circpad_setup_machine_on_circ(circuit_t *on_circ,
+                                        const circpad_machine_spec_t *machine);
+static double circpad_distribution_sample(circpad_distribution_t dist);
+
+static inline void circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi);
+
+/** Cached consensus params */
+static uint8_t circpad_padding_disabled;
+static uint8_t circpad_padding_reduced;
+static uint8_t circpad_global_max_padding_percent;
+static uint16_t circpad_global_allowed_cells;
+static uint16_t circpad_max_circ_queued_cells;
+
+/** Global cell counts, for rate limiting */
+static uint64_t circpad_global_padding_sent;
+static uint64_t circpad_global_nonpadding_sent;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 1 (ie: are for client side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *origin_padding_machines = NULL;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 0 (ie: are for relay side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *relay_padding_machines = NULL;
+
+/** Loop over the current padding state machines using <b>loop_var</b> as the
+ *  loop variable. */
+#define FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                         \
+  STMT_BEGIN                                                             \
+  for (int loop_var = 0; loop_var < CIRCPAD_MAX_MACHINES; loop_var++) {
+#define FOR_EACH_CIRCUIT_MACHINE_END } STMT_END ;
+
+/** Loop over the current active padding state machines using <b>loop_var</b>
+ *  as the loop variable. If a machine is not active, skip it. */
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(loop_var, circ)            \
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                               \
+  if (!(circ)->padding_info[loop_var])                           \
+    continue;
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ;
+
+/**
+ * Free the machineinfo at an index
+ */
+static void
+circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx)
+{
+  if (circ->padding_info[idx]) {
+    log_fn(LOG_INFO,LD_CIRC, "Freeing padding info idx %d on circuit %u (%d)",
+           idx, CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+           circ->purpose);
+
+    tor_free(circ->padding_info[idx]->histogram);
+    timer_free(circ->padding_info[idx]->padding_timer);
+    tor_free(circ->padding_info[idx]);
+  }
+}
+
+/**
+ * Return true if circpad has decided to hold the circuit open for additional
+ * padding. This function is used to take and retain ownership of certain
+ * types of circuits that have padding machines on them, that have been passed
+ * to circuit_mark_for_close().
+ *
+ * circuit_mark_for_close() calls this function to ask circpad if any padding
+ * machines want to keep the circuit open longer to pad.
+ *
+ * Any non-measurement circuit that was closed for a normal, non-error reason
+ * code may be held open for up to CIRCPAD_DELAY_INFINITE microseconds between
+ * network-driven cell events.
+ *
+ * After CIRCPAD_DELAY_INFINITE microseconds of silence on a circuit, this
+ * function will no longer hold it open (it will return 0 regardless of
+ * what the machines ask for, and thus circuit_expire_old_circuits_clientside()
+ * will close the circuit after roughly 1.25hr of idle time, maximum,
+ * regardless of the padding machine state.
+ */
+int
+circpad_marked_circuit_for_padding(circuit_t *circ, int reason)
+{
+  /* If the circuit purpose is measurement or path bias, don't
+   * hold it open */
+  if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING ||
+      circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+    return 0;
+  }
+
+  /* If the circuit is closed for any reason other than these three valid,
+   * client-side close reasons, do not try to keep it open. It is probably
+   * damaged or unusable. Note this is OK with vanguards because
+   * controller-closed circuits have REASON=REQUESTED, so vanguards-closed
+   * circuits will not be held open (we want them to close ASAP). */
+  if (!(reason == END_CIRC_REASON_NONE ||
+        reason == END_CIRC_REASON_FINISHED ||
+        reason == END_CIRC_REASON_IP_NOW_REDUNDANT)) {
+    return 0;
+  }
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    circpad_machine_runtime_t *mi = circ->padding_info[i];
+    if (!mi) {
+      continue; // No padding runtime info; check next machine
+    }
+
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+
+    /* If we're in END state (NULL here), then check next machine */
+    if (!state) {
+      continue; // check next machine
+    }
+
+    /* If the machine does not want to control the circuit close itself, then
+     * check the next machine */
+    if (!circ->padding_machine[i]->manage_circ_lifetime) {
+      continue; // check next machine
+    }
+
+    /* If the machine has reached the END state, we can close. Check next
+     * machine. */
+    if (mi->current_state == CIRCPAD_STATE_END) {
+      continue; // check next machine
+    }
+
+    log_info(LD_CIRC, "Circuit %d is not marked for close because of a "
+             "pending padding machine in index %d.",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0, i);
+
+    /* If the machine has had no network events at all within the
+     * last circpad_delay_t timespan, it's in some deadlock state.
+     * Tell circuit_mark_for_close() that we don't own it anymore.
+     * This will allow circuit_expire_old_circuits_clientside() to
+     * close it.
+     */
+    if (circ->padding_info[i]->last_cell_time_sec +
+        (time_t)CIRCPAD_DELAY_MAX_SECS < approx_time()) {
+      log_notice(LD_BUG, "Circuit %d was not marked for close because of a "
+               "pending padding machine in index %d for over an hour. "
+               "Circuit is a %s",
+               CIRCUIT_IS_ORIGIN(circ) ?
+               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+               i, circuit_purpose_to_string(circ->purpose));
+
+      return 0; // abort timer reached; mark the circuit for close now
+    }
+
+    /* If we weren't marked dirty yet, let's pretend we're dirty now.
+     * ("Dirty" means that a circuit has been used for application traffic
+     * by Tor.. Dirty circuits have different expiry times, and are not
+     * considered in counts of built circuits, etc. By claiming that we're
+     * dirty, the rest of Tor will make decisions as if we were actually
+     * used by application data.
+     *
+     * This is most important for circuit_expire_old_circuits_clientside(),
+     * where we want that function to expire us after the padding machine
+     * has shut down, but using the MaxCircuitDirtiness timer instead of
+     * the idle circuit timer (again, we want this because we're not
+     * supposed to look idle to Guard nodes that can see our lifespan). */
+    if (!circ->timestamp_dirty)
+      circ->timestamp_dirty = approx_time();
+
+    /* Take ownership of the circuit */
+    circuit_change_purpose(circ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+    return 1;
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+
+  return 0; // No machine wanted to keep the circuit open; mark for close
+}
+
+/**
+ * Free all the machineinfos in <b>circ</b> that match <b>machine_num</b>.
+ *
+ * Returns true if any machineinfos with that number were freed.
+ * False otherwise. */
+static int
+free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num)
+{
+  int found = 0;
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    if (circ->padding_machine[i] &&
+        circ->padding_machine[i]->machine_num == machine_num) {
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      circ->padding_machine[i] = NULL;
+      found = 1;
+    }
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return found;
+}
+
+/**
+ * Free all padding machines and mutable info associated with circuit
+ */
+void
+circpad_circuit_free_all_machineinfos(circuit_t *circ)
+{
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    circpad_circuit_machineinfo_free_idx(circ, i);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Allocate a new mutable machineinfo structure.
+ */
+STATIC circpad_machine_runtime_t *
+circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index)
+{
+  circpad_machine_runtime_t *mi =
+    tor_malloc_zero(sizeof(circpad_machine_runtime_t));
+  mi->machine_index = machine_index;
+  mi->on_circ = on_circ;
+  mi->last_cell_time_sec = approx_time();
+
+  return mi;
+}
+
+/**
+ * Return the circpad_state_t for the current state based on the
+ * mutable info.
+ *
+ * This function returns NULL when the machine is in the end state or in an
+ * invalid state.
+ */
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    return NULL;
+  } else if (BUG(mi->current_state >= machine->num_states)) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Invalid circuit padding state %d",
+           mi->current_state);
+
+    return NULL;
+  }
+
+  return &machine->states[mi->current_state];
+}
+
+/**
+ * Get the lower bound of a histogram bin.
+ *
+ * You can obtain the upper bound using histogram_get_bin_upper_bound().
+ *
+ * This function can also be called with 'bin' set to a value equal or greater
+ * than histogram_len in which case the infinity bin is chosen and
+ * CIRCPAD_DELAY_INFINITE is returned.
+ */
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* The infinity bin has an upper bound of infinity, so make sure we return
+   * that if they ask for it. */
+  if (bin > CIRCPAD_INFINITY_BIN(state)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  return state->histogram_edges[bin] + rtt_add_usec;
+}
+
+/**
+ * Like circpad_histogram_bin_to_usec() but return the upper bound of bin.
+ * (The upper bound is included in the bin.)
+ */
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  return circpad_histogram_bin_to_usec(mi, bin+1) - 1;
+}
+
+/** Return the midpoint of the histogram bin <b>bin_index</b>. */
+static circpad_delay_t
+circpad_get_histogram_bin_midpoint(const circpad_machine_runtime_t *mi,
+                           int bin_index)
+{
+  circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index);
+  circpad_delay_t right_bound = histogram_get_bin_upper_bound(mi, bin_index);
+
+  return left_bound + (right_bound - left_bound)/2;
+}
+
+/**
+ * Return the bin that contains the usec argument.
+ * "Contains" is defined as us in [lower, upper).
+ *
+ * This function will never return the infinity bin (histogram_len-1), in order
+ * to simplify the rest of the code, so if a usec is provided that falls above
+ * the highest non-infinity bin, that bin index will be returned.
+ */
+STATIC circpad_hist_index_t
+circpad_histogram_usec_to_bin(const circpad_machine_runtime_t *mi,
+                              circpad_delay_t usec)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+  circpad_hist_index_t bin;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return 0;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  /* Walk through the bins and check the upper bound of each bin, if 'usec' is
+   * less-or-equal to that, return that bin. If rtt_estimate is enabled then
+   * add that to the upper bound of each bin.
+   *
+   * We don't want to return the infinity bin here, so don't go there. */
+  for (bin = 0 ; bin < CIRCPAD_INFINITY_BIN(state) ; bin++) {
+    if (usec <= histogram_get_bin_upper_bound(mi, bin) + rtt_add_usec) {
+      return bin;
+    }
+  }
+
+  /* We don't want to return the infinity bin here, so if we still didn't find
+   * the right bin, return the highest non-infinity bin */
+  return CIRCPAD_INFINITY_BIN(state)-1;
+}
+
+/**
+ * Return true if the machine supports token removal.
+ *
+ * Token removal is equivalent to having a mutable histogram in the
+ * circpad_machine_runtime_t mutable info. So while we're at it,
+ * let's assert that everything is consistent between the mutable
+ * runtime and the readonly machine spec.
+ */
+static inline int
+circpad_is_token_removal_supported(circpad_machine_runtime_t *mi)
+{
+  /* No runtime histogram == no token removal */
+  if (mi->histogram == NULL) {
+    /* Machines that don't want token removal are trying to avoid
+     * potentially expensive mallocs, extra memory accesses, and/or
+     * potentially expensive monotime calls. Let's minimize checks
+     * and keep this path fast. */
+    tor_assert_nonfatal(mi->histogram_len == 0);
+    return 0;
+  } else {
+    /* Machines that do want token removal are less sensitive to performance.
+     * Let's spend some time to check that our state is consistent and sane */
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return 1;
+    }
+    tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE);
+    tor_assert_nonfatal(state->histogram_len == mi->histogram_len);
+    tor_assert_nonfatal(mi->histogram_len != 0);
+    return 1;
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * This function frees any token bins allocated from a previous state
+ *
+ * Called after a state transition, or if the bins are empty.
+ */
+STATIC void
+circpad_machine_setup_tokens(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+
+  /* If this state doesn't exist, or doesn't have token removal,
+   * free any previous state's runtime histogram, and bail.
+   *
+   * If we don't have a token removal strategy, we also don't need a runtime
+   * histogram and we rely on the immutable one in machine_spec_t. */
+  if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) {
+    if (mi->histogram) {
+      tor_free(mi->histogram);
+      mi->histogram = NULL;
+      mi->histogram_len = 0;
+    }
+    return;
+  }
+
+  /* Try to avoid re-mallocing if we don't really need to */
+  if (!mi->histogram || (mi->histogram
+          && mi->histogram_len != state->histogram_len)) {
+    tor_free(mi->histogram); // null ok
+    mi->histogram = tor_malloc_zero(sizeof(circpad_hist_token_t)
+                                    *state->histogram_len);
+  }
+  mi->histogram_len = state->histogram_len;
+
+  memcpy(mi->histogram, state->histogram,
+         sizeof(circpad_hist_token_t)*state->histogram_len);
+}
+
+/**
+ * Choose a length for this state (in cells), if specified.
+ */
+static void
+circpad_choose_state_length(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  double length;
+
+  if (!state || state->length_dist.type == CIRCPAD_DIST_NONE) {
+    mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+    return;
+  }
+
+  length = circpad_distribution_sample(state->length_dist);
+  length = MAX(0, length);
+  length += state->start_length;
+
+  if (state->max_length) {
+    length = MIN(length, state->max_length);
+  }
+
+  mi->state_length = clamp_double_to_int64(length);
+
+  log_info(LD_CIRC, "State length sampled to %"PRIu64" for circuit %u",
+      mi->state_length, CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+}
+
+/**
+ * Sample a value from our iat_dist, and clamp it safely
+ * to circpad_delay_t.
+ *
+ * Before returning, add <b>delay_shift</b> (can be zero) to the sampled value.
+ */
+static circpad_delay_t
+circpad_distribution_sample_iat_delay(const circpad_state_t *state,
+                                      circpad_delay_t delay_shift)
+{
+  double val = circpad_distribution_sample(state->iat_dist);
+  /* These comparisons are safe, because the output is in the range
+   * [0, 2**32), and double has a precision of 53 bits. */
+  /* We want a positive sample value */
+  val = MAX(0, val);
+  /* Respect the maximum sample setting */
+  val = MIN(val, state->dist_max_sample_usec);
+
+  /* Now apply the shift:
+   * This addition is exact: val is at most 2**32-1, delay_shift is at most
+   * 2**32-1, and doubles have a precision of 53 bits. */
+  val += delay_shift;
+
+  /* Clamp the distribution at infinite delay val */
+  return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE);
+}
+
+/**
+ * Sample an expected time-until-next-packet delay from the histogram or
+ * probability distribution.
+ *
+ * A bin of the histogram is chosen with probability proportional to the number
+ * of tokens in each bin, and then a time value is chosen uniformly from that
+ * bin's [start,end) time range.
+ */
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  const circpad_hist_token_t *histogram = NULL;
+  circpad_hist_index_t curr_bin = 0;
+  circpad_delay_t bin_start, bin_end;
+  /* These three must all be larger than circpad_hist_token_t, because
+   * we sum several circpad_hist_token_t values across the histogram */
+  uint64_t curr_weight = 0;
+  uint64_t histogram_total_tokens = 0;
+  uint64_t bin_choice;
+
+  tor_assert(state);
+
+  if (state->iat_dist.type != CIRCPAD_DIST_NONE) {
+    /* Sample from a fixed IAT distribution and return */
+    circpad_delay_t iat_delay_shift = state->use_rtt_estimate ?
+      mi->rtt_estimate_usec + state->dist_added_shift_usec :
+      state->dist_added_shift_usec;
+    return circpad_distribution_sample_iat_delay(state, iat_delay_shift);
+  } else if (circpad_is_token_removal_supported(mi)) {
+    histogram = mi->histogram;
+    for (circpad_hist_index_t b = 0; b < state->histogram_len; b++)
+      histogram_total_tokens += histogram[b];
+  } else {
+    /* We have a histogram, but it's immutable */
+    histogram = state->histogram;
+    histogram_total_tokens = state->histogram_total_tokens;
+  }
+
+  /* If we are out of tokens, don't schedule padding. */
+  if (!histogram_total_tokens) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  bin_choice = crypto_fast_rng_get_uint64(get_thread_fast_rng(),
+                                          histogram_total_tokens);
+
+  /* Skip all the initial zero bins */
+  while (!histogram[curr_bin]) {
+    curr_bin++;
+  }
+  curr_weight = histogram[curr_bin];
+
+  // TODO: This is not constant-time. Pretty sure we don't
+  // really need it to be, though.
+  while (curr_weight < bin_choice) {
+    curr_bin++;
+    /* It should be impossible to run past the end of the histogram */
+    if (BUG(curr_bin >= state->histogram_len)) {
+      return CIRCPAD_DELAY_INFINITE;
+    }
+    curr_weight += histogram[curr_bin];
+  }
+
+  /* Do some basic checking of the current bin we are in */
+  if (BUG(curr_bin >= state->histogram_len) ||
+      BUG(histogram[curr_bin] == 0)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  // Store this index to remove the token upon callback.
+  if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) {
+    mi->chosen_bin = curr_bin;
+  }
+
+  if (curr_bin >= CIRCPAD_INFINITY_BIN(state)) {
+    if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE &&
+        mi->histogram[curr_bin] > 0) {
+      mi->histogram[curr_bin]--;
+    }
+
+    // Infinity: Don't send a padding packet. Wait for a real packet
+    // and then see if our bins are empty or what else we should do.
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state));
+
+  bin_start = circpad_histogram_bin_to_usec(mi, curr_bin);
+  /* We don't need to reduct 1 from the upper bound because the random range
+   * function below samples from [bin_start, bin_end) */
+  bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1);
+
+  /* Bin edges are monotonically increasing so this is a bug. Handle it. */
+  if (BUG(bin_start >= bin_end)) {
+    return bin_start;
+  }
+
+  return (circpad_delay_t)crypto_fast_rng_uint64_range(get_thread_fast_rng(),
+                                                       bin_start, bin_end);
+}
+
+/**
+ * Sample a value from the specified probability distribution.
+ *
+ * Uses functions from src/lib/math/prob_distr.c .
+ */
+static double
+circpad_distribution_sample(circpad_distribution_t dist)
+{
+  log_fn(LOG_DEBUG,LD_CIRC, "Sampling delay with distribution %d",
+         dist.type);
+
+  switch (dist.type) {
+    case CIRCPAD_DIST_NONE:
+      {
+        /* We should not get in here like this */
+        tor_assert_nonfatal_unreached();
+        return 0;
+      }
+    case CIRCPAD_DIST_UNIFORM:
+      {
+        // param2 is upper bound, param1 is lower
+        const struct uniform my_uniform = {
+          .base = UNIFORM(my_uniform),
+          .a = dist.param1,
+          .b = dist.param2,
+        };
+        return dist_sample(&my_uniform.base);
+      }
+    case CIRCPAD_DIST_LOGISTIC:
+      {
+      /* param1 is Mu, param2 is sigma. */
+        const struct logistic my_logistic = {
+          .base = LOGISTIC(my_logistic),
+          .mu = dist.param1,
+          .sigma = dist.param2,
+        };
+        return dist_sample(&my_logistic.base);
+      }
+    case CIRCPAD_DIST_LOG_LOGISTIC:
+      {
+        /* param1 is Alpha, param2 is 1.0/Beta */
+        const struct log_logistic my_log_logistic = {
+          .base = LOG_LOGISTIC(my_log_logistic),
+          .alpha = dist.param1,
+          .beta = dist.param2,
+        };
+        return dist_sample(&my_log_logistic.base);
+      }
+    case CIRCPAD_DIST_GEOMETRIC:
+      {
+        /* param1 is 'p' (success probability) */
+        const struct geometric my_geometric = {
+          .base = GEOMETRIC(my_geometric),
+          .p = dist.param1,
+        };
+        return dist_sample(&my_geometric.base);
+      }
+    case CIRCPAD_DIST_WEIBULL:
+      {
+        /* param1 is k, param2 is Lambda */
+        const struct weibull my_weibull = {
+          .base = WEIBULL(my_weibull),
+          .k = dist.param1,
+          .lambda = dist.param2,
+        };
+        return dist_sample(&my_weibull.base);
+      }
+    case CIRCPAD_DIST_PARETO:
+      {
+        /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */
+        const struct genpareto my_genpareto = {
+          .base = GENPARETO(my_genpareto),
+          .mu = 0,
+          .sigma = dist.param1,
+          .xi = dist.param2,
+        };
+        return dist_sample(&my_genpareto.base);
+      }
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * Find the index of the first bin whose upper bound is
+ * greater than the target, and that has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_higher_index(const circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  /* Don't remove from the infinity bin */
+  for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) {
+    if (mi->histogram[bin] &&
+        histogram_get_bin_upper_bound(mi, bin) >= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return mi->histogram_len;
+}
+
+/**
+ * Find the index of the first bin whose lower bound is lower or equal to
+ * <b>target_bin_usec</b>, and that still has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_lower_index(const circpad_machine_runtime_t *mi,
+                                  circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  for (; bin >= 0; bin--) {
+    if (mi->histogram[bin] &&
+        circpad_histogram_bin_to_usec(mi, bin) <= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return -1;
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * greater than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                    circpad_delay_t target_bin_usec)
+{
+  /* We need to remove the token from the first bin
+   * whose upper bound is greater than the target, and that
+   * has tokens remaining. */
+  circpad_hist_index_t bin = circpad_machine_first_higher_index(mi,
+                                                     target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * lower than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_machine_first_lower_index(mi,
+          target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/* Helper macro: Ensure that the bin has tokens available, and BUG out of the
+ * function if it's not the case. */
+#define ENSURE_BIN_CAPACITY(bin_index) \
+  if (BUG(mi->histogram[bin_index] == 0)) {                   \
+    return;                                                   \
+  }
+
+/**
+ * Remove a token from the closest non-empty bin to the target.
+ *
+ * If use_usec is true, measure "closest" in terms of the next closest bin
+ * midpoint.
+ *
+ * If it is false, use bin index distance only.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                     circpad_delay_t target_bin_usec,
+                                     bool use_usec)
+{
+  circpad_hist_index_t lower, higher, current;
+  circpad_hist_index_t bin_to_remove = -1;
+
+  lower = circpad_machine_first_lower_index(mi, target_bin_usec);
+  higher = circpad_machine_first_higher_index(mi, target_bin_usec);
+  current = circpad_histogram_usec_to_bin(mi, target_bin_usec);
+
+  /* Sanity check the results */
+  if (BUG(lower > current) || BUG(higher < current)) {
+    return;
+  }
+
+  /* Take care of edge cases first */
+  if (higher == mi->histogram_len && lower == -1) {
+    /* All bins are empty */
+    return;
+  } else if (higher == mi->histogram_len) {
+    /* All higher bins are empty */
+    ENSURE_BIN_CAPACITY(lower);
+    mi->histogram[lower]--;
+    return;
+  } else if (lower == -1) {
+    /* All lower bins are empty */
+    ENSURE_BIN_CAPACITY(higher);
+    mi->histogram[higher]--;
+    return;
+  }
+
+  /* Now handle the intermediate cases */
+  if (use_usec) {
+    /* Find the closest bin midpoint to the target */
+    circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower);
+    circpad_delay_t higher_usec =
+      circpad_get_histogram_bin_midpoint(mi, higher);
+
+    if (target_bin_usec < lower_usec) {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    } else if (target_bin_usec > higher_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else if (target_bin_usec-lower_usec > higher_usec-target_bin_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    }
+    mi->histogram[bin_to_remove]--;
+    log_debug(LD_CIRC, "Removing token from bin %d", bin_to_remove);
+    return;
+  } else {
+    if (current - lower > higher - current) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      mi->histogram[higher]--;
+      return;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      mi->histogram[lower]--;
+      return;
+    }
+  }
+}
+
+#undef ENSURE_BIN_CAPACITY
+
+/**
+ * Remove a token from the exact bin corresponding to the target.
+ *
+ * If it is empty, do nothing.
+ *
+ * Used for histograms with token removal.
+ */
+static void
+circpad_machine_remove_exact(circpad_machine_runtime_t *mi,
+                             circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+          target_bin_usec);
+
+  if (mi->histogram[bin] > 0)
+    mi->histogram[bin]--;
+}
+
+/**
+ * Check our state's cell limit count and tokens.
+ *
+ * Returns 1 if either limits are hit and we decide to change states,
+ * otherwise returns 0.
+ */
+static circpad_decision_t
+check_machine_token_supply(circpad_machine_runtime_t *mi)
+{
+  uint32_t histogram_total_tokens = 0;
+
+  /* Check if bins empty. This requires summing up the current mutable
+   * machineinfo histogram token total and checking if it is zero.
+   * Machineinfo does not keep a running token count. We're assuming the
+   * extra space is not worth this short loop iteration.
+   *
+   * We also do not count infinity bin in histogram totals.
+   */
+  if (circpad_is_token_removal_supported(mi)) {
+    for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++)
+      histogram_total_tokens += mi->histogram[b];
+
+    /* If we change state, we're done */
+    if (histogram_total_tokens == 0) {
+      if (circpad_internal_event_bins_empty(mi) == CIRCPAD_STATE_CHANGED)
+        return CIRCPAD_STATE_CHANGED;
+    }
+  }
+
+  if (mi->state_length == 0) {
+    return circpad_internal_event_state_length_up(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Count that a padding packet was sent.
+ *
+ * This updates our state length count, our machine rate limit counts,
+ * and if token removal is used, decrements the histogram.
+ */
+static inline void
+circpad_machine_count_padding_sent(circpad_machine_runtime_t *mi)
+{
+  /* If we have a valid state length bound, consider it */
+  if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE &&
+      !BUG(mi->state_length <= 0)) {
+    mi->state_length--;
+  }
+
+  /*
+   * Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct.
+   */
+  mi->padding_sent++;
+  if (mi->padding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  circpad_global_padding_sent++;
+
+  /* If we have a mutable histogram, reduce the token count from
+   * the chosen padding bin (this assumes we always send padding
+   * when we intended to). */
+  if (circpad_is_token_removal_supported(mi)) {
+    /* Check array bounds and token count before removing */
+    if (!BUG(mi->chosen_bin >= mi->histogram_len) &&
+        !BUG(mi->histogram[mi->chosen_bin] == 0)) {
+      mi->histogram[mi->chosen_bin]--;
+    }
+  }
+}
+
+/**
+ * Count a nonpadding packet as being sent.
+ *
+ * This function updates our overhead accounting variables, as well
+ * as decrements the state limit packet counter, if the latter was
+ * flagged as applying to non-padding as well.
+ */
+static inline void
+circpad_machine_count_nonpadding_sent(circpad_machine_runtime_t *mi)
+{
+  /* Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct. */
+  mi->nonpadding_sent++;
+  if (mi->nonpadding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  /* Update any state packet length limits that apply */
+  circpad_machine_update_state_length_for_nonpadding(mi);
+
+  /* Remove a token from the histogram, if applicable */
+  circpad_machine_remove_token(mi);
+}
+
+/**
+ * Decrement the state length counter for a non-padding packet.
+ *
+ * Only updates the state length if we're using that feature, we
+ * have a state, and the machine wants to count non-padding packets
+ * towards the state length.
+ */
+static inline void
+circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+
+  if (mi->state_length == CIRCPAD_STATE_LENGTH_INFINITE)
+    return;
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+
+  /* If we're enforcing a state length on non-padding packets,
+   * decrement it */
+  if (state->length_includes_nonpadding &&
+      mi->state_length > 0) {
+    mi->state_length--;
+  }
+}
+
+/**
+ * When a non-padding packet arrives, remove a token from the bin
+ * corresponding to the delta since last sent packet. If that bin
+ * is empty, choose a token based on the specified removal strategy
+ * in the state machine.
+ */
+STATIC void
+circpad_machine_remove_token(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+  circpad_time_t current_time;
+  circpad_delay_t target_bin_usec;
+
+  /* Dont remove any tokens if there was no padding scheduled */
+  if (!mi->padding_scheduled_at_usec) {
+    return;
+  }
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+  /* Don't remove any tokens if we're not doing token removal */
+  if (state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE)
+    return;
+
+  current_time = monotime_absolute_usec();
+
+  /* If we have scheduled padding some time in the future, we want to see what
+     bin we are in at the current time */
+  target_bin_usec = (circpad_delay_t)
+                  MIN((current_time - mi->padding_scheduled_at_usec),
+                      CIRCPAD_DELAY_INFINITE-1);
+
+  /* We are treating this non-padding cell as a padding cell, so we cancel
+     padding timer, if present. */
+  mi->padding_scheduled_at_usec = 0;
+  if (mi->is_padding_timer_scheduled) {
+    mi->is_padding_timer_scheduled = 0;
+    timer_disable(mi->padding_timer);
+  }
+
+  /* Perform the specified token removal strategy */
+  switch (state->token_removal) {
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 1);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 0);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_LOWER:
+      circpad_machine_remove_lower_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_HIGHER:
+      circpad_machine_remove_higher_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_EXACT:
+      circpad_machine_remove_exact(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_NONE:
+    default:
+      tor_assert_nonfatal_unreached();
+      log_warn(LD_BUG, "Circpad: Unknown token removal strategy %d",
+               state->token_removal);
+      break;
+  }
+}
+
+/**
+ * Send a relay command with a relay cell payload on a circuit to
+ * the particular hopnum.
+ *
+ * Hopnum starts at 1 (1=guard, 2=middle, 3=exit, etc).
+ *
+ * Payload may be null.
+ *
+ * Returns negative on error, 0 on success.
+ */
+MOCK_IMPL(STATIC signed_error_t,
+circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len))
+{
+  crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum);
+  signed_error_t ret;
+
+  /* Check that the cpath has the target hop */
+  if (!target_hop) {
+    log_fn(LOG_WARN, LD_BUG, "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier, circuit_get_cpath_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Check that the target hop is opened */
+  if (target_hop->state != CPATH_STATE_OPEN) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier,
+           circuit_get_cpath_opened_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Send the drop command to the second hop */
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ), relay_command,
+                                     (const char*)payload, payload_len,
+                                     target_hop);
+  return ret;
+}
+
+/**
+ * Callback helper to send a padding cell.
+ *
+ * This helper is called after our histogram-sampled delay period passes
+ * without another packet being sent first. If a packet is sent before this
+ * callback happens, it is canceled. So when we're called here, send padding
+ * right away.
+ *
+ * If sending this padding cell forced us to transition states return
+ * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED.
+ */
+circpad_decision_t
+circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi)
+{
+  circuit_t *circ = mi->on_circ;
+  int machine_idx = mi->machine_index;
+  mi->padding_scheduled_at_usec = 0;
+  circpad_statenum_t state = mi->current_state;
+
+  /* Make sure circuit didn't close on us */
+  if (mi->on_circ->marked_for_close) {
+    log_fn(LOG_INFO,LD_CIRC,
+           "Padding callback on circuit marked for close (%u). Ignoring.",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+         TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_CHANGED;
+  }
+
+  circpad_machine_count_padding_sent(mi);
+
+  if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+    circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ),
+                                CIRCPAD_GET_MACHINE(mi)->target_hopnum,
+                                RELAY_COMMAND_DROP, NULL, 0);
+    log_info(LD_CIRC, "Callback: Sending padding to origin circuit %u"
+             " (%d) [length: %"PRIu64"]",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier,
+             mi->on_circ->purpose, mi->state_length);
+  } else {
+    // If we're a non-origin circ, we can just send from here as if we're the
+    // edge.
+    if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) {
+      log_info(LD_CIRC, "Callback: Sending padding to circuit (%d)"
+               " [length: %"PRIu64"]", mi->on_circ->purpose, mi->state_length);
+      relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL,
+                                   0, NULL);
+      rep_hist_padding_count_write(PADDING_TYPE_DROP);
+    } else {
+      static ratelim_t cell_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC,
+                     "Too many cells (%d) in circ queue to send padding.",
+                      TO_OR_CIRCUIT(circ)->p_chan_cells.n);
+    }
+  }
+
+  /* This is a padding cell sent from the client or from the middle node,
+   * (because it's invoked from circuitpadding.c) */
+  circpad_cell_event_padding_sent(circ);
+
+  /* The circpad_cell_event_padding_sent() could cause us to transition.
+   * Check that we still have a padding machineinfo, and then check our token
+   * supply. */
+  if (circ->padding_info[machine_idx] != NULL) {
+    if (state != circ->padding_info[machine_idx]->current_state)
+      return CIRCPAD_STATE_CHANGED;
+    else
+      return check_machine_token_supply(circ->padding_info[machine_idx]);
+  } else {
+    return CIRCPAD_STATE_CHANGED;
+  }
+}
+
+/**
+ * Tor-timer compatible callback that tells us to send a padding cell.
+ *
+ * Timers are associated with circpad_machine_runtime_t's. When the machineinfo
+ * is freed on a circuit, the timers are cancelled. Since the lifetime
+ * of machineinfo is always longer than the timers, handles are not
+ * needed.
+ */
+static void
+circpad_send_padding_callback(tor_timer_t *timer, void *args,
+                              const struct monotime_t *time)
+{
+  circpad_machine_runtime_t *mi = ((circpad_machine_runtime_t*)args);
+  (void)timer; (void)time;
+
+  if (mi && mi->on_circ) {
+    assert_circuit_ok(mi->on_circ);
+    circpad_send_padding_cell_for_callback(mi);
+  } else {
+    // This shouldn't happen (represents a timer leak)
+    log_fn(LOG_WARN,LD_CIRC,
+            "Circuit closed while waiting for padding timer.");
+    tor_fragile_assert();
+  }
+
+  // TODO-MP-AP: Unify this counter with channelpadding for rephist stats
+  //total_timers_pending--;
+}
+
+/**
+ * Cache our consensus parameters upon consensus update.
+ */
+void
+circpad_new_consensus_params(const networkstatus_t *ns)
+{
+  circpad_padding_disabled =
+      networkstatus_get_param(ns, "circpad_padding_disabled",
+         0, 0, 1);
+
+  circpad_padding_reduced =
+      networkstatus_get_param(ns, "circpad_padding_reduced",
+         0, 0, 1);
+
+  circpad_global_allowed_cells =
+      networkstatus_get_param(ns, "circpad_global_allowed_cells",
+         0, 0, UINT16_MAX-1);
+
+  circpad_global_max_padding_percent =
+      networkstatus_get_param(ns, "circpad_global_max_padding_pct",
+         0, 0, 100);
+
+  circpad_max_circ_queued_cells =
+      networkstatus_get_param(ns, "circpad_max_circ_queued_cells",
+         CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX);
+}
+
+/**
+ * Return true if padding is allowed by torrc and consensus.
+ */
+static bool
+circpad_is_padding_allowed(void)
+{
+  /* If padding has been disabled in the consensus, don't send any more
+   * padding. Technically the machine should be shut down when the next
+   * machine condition check happens, but machine checks only happen on
+   * certain circuit events, and if padding is disabled due to some
+   * network overload or DoS condition, we really want to stop ASAP. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * Check this machine against its padding limits, as well as global
+ * consensus limits.
+ *
+ * We have two limits: a percent and a cell count. The cell count
+ * limit must be reached before the percent is enforced (this is to
+ * optionally allow very light padding of things like circuit setup
+ * while there is no other traffic on the circuit).
+ *
+ * TODO: Don't apply limits to machines form torrc.
+ *
+ * Returns 1 if limits are set and we've hit them. Otherwise returns 0.
+ */
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  /* If machine_padding_pct is non-zero, and we've sent more
+   * than the allowed count of padding cells, then check our
+   * percent limits for this machine. */
+  if (machine->max_padding_percent &&
+      mi->padding_sent >= machine->allowed_padding_count) {
+    uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*(uint32_t)mi->padding_sent) / total_cells >
+        machine->max_padding_percent) {
+      return 1; // limit is reached. Stop.
+    }
+  }
+
+  /* If circpad_max_global_padding_pct is non-zero, and we've
+   * sent more than the global padding cell limit, then check our
+   * global tor process percentage limit on padding. */
+  if (circpad_global_max_padding_percent &&
+      circpad_global_padding_sent >= circpad_global_allowed_cells) {
+    uint64_t total_cells = circpad_global_padding_sent +
+              circpad_global_nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*circpad_global_padding_sent) / total_cells >
+        circpad_global_max_padding_percent) {
+      return 1; // global limit reached. Stop.
+    }
+  }
+
+  return 0; // All good!
+}
+
+/**
+ * Schedule the next padding time according to the machineinfo on a
+ * circuit.
+ *
+ * The histograms represent inter-packet-delay. Whenever you get an packet
+ * event you should be scheduling your next timer (after cancelling any old
+ * ones and updating tokens accordingly).
+ *
+ * Returns 1 if we decide to transition states (due to infinity bin),
+ * 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi))
+{
+  circpad_delay_t in_usec = 0;
+  struct timeval timeout;
+  tor_assert(mi);
+
+  /* Don't schedule padding if it is disabled */
+  if (!circpad_is_padding_allowed()) {
+    static ratelim_t padding_lim = RATELIM_INIT(600);
+    log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+         "Padding has been disabled, but machine still on circuit %"PRIu64
+         ", %d",
+         mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+         mi->on_circ->n_circ_id);
+
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Don't schedule padding if we are currently in dormant mode. */
+  if (!is_participating_on_network()) {
+    log_info(LD_CIRC, "Not scheduling padding because we are dormant.");
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  // Don't pad in end (but  also don't cancel any previously
+  // scheduled padding either).
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    log_fn(LOG_INFO, LD_CIRC, "Padding end state on circuit %u",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check our padding limits */
+  if (circpad_machine_reached_padding_limit(mi)) {
+   if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Padding machine has reached padding limit on circuit %u",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier);
+    } else {
+      static ratelim_t padding_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+           "Padding machine has reached padding limit on circuit %"PRIu64
+           ", %d",
+           mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+           mi->on_circ->n_circ_id);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (mi->is_padding_timer_scheduled) {
+    /* Cancel current timer (if any) */
+    timer_disable(mi->padding_timer);
+    mi->is_padding_timer_scheduled = 0;
+  }
+
+  /* in_usec = in microseconds */
+  in_usec = circpad_machine_sample_delay(mi);
+  /* If we're using token removal, we need to know when the padding
+   * was scheduled at, so we can remove the appropriate token if
+   * a non-padding cell is sent before the padding timer expires.
+   *
+   * However, since monotime is unpredictably expensive, let's avoid
+   * using it for machines that don't need token removal. */
+  if (circpad_is_token_removal_supported(mi)) {
+    mi->padding_scheduled_at_usec = monotime_absolute_usec();
+  } else {
+    mi->padding_scheduled_at_usec = 1;
+  }
+  log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec on circuit %u", in_usec,
+       CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+
+  // Don't schedule if we have infinite delay.
+  if (in_usec == CIRCPAD_DELAY_INFINITE) {
+    return circpad_internal_event_infinity(mi);
+  }
+
+  if (mi->state_length == 0) {
+    /* If we're at length 0, that means we hit 0 after sending
+     * a cell earlier, and emitted an event for it, but
+     * for whatever reason we did not decide to change states then.
+     * So maybe the machine is waiting for bins empty, or for an
+     * infinity event later? That would be a strange machine,
+     * but there's no reason to make it impossible. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (in_usec <= 0) {
+    return circpad_send_padding_cell_for_callback(mi);
+  }
+
+  timeout.tv_sec = in_usec/TOR_USEC_PER_SEC;
+  timeout.tv_usec = (in_usec%TOR_USEC_PER_SEC);
+
+  log_fn(LOG_INFO, LD_CIRC, "\tPadding circuit %u in %u sec, %u usec",
+     CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+          (unsigned)timeout.tv_sec, (unsigned)timeout.tv_usec);
+
+  if (mi->padding_timer) {
+    timer_set_cb(mi->padding_timer, circpad_send_padding_callback, mi);
+  } else {
+    mi->padding_timer =
+        timer_new(circpad_send_padding_callback, mi);
+  }
+  timer_schedule(mi->padding_timer, &timeout);
+  mi->is_padding_timer_scheduled = 1;
+
+  // TODO-MP-AP: Unify with channelpadding counter
+  //rep_hist_padding_count_timers(++total_timers_pending);
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * If the machine transitioned to the END state, we need
+ * to check to see if it wants us to shut it down immediately.
+ * If it does, then we need to send the appropiate negotiation commands
+ * depending on which side it is.
+ *
+ * After this function is called, mi may point to freed memory. Do
+ * not access it.
+ */
+static void
+circpad_machine_spec_transitioned_to_end(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+  circuit_t *on_circ = mi->on_circ;
+
+  log_fn(LOG_INFO,LD_CIRC, "Padding machine in end state on circuit %u (%d)",
+         CIRCUIT_IS_ORIGIN(on_circ) ?
+         TO_ORIGIN_CIRCUIT(on_circ)->global_identifier : 0,
+         on_circ->purpose);
+
+  /*
+   * We allow machines to shut down and delete themselves as opposed
+   * to just going back to START or waiting forever in END so that
+   * we can handle the case where this machine started while it was
+   * the only machine that matched conditions, but *since* then more
+   * "higher ranking" machines now match the conditions, and would
+   * be given a chance to take precedence over this one in
+   * circpad_add_matching_machines().
+   *
+   * Returning to START or waiting forever in END would not give those
+   * other machines a chance to be launched, where as shutting down
+   * here does.
+   */
+  if (machine->should_negotiate_end) {
+    if (machine->is_origin_side) {
+      /* We free the machine info here so that we can be replaced
+       * by a different machine. But we must leave the padding_machine
+       * in place to wait for the negotiated response */
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(on_circ),
+                                machine->machine_num,
+                                machine->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    } else {
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_padding_negotiated(on_circ,
+                                machine->machine_num,
+                                CIRCPAD_COMMAND_STOP,
+                                CIRCPAD_RESPONSE_OK);
+      on_circ->padding_machine[machine->machine_index] = NULL;
+    }
+  }
+}
+
+/**
+ * Generic state transition function for padding state machines.
+ *
+ * Given an event and our mutable machine info, decide if/how to
+ * transition to a different state, and perform actions accordingly.
+ *
+ * Returns 1 if we transition states, 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_spec_transition,(circpad_machine_runtime_t *mi,
+                            circpad_event_t event))
+{
+  const circpad_state_t *state =
+      circpad_machine_current_state(mi);
+
+  /* If state is null we are in the end state. */
+  if (!state) {
+    /* If we in end state we don't pad no matter what. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check if this event is ignored or causes a cancel */
+  if (state->next_state[event] == CIRCPAD_STATE_IGNORE) {
+    return CIRCPAD_STATE_UNCHANGED;
+  } else if (state->next_state[event] == CIRCPAD_STATE_CANCEL) {
+    /* Check cancel events and cancel any pending padding */
+    mi->padding_scheduled_at_usec = 0;
+    if (mi->is_padding_timer_scheduled) {
+      mi->is_padding_timer_scheduled = 0;
+      /* Cancel current timer (if any) */
+      timer_disable(mi->padding_timer);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  } else {
+    circpad_statenum_t s = state->next_state[event];
+    /* See if we need to transition to any other states based on this event.
+     * Whenever a transition happens, even to our own state, we schedule
+     * padding.
+     *
+     * So if a state only wants to schedule padding for an event, it specifies
+     * a transition to itself. All non-specified events are ignored.
+     */
+    log_fn(LOG_INFO, LD_CIRC,
+           "Circuit %u circpad machine %d transitioning from %u to %u",
+             CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+           mi->machine_index, mi->current_state, s);
+
+    /* If this is not the same state, switch and init tokens,
+     * otherwise just reschedule padding. */
+    if (mi->current_state != s) {
+      mi->current_state = s;
+      circpad_machine_setup_tokens(mi);
+      circpad_choose_state_length(mi);
+
+      /* If we transition to the end state, check to see
+       * if this machine wants to be shut down at end */
+      if (s == CIRCPAD_STATE_END) {
+        circpad_machine_spec_transitioned_to_end(mi);
+        /* We transitioned but we don't pad in end. Also, mi
+         * may be freed. Returning STATE_CHANGED prevents us
+         * from accessing it in any callers of this function. */
+        return CIRCPAD_STATE_CHANGED;
+      }
+
+      /* We transitioned to a new state, schedule padding */
+      circpad_machine_schedule_padding(mi);
+      return CIRCPAD_STATE_CHANGED;
+    }
+
+    /* We transitioned back to the same state. Schedule padding,
+     * and inform if that causes a state transition. */
+    return circpad_machine_schedule_padding(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Estimate the circuit RTT from the current middle hop out to the
+ * end of the circuit.
+ *
+ * We estimate RTT by calculating the time between "receive" and
+ * "send" at a middle hop. This is because we "receive" a cell
+ * from the origin, and then relay it towards the exit before a
+ * response comes back. It is that response time from the exit side
+ * that we want to measure, so that we can make use of it for synthetic
+ * response delays.
+ */
+static void
+circpad_estimate_circ_rtt_on_received(circuit_t *circ,
+                                      circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update)
+    return;
+
+  /* If we already have a last received packet time, that means we
+   * did not get a response before this packet. The RTT estimate
+   * only makes sense if we do not have multiple packets on the
+   * wire, so stop estimating if this is the second packet
+   * back to back. However, for the first set of back-to-back
+   * packets, we can wait until the very first response comes back
+   * to us, to measure that RTT (for the response to optimistic
+   * data, for example). Hence stop_rtt_update is only checked
+   * in this received side function, and not in send side below.
+   */
+  if (mi->last_received_time_usec) {
+    /* We also allow multiple back-to-back packets if the circuit is not
+     * opened, to handle var cells.
+     * XXX: Will this work with out var cell plans? Maybe not,
+     * since we're opened at the middle hop as soon as we process
+     * one var extend2 :/ */
+    if (circ->state == CIRCUIT_STATE_OPEN) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Stopping padding RTT estimation on circuit (%"PRIu64
+           ", %d) after two back to back packets. Current RTT: %d",
+           circ->n_chan ?  circ->n_chan->global_identifier : 0,
+           circ->n_circ_id, mi->rtt_estimate_usec);
+      mi->stop_rtt_update = 1;
+
+      if (!mi->rtt_estimate_usec) {
+        static ratelim_t rtt_lim = RATELIM_INIT(600);
+        log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+          "Circuit got two cells back to back before estimating RTT.");
+      }
+    }
+  } else {
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return;
+    }
+
+    /* Since monotime is unpredictably expensive, only update this field
+     * if rtt estimates are needed. Otherwise, stop the rtt update. */
+    if (state->use_rtt_estimate) {
+      mi->last_received_time_usec = monotime_absolute_usec();
+    } else {
+      /* Let's fast-path future decisions not to update rtt if the
+       * feature is not in use. */
+      mi->stop_rtt_update = 1;
+    }
+  }
+}
+
+/**
+ * Handles the "send" side of RTT calculation at middle nodes.
+ *
+ * This function calculates the RTT from the middle to the end
+ * of the circuit by subtracting the last received cell timestamp
+ * from the current time. It allows back-to-back cells until
+ * the circuit is opened, to allow for var cell handshakes.
+ * XXX: Check our var cell plans to make sure this will work.
+ */
+static void
+circpad_estimate_circ_rtt_on_send(circuit_t *circ,
+                                  circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ))
+    return;
+
+  /* If last_received_time_usec is non-zero, we are waiting for a response
+   * from the exit side. Calculate the time delta and use it as RTT. */
+  if (mi->last_received_time_usec) {
+    circpad_time_t rtt_time = monotime_absolute_usec() -
+        mi->last_received_time_usec;
+
+    /* Reset the last RTT packet time, so we can tell if two cells
+     * arrive back to back */
+    mi->last_received_time_usec = 0;
+
+    /* Use INT32_MAX to ensure the addition doesn't overflow */
+    if (rtt_time >= INT32_MAX) {
+      log_fn(LOG_WARN,LD_CIRC,
+             "Circuit padding RTT estimate overflowed: %"PRIu64
+             " vs %"PRIu64, monotime_absolute_usec(),
+               mi->last_received_time_usec);
+      return;
+    }
+
+    /* If the old RTT estimate is lower than this one, use this one, because
+     * the circuit is getting longer. If this estimate is somehow
+     * faster than the previous, then maybe that was network jitter, or a
+     * bad monotonic clock source (so our ratchet returned a zero delta).
+     * In that case, average them. */
+    if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) {
+      mi->rtt_estimate_usec = (circpad_delay_t)rtt_time;
+    } else {
+      mi->rtt_estimate_usec += (circpad_delay_t)rtt_time;
+      mi->rtt_estimate_usec /= 2;
+    }
+  } else if (circ->state == CIRCUIT_STATE_OPEN) {
+    /* If last_received_time_usec is zero, then we have gotten two cells back
+     * to back. Stop estimating RTT in this case. Note that we only
+     * stop RTT update if the circuit is opened, to allow for RTT estimates
+     * of var cells during circ setup. */
+    if (!mi->rtt_estimate_usec && !mi->stop_rtt_update) {
+      static ratelim_t rtt_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+        "Circuit sent two cells back to back before estimating RTT.");
+    }
+    mi->stop_rtt_update = 1;
+  }
+}
+
+/**
+ * A "non-padding" cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_sent(circuit_t *on_circ)
+{
+  /* Update global cell count */
+  circpad_global_nonpadding_sent++;
+
+  /* If there are no machines then this loop should not iterate */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]);
+
+    /* Then, do accounting */
+    circpad_machine_count_nonpadding_sent(on_circ->padding_info[i]);
+
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                                 CIRCPAD_EVENT_NONPADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/** Check if this cell or circuit are related to circuit padding and handle
+ *  them if so.  Return 0 if the cell was handled in this subsystem and does
+ *  not need any other consideration, otherwise return 1.
+ */
+int
+circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                            crypt_path_t *layer_hint,
+                            const relay_header_t *rh)
+{
+  /* First handle the padding commands, since we want to ignore any other
+   * commands if this circuit is padding-specific. */
+  switch (rh->command) {
+    case RELAY_COMMAND_DROP:
+      /* Already examined in circpad_deliver_recognized_relay_cell_events */
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATE:
+      circpad_handle_padding_negotiate(circ, cell);
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATED:
+      if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0)
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
+      return 0;
+  }
+
+  /* If this is a padding circuit we don't need to parse any other commands
+   * than the padding ones. Just drop them to the floor.
+   *
+   * Note: we deliberately do not call circuit_read_valid_data() here. The
+   * vanguards addon (specifically the 'bandguards' component's dropped cell
+   * detection) will thus close this circuit, as it would for any other
+   * unexpected cell. However, default tor will *not* close the circuit.
+   *
+   * This is intentional. We are not yet certain that is it optimal to keep
+   * padding circuits open in cases like these, rather than closing them.
+   * We suspect that continuing to pad is optimal against a passive classifier,
+   * but as soon as the adversary is active (even as a client adversary) this
+   * might change.
+   *
+   * So as a way forward, we log the cell command and circuit number, to
+   * help us enumerate the most common instances of this in testing with
+   * vanguards, to see which are common enough to verify and handle
+   * properly.
+   * - Mike
+   */
+  if (circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Ignored cell (%d) that arrived in padding circuit "
+                      " %u.", rh->command, CIRCUIT_IS_ORIGIN(circ) ?
+                           TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * A "non-padding" cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_received(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]);
+
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                               CIRCPAD_EVENT_NONPADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_padding_sent(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+
+      on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                             CIRCPAD_EVENT_PADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_padding_received(circuit_t *on_circ)
+{
+  /* identical to padding sent */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                              CIRCPAD_EVENT_PADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * An "infinite" delay has ben chosen from one of our histograms.
+ *
+ * "Infinite" delays mean don't send padding -- but they can also
+ * mean transition to another state depending on the state machine
+ * definitions. Check the rules and react accordingly.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY);
+}
+
+/**
+ * All of the bins of our current state's histogram's are empty.
+ *
+ * Check to see if this means transition to another state, and if
+ * not, refill the tokens.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *mi)
+{
+  if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY)
+      == CIRCPAD_STATE_CHANGED) {
+    return CIRCPAD_STATE_CHANGED;
+  } else {
+    /* If we dont transition, then we refill the tokens */
+    circpad_machine_setup_tokens(mi);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+}
+
+/**
+ * This state has used up its cell count. Emit the event and
+ * see if we transition.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_state_length_up(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT);
+}
+
+/**
+ * Returns true if the circuit matches the conditions.
+ */
+static inline bool
+circpad_machine_conditions_met(origin_circuit_t *circ,
+                               const circpad_machine_spec_t *machine)
+{
+  /* If padding is disabled, no machines should match/apply. This has
+   * the effect of shutting down all machines, and not adding any more. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding)
+    return 0;
+
+  /* If the consensus or our torrc has selected reduced connection padding,
+   * then only allow this machine if it is flagged as acceptable under
+   * reduced padding conditions */
+  if (circpad_padding_reduced || get_options()->ReducedCircuitPadding) {
+    if (!machine->conditions.reduced_padding_ok)
+      return 0;
+  }
+
+  if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose)
+      & machine->conditions.purpose_mask))
+    return 0;
+
+  if (machine->conditions.requires_vanguards) {
+    const or_options_t *options = get_options();
+
+    /* Pinned middles are effectively vanguards */
+    if (!(options->HSLayer2Nodes || options->HSLayer3Nodes))
+      return 0;
+  }
+
+  /* We check for any bits set in the circuit state mask so that machines
+   * can say any of the following through their state bitmask:
+   * "I want to apply to circuits with either streams or no streams"; OR
+   * "I only want to apply to circuits with streams"; OR
+   * "I only want to apply to circuits without streams". */
+  if (!(circpad_circuit_state(circ) & machine->conditions.state_mask))
+    return 0;
+
+  if (circuit_get_cpath_opened_len(circ) < machine->conditions.min_hops)
+    return 0;
+
+  return 1;
+}
+
+/**
+ * Returns a minimized representation of the circuit state.
+ *
+ * The padding code only cares if the circuit is building,
+ * opened, used for streams, and/or still has relay early cells.
+ * This returns a bitmask of all state properities that apply to
+ * this circuit.
+ */
+static inline
+circpad_circuit_state_t
+circpad_circuit_state(origin_circuit_t *circ)
+{
+  circpad_circuit_state_t retmask = 0;
+
+  if (circ->p_streams)
+    retmask |= CIRCPAD_CIRC_STREAMS;
+  else
+    retmask |= CIRCPAD_CIRC_NO_STREAMS;
+
+  /* We use has_opened to prevent cannibialized circs from flapping. */
+  if (circ->has_opened)
+    retmask |= CIRCPAD_CIRC_OPENED;
+  else
+    retmask |= CIRCPAD_CIRC_BUILDING;
+
+  if (circ->remaining_relay_early_cells > 0)
+    retmask |= CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  else
+    retmask |= CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+
+  return retmask;
+}
+
+/**
+ * Convert a normal circuit purpose into a bitmask that we can
+ * use for determining matching circuits.
+ */
+circpad_purpose_mask_t
+circpad_circ_purpose_to_mask(uint8_t circ_purpose)
+{
+  /* Treat OR circ purposes as ignored. They should not be passed here*/
+  if (BUG(circ_purpose <= CIRCUIT_PURPOSE_OR_MAX_)) {
+    return 0;
+  }
+
+  /* Treat new client circuit purposes as "OMG ITS EVERYTHING".
+   * This also should not happen */
+  if (BUG(circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1 > 32)) {
+    return CIRCPAD_PURPOSE_ALL;
+  }
+
+  /* Convert the purpose to a bit position */
+  return 1 << (circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1);
+}
+
+/**
+ * Shut down any machines whose conditions no longer match
+ * the current circuit.
+ */
+static void
+circpad_shutdown_old_machines(origin_circuit_t *on_circ)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    if (!circpad_machine_conditions_met(on_circ,
+                                        circ->padding_machine[i])) {
+      // Clear machineinfo (frees timers)
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      // Send padding negotiate stop
+      circpad_negotiate_padding(on_circ,
+                                circ->padding_machine[i]->machine_num,
+                                circ->padding_machine[i]->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Negotiate new machines that would apply to this circuit, given the machines
+ * inside <b>machines_sl</b>.
+ *
+ * This function checks to see if we have any free machine indexes,
+ * and for each free machine index, it initializes the most recently
+ * added origin-side padding machine that matches the target machine
+ * index and circuit conditions, and negotiates it with the appropriate
+ * middle relay.
+ */
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+#ifdef TOR_UNIT_TESTS
+  /* Tests don't have to init our padding machines */
+  if (!machines_sl)
+    return;
+#endif
+
+  /* If padding negotiation failed before, do not try again */
+  if (on_circ->padding_negotiation_failed)
+    return;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* If there is a padding machine info, this index is occupied.
+     * No need to check conditions for this index. */
+    if (circ->padding_info[i])
+      continue;
+
+    /* We have a free machine index. Check the origin padding
+     * machines in reverse order, so that more recently added
+     * machines take priority over older ones. */
+    SMARTLIST_FOREACH_REVERSE_BEGIN(machines_sl,
+                                    circpad_machine_spec_t *,
+                                    machine) {
+      /* Machine definitions have a specific target machine index.
+       * This is so event ordering is deterministic with respect
+       * to which machine gets events first when there are two
+       * machines installed on a circuit. Make sure we only
+       * add this machine if its target machine index is free. */
+      if (machine->machine_index == i &&
+          circpad_machine_conditions_met(on_circ, machine)) {
+
+        // We can only replace this machine if the target hopnum
+        // is the same, otherwise we'll get invalid data
+        if (circ->padding_machine[i]) {
+          if (circ->padding_machine[i]->target_hopnum !=
+              machine->target_hopnum)
+            continue;
+          /* Replace it. (Don't free - is global). */
+          circ->padding_machine[i] = NULL;
+        }
+
+        /* Set up the machine immediately so that the slot is occupied.
+         * We will tear it down on error return, or if there is an error
+         * response from the relay. */
+        circpad_setup_machine_on_circ(circ, machine);
+        if (circpad_negotiate_padding(on_circ, machine->machine_num,
+                                  machine->target_hopnum,
+                                  CIRCPAD_COMMAND_START) < 0) {
+          log_info(LD_CIRC,
+                   "Padding not negotiated. Cleaning machine from circuit %u",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+          circpad_circuit_machineinfo_free_idx(circ, i);
+          circ->padding_machine[i] = NULL;
+          on_circ->padding_negotiation_failed = 1;
+        } else {
+          /* Success. Don't try any more machines */
+          return;
+        }
+      }
+    } SMARTLIST_FOREACH_END(machine);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Event that tells us we added a hop to an origin circuit.
+ *
+ * This event is used to decide if we should create a padding machine
+ * on a circuit.
+ */
+void
+circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ)
+{
+  /* Since our padding conditions do not specify a max_hops,
+   * all we can do is add machines here */
+  circpad_add_matching_machines(on_circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is now built.
+ *
+ * Shut down any machines that only applied to un-built circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_built(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Circpad purpose changed event.
+ *
+ * Shut down any machines that don't apply to our circ purpose.
+ * Activate any new ones that do.
+ */
+void
+circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is out of RELAY_EARLY
+ * cells.
+ *
+ * Shut down any machines that only applied to RELAY_EARLY circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams attached event.
+ *
+ * Called from link_apconn_to_circ() and handle_hs_exit_conn()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams detached event.
+ *
+ * Called from circuit_detach_stream()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Verify that padding is coming from the expected hop.
+ *
+ * Returns true if from_hop matches the target hop from
+ * one of our padding machines.
+ *
+ * Returns false if we're not an origin circuit, or if from_hop
+ * does not match one of the padding machines.
+ */
+bool
+circpad_padding_is_from_expected_hop(circuit_t *circ,
+                                     crypt_path_t *from_hop)
+{
+  crypt_path_t *target_hop = NULL;
+  if (!CIRCUIT_IS_ORIGIN(circ))
+    return 0;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* We have to check padding_machine and not padding_info/active
+     * machines here because padding may arrive after we shut down a
+     * machine. The info is gone, but the padding_machine waits
+     * for the padding_negotiated response to come back. */
+    if (!circ->padding_machine[i])
+      continue;
+
+    target_hop = circuit_get_cpath_hop(TO_ORIGIN_CIRCUIT(circ),
+                    circ->padding_machine[i]->target_hopnum);
+
+    if (target_hop == from_hop)
+      return 1;
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return 0;
+}
+
+/**
+ * Deliver circpad events for an "unrecognized cell".
+ *
+ * Unrecognized cells are sent to relays and are forwarded
+ * onto the next hop of their circuits. Unrecognized cells
+ * are by definition not padding. We need to tell relay-side
+ * state machines that a non-padding cell was sent or received,
+ * depending on the direction, so they can update their histograms
+ * and decide to pad or not.
+ */
+void
+circpad_deliver_unrecognized_cell_events(circuit_t *circ,
+                                         cell_direction_t dir)
+{
+  // We should never see unrecognized cells at origin.
+  // Our caller emits a warn when this happens.
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    return;
+  }
+
+  if (dir == CELL_DIRECTION_OUT) {
+    /* When direction is out (away from origin), then we received non-padding
+       cell coming from the origin to us. */
+    circpad_cell_event_nonpadding_received(circ);
+  } else if (dir == CELL_DIRECTION_IN) {
+    /* It's in and not origin, so the cell is going away from us.
+     * So we are relaying a non-padding cell towards the origin. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for "recognized" relay cells.
+ *
+ * Recognized cells are destined for this hop, either client or middle.
+ * Check if this is a padding cell or not, and send the appropiate
+ * received event.
+ */
+void
+circpad_deliver_recognized_relay_cell_events(circuit_t *circ,
+                                             uint8_t relay_command,
+                                             crypt_path_t *layer_hint)
+{
+  if (relay_command == RELAY_COMMAND_DROP) {
+    rep_hist_padding_count_read(PADDING_TYPE_DROP);
+
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      if (circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), 0);
+      } else {
+        /* This is unexpected padding. Ignore it for now. */
+        return;
+      }
+    }
+
+    /* The cell should be recognized by now, which means that we are on the
+       destination, which means that we received a padding cell. We might be
+       the client or the Middle node, still, because leaky-pipe. */
+    circpad_cell_event_padding_received(circ);
+    log_fn(LOG_INFO, LD_CIRC, "Got padding cell on %s circuit %u.",
+           CIRCUIT_IS_ORIGIN(circ) ? "origin" : "non-origin",
+           CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+  } else {
+    /* We received a non-padding cell on the edge */
+    circpad_cell_event_nonpadding_received(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for relay cells sent from us.
+ *
+ * If this is a padding cell, update our padding stats
+ * and deliver the event. Otherwise just deliver the event.
+ */
+void
+circpad_deliver_sent_relay_cell_events(circuit_t *circ,
+                                       uint8_t relay_command)
+{
+  /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in
+   * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed
+   * or processed here).
+   *
+   * We do generate events for PADDING_NEGOTIATE and PADDING_NEGOTIATED cells.
+   */
+  if (relay_command == RELAY_COMMAND_DROP) {
+    /* Optimization: The event for RELAY_COMMAND_DROP is sent directly
+     * from circpad_send_padding_cell_for_callback(). This is to avoid
+     * putting a cell_t and a relay_header_t on the stack repeatedly
+     * if we decide to send a long train of padding cells back-to-back
+     * with 0 delay. So we do nothing here. */
+    return;
+  } else {
+    /* This is a non-padding cell sent from the client or from
+     * this node. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Initialize the states array for a circpad machine.
+ */
+void
+circpad_machine_states_init(circpad_machine_spec_t *machine,
+                            circpad_statenum_t num_states)
+{
+  if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) {
+    num_states = CIRCPAD_MAX_MACHINE_STATES;
+  }
+
+  machine->num_states = num_states;
+  machine->states = tor_malloc_zero(sizeof(circpad_state_t)*num_states);
+
+  /* Initialize the default next state for all events to
+   * "ignore" -- if events aren't specified, they are ignored. */
+  for (circpad_statenum_t s = 0; s < num_states; s++) {
+    for (int e = 0; e < CIRCPAD_NUM_EVENTS; e++) {
+      machine->states[s].next_state[e] = CIRCPAD_STATE_IGNORE;
+    }
+  }
+}
+
+static void
+circpad_setup_machine_on_circ(circuit_t *on_circ,
+                              const circpad_machine_spec_t *machine)
+{
+  if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up non-origin machine on origin circuit!");
+    return;
+  }
+
+  if (!CIRCUIT_IS_ORIGIN(on_circ) && machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up origin machine on non-origin circuit!");
+    return;
+  }
+
+  IF_BUG_ONCE(on_circ->padding_machine[machine->machine_index] != NULL) {
+    return;
+  }
+  IF_BUG_ONCE(on_circ->padding_info[machine->machine_index] != NULL) {
+    return;
+  }
+
+  /* Log message */
+  if (CIRCUIT_IS_ORIGIN(on_circ)) {
+    log_info(LD_CIRC, "Registering machine %s to origin circ %u (%d)",
+             machine->name,
+             TO_ORIGIN_CIRCUIT(on_circ)->global_identifier, on_circ->purpose);
+  } else {
+    log_info(LD_CIRC, "Registering machine %s to non-origin circ (%d)",
+             machine->name, on_circ->purpose);
+  }
+
+  on_circ->padding_info[machine->machine_index] =
+      circpad_circuit_machineinfo_new(on_circ, machine->machine_index);
+  on_circ->padding_machine[machine->machine_index] = machine;
+}
+
+/** Validate a single state of a padding machine */
+static bool
+padding_machine_state_is_valid(const circpad_state_t *state)
+{
+  int b;
+  uint32_t tokens_count = 0;
+  circpad_delay_t prev_bin_edge = 0;
+
+  /* We only validate histograms */
+  if (!state->histogram_len) {
+    return true;
+  }
+
+  /* We need at least two bins in a histogram */
+  if (state->histogram_len < 2) {
+    log_warn(LD_CIRC, "You can't have a histogram with less than 2 bins");
+    return false;
+  }
+
+  /* For each machine state, if it's a histogram, make sure all the
+   * histogram edges are well defined (i.e. are strictly monotonic). */
+  for (b = 0 ; b < state->histogram_len ; b++) {
+    /* Check that histogram edges are strictly increasing. Ignore the first
+     * edge since it can be zero. */
+    if (prev_bin_edge >= state->histogram_edges[b] && b > 0) {
+      log_warn(LD_CIRC, "Histogram edges are not increasing [%u/%u]",
+               prev_bin_edge, state->histogram_edges[b]);
+      return false;
+    }
+
+    prev_bin_edge = state->histogram_edges[b];
+
+    /* Also count the number of tokens as we go through the histogram states */
+    tokens_count += state->histogram[b];
+  }
+  /* Verify that the total number of tokens is correct */
+  if (tokens_count != state->histogram_total_tokens) {
+    log_warn(LD_CIRC, "Histogram token count is wrong [%u/%u]",
+             tokens_count, state->histogram_total_tokens);
+    return false;
+  }
+
+  return true;
+}
+
+/** Basic validation of padding machine */
+static bool
+padding_machine_is_valid(const circpad_machine_spec_t *machine)
+{
+  int i;
+
+  /* Validate the histograms of the padding machine */
+  for (i = 0 ; i < machine->num_states ; i++) {
+    if (!padding_machine_state_is_valid(&machine->states[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/* Validate and register <b>machine</b> into <b>machine_list</b>. If
+ * <b>machine_list</b> is NULL, then just validate. */
+void
+circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                 smartlist_t *machine_list)
+{
+  if (!padding_machine_is_valid(machine)) {
+    log_warn(LD_CIRC, "Machine #%u is invalid. Ignoring.",
+             machine->machine_num);
+    return;
+  }
+
+  if (machine_list) {
+    smartlist_add(machine_list, machine);
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+/* These padding machines are only used for tests pending #28634. */
+static void
+circpad_circ_client_machine_init(void)
+{
+  circpad_machine_spec_t *circ_client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  circ_client_machine->conditions.min_hops = 2;
+  circ_client_machine->conditions.state_mask =
+      CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circ_client_machine->conditions.reduced_padding_ok = 1;
+
+  circ_client_machine->target_hopnum = 2;
+  circ_client_machine->is_origin_side = 1;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_client_machine, 3);
+
+  circ_client_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* If we are in burst state, and we send a non-padding cell, then we cancel
+     the timer for the next padding cell:
+     We dont want to send fake extends when actual extends are going on */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1]= 1000000;
+
+  /* We have 5 tokens in the histogram, which means that all circuits will look
+   * like they have 7 hops (since we start this machine after the second hop,
+   * and tokens are decremented for any valid hops, and fake extends are
+   * used after that -- 2+5==7). */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5;
+
+  circ_client_machine->machine_num = smartlist_len(origin_padding_machines);
+  circpad_register_padding_machine(circ_client_machine,
+                                   origin_padding_machines);
+}
+
+static void
+circpad_circ_responder_machine_init(void)
+{
+  circpad_machine_spec_t *circ_responder_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Shut down the machine after we've sent enough packets */
+  circ_responder_machine->should_negotiate_end = 1;
+
+  /* The relay-side doesn't care what hopnum it is, but for consistency,
+   * let's match the client */
+  circ_responder_machine->target_hopnum = 2;
+  circ_responder_machine->is_origin_side = 0;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_responder_machine, 3);
+
+  /* This is the settings of the state machine. In the future we are gonna
+     serialize this into the consensus or the torrc */
+
+  /* We transition to the burst state on padding receive and on non-padding
+   * recieve */
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we _stay_ in the burst state when a non-padding
+   * is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we transition to the gap state when we receive a
+   * padding cell */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* These describe the padding charasteristics when in burst state */
+
+  /* use_rtt_estimate tries to estimate how long padding cells take to go from
+     C->M, and uses that as what as the base of the histogram */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+  /* The histogram is 2 bins: an empty one, and infinity */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1] =
+    1000000;
+  /* During burst state we wait forever for padding to arrive.
+
+     We are waiting for a padding cell from the client to come in, so that we
+     respond, and we immitate how extend looks like */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  // Only infinity bin:
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      histogram_total_tokens = 1;
+
+  /* From the gap state, we _stay_ in the gap state, when we receive padding
+   * or non padding */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* And from the gap state, we go to the end, when the bins are empty or a
+   * non-padding cell is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_END;
+
+  // FIXME: Tune this histogram
+
+  /* The gap state is the delay you wait after you receive a padding cell
+     before you send a padding response */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6;
+  /* Specify histogram bins */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[1]= 1000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[2]= 2000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[3]= 4000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[4]= 8000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[5]= 16000;
+  /* Specify histogram tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[3] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1;
+  /* Total number of tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6;
+
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC;
+
+  circ_responder_machine->machine_num = smartlist_len(relay_padding_machines);
+  circpad_register_padding_machine(circ_responder_machine,
+                                   relay_padding_machines);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize all of our padding machines.
+ *
+ * This is called at startup. It sets up some global machines, and then
+ * loads some from torrc, and from the tor consensus.
+ */
+void
+circpad_machines_init(void)
+{
+  tor_assert_nonfatal(origin_padding_machines == NULL);
+  tor_assert_nonfatal(relay_padding_machines == NULL);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  /* Register machines for hiding client-side intro circuits */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+
+  /* Register machines for hiding client-side rendezvous circuits */
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  // TODO: Parse machines from consensus and torrc
+#ifdef TOR_UNIT_TESTS
+  circpad_circ_client_machine_init();
+  circpad_circ_responder_machine_init();
+#endif
+}
+
+/**
+ * Free our padding machines
+ */
+void
+circpad_machines_free(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH(origin_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(origin_padding_machines);
+  }
+
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH(relay_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/**
+ * Check the Protover info to see if a node supports padding.
+ */
+static bool
+circpad_node_supports_padding(const node_t *node)
+{
+  if (node->rs) {
+    log_fn(LOG_INFO, LD_CIRC, "Checking padding: %s",
+           node->rs->pv.supports_hs_setup_padding ?
+              "supported" : "unsupported");
+    return node->rs->pv.supports_hs_setup_padding;
+  }
+
+  log_fn(LOG_INFO, LD_CIRC, "Empty routerstatus in padding check");
+  return 0;
+}
+
+/**
+ * Get a node_t for the nth hop in our circuit, starting from 1.
+ *
+ * Returns node_t from the consensus for that hop, if it is opened.
+ * Otherwise returns NULL.
+ */
+MOCK_IMPL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop))
+{
+  crypt_path_t *iter = circuit_get_cpath_hop(circ, hop);
+
+  if (!iter || iter->state != CPATH_STATE_OPEN)
+    return NULL;
+
+  return node_get_by_id(iter->extend_info->identity_digest);
+}
+
+/**
+ * Return true if a particular circuit supports padding
+ * at the desired hop.
+ */
+static bool
+circpad_circuit_supports_padding(origin_circuit_t *circ,
+                                 int target_hopnum)
+{
+  const node_t *hop;
+
+  if (!(hop = circuit_get_nth_node(circ, target_hopnum))) {
+    return 0;
+  }
+
+  return circpad_node_supports_padding(hop);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_negotiate_padding(origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command)
+{
+  circpad_negotiate_t type;
+  cell_t cell;
+  ssize_t len;
+
+  /* Check that the target hop lists support for padding in
+   * its ProtoVer fields */
+  if (!circpad_circuit_supports_padding(circ, target_hopnum)) {
+    return -1;
+  }
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiate_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiate_set_command(&type, command);
+  circpad_negotiate_set_version(&type, 0);
+  circpad_negotiate_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return -1;
+
+  log_fn(LOG_INFO,LD_CIRC,
+         "Negotiating padding on circuit %u (%d), command %d",
+         circ->global_identifier, TO_CIRCUIT(circ)->purpose, command);
+
+  return circpad_send_command_to_hop(circ, target_hopnum,
+                                     RELAY_COMMAND_PADDING_NEGOTIATE,
+                                     cell.payload, len);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns 1 if successful (or already set up), 0 otherwise.
+ */
+bool
+circpad_padding_negotiated(circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response)
+{
+  circpad_negotiated_t type;
+  cell_t cell;
+  ssize_t len;
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiated_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiated_set_command(&type, command);
+  circpad_negotiated_set_response(&type, response);
+  circpad_negotiated_set_version(&type, 0);
+  circpad_negotiated_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiated_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return 0;
+
+  /* Use relay_send because we're from the middle to the origin. We don't
+   * need to specify a target hop or layer_hint. */
+  return relay_send_command_from_edge(0, circ,
+                                      RELAY_COMMAND_PADDING_NEGOTIATED,
+                                      (void*)cell.payload,
+                                      (size_t)len, NULL) == 0;
+}
+
+/**
+ * Parse and react to a padding_negotiate cell.
+ *
+ * This is called at the middle node upon receipt of the client's choice of
+ * state machine, so that it can use the requested state machine index, if
+ * it is available.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell)
+{
+  int retval = 0;
+  circpad_negotiate_t *negotiate;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiate cell unsupported at origin (circuit %u)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATE cell; dropping.");
+    return -1;
+  }
+
+  if (negotiate->command == CIRCPAD_COMMAND_STOP) {
+    /* Free the machine corresponding to this machine type */
+    if (free_circ_machineinfos_with_machine_num(circ,
+                negotiate->machine_type)) {
+      log_info(LD_CIRC, "Received STOP command for machine %u",
+               negotiate->machine_type);
+      goto done;
+    }
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received circuit padding stop command for unknown machine.");
+    goto err;
+  } else if (negotiate->command == CIRCPAD_COMMAND_START) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            const circpad_machine_spec_t *, m) {
+      if (m->machine_num == negotiate->machine_type) {
+        circpad_setup_machine_on_circ(circ, m);
+        circpad_cell_event_nonpadding_received(circ);
+        goto done;
+      }
+    } SMARTLIST_FOREACH_END(m);
+  }
+
+  err:
+    retval = -1;
+
+  done:
+    circpad_padding_negotiated(circ, negotiate->machine_type,
+                   negotiate->command,
+                   (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR);
+    circpad_negotiate_free(negotiate);
+
+    return retval;
+}
+
+/**
+ * Parse and react to a padding_negotiated cell.
+ *
+ * This is called at the origin upon receipt of the middle's response
+ * to our choice of state machine.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell,
+                                  crypt_path_t *layer_hint)
+{
+  circpad_negotiated_t *negotiated;
+
+  if (!CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell unsupported at non-origin.");
+    return -1;
+  }
+
+  /* Verify this came from the expected hop */
+  if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell from wrong hop on circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATED cell on circuit %u; "
+          "dropping.", TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (negotiated->command == CIRCPAD_COMMAND_STOP) {
+    log_info(LD_CIRC,
+             "Received STOP command on PADDING_NEGOTIATED for circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    /* There may not be a padding_info here if we shut down the
+     * machine in circpad_shutdown_old_machines(). Or, if
+     * circpad_add_matching_matchines() added a new machine,
+     * there may be a padding_machine for a different machine num
+     * than this response. */
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+  } else if (negotiated->command == CIRCPAD_COMMAND_START &&
+             negotiated->response == CIRCPAD_RESPONSE_ERR) {
+    // This can happen due to consensus drift.. free the machines
+    // and be sad
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+    TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1;
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Middle node did not accept our padding request on circuit %u (%d)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier,
+           circ->purpose);
+  }
+
+  circpad_negotiated_free(negotiated);
+  return 0;
+}
+
+/** Free memory allocated by this machine spec. */
+STATIC void
+machine_spec_free_(circpad_machine_spec_t *m)
+{
+  if (!m) return;
+
+  tor_free(m->states);
+  tor_free(m);
+}
+
+/** Free all memory allocated by the circuitpadding subsystem. */
+void
+circpad_free_all(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(origin_padding_machines);
+  }
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/* Serialization */
+// TODO: Should we use keyword=value here? Are there helpers for that?
+#if 0
+static void
+circpad_state_serialize(const circpad_state_t *state,
+                        smartlist_t *chunks)
+{
+  smartlist_add_asprintf(chunks, " %u", state->histogram[0]);
+  for (int i = 1; i < state->histogram_len; i++) {
+    smartlist_add_asprintf(chunks, ",%u",
+                           state->histogram[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " 0x%x",
+                         state->transition_cancel_events);
+
+  for (int i = 0; i < CIRCPAD_NUM_STATES; i++) {
+    smartlist_add_asprintf(chunks, ",0x%x",
+                           state->transition_events[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " %u %u",
+                         state->use_rtt_estimate,
+                         state->token_removal);
+}
+
+char *
+circpad_machine_spec_to_string(const circpad_machine_spec_t *machine)
+{
+  smartlist_t *chunks = smartlist_new();
+  char *out;
+  (void)machine;
+
+  circpad_state_serialize(&machine->start, chunks);
+  circpad_state_serialize(&machine->gap, chunks);
+  circpad_state_serialize(&machine->burst, chunks);
+
+  out = smartlist_join_strings(chunks, "", 0, NULL);
+
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+  return out;
+}
+
+// XXX: Writeme
+const circpad_machine_spec_t *
+circpad_string_to_machine(const char *str)
+{
+  (void)str;
+  return NULL;
+}
+
+#endif /* 0 */
diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h
new file mode 100644
index 0000000000..e9eb32c618
--- /dev/null
+++ b/src/core/or/circuitpadding.h
@@ -0,0 +1,813 @@
+/*
+ * Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.h
+ * \brief Header file for circuitpadding.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_H
+#define TOR_CIRCUITPADDING_H
+
+#include "trunnel/circpad_negotiation.h"
+#include "lib/evloop/timers.h"
+
+struct circuit_t;
+struct origin_circuit_t;
+struct cell_t;
+
+/**
+ * Signed error return with the specific property that negative
+ * values mean error codes of various semantics, 0 means success,
+ * and positive values are unused.
+ *
+ * XXX: Tor uses this concept a lot but just calls it int. Should we move
+ * this somewhere centralized? Where?
+ */
+typedef int signed_error_t;
+
+/**
+ * These constants specify the types of events that can cause
+ * transitions between state machine states.
+ *
+ * Note that SENT and RECV are relative to this endpoint. For
+ * relays, SENT means packets destined towards the client and
+ * RECV means packets destined towards the relay. On the client,
+ * SENT means packets destined towards the relay, where as RECV
+ * means packets destined towards the client.
+ */
+typedef enum {
+  /* A non-padding cell was received. */
+  CIRCPAD_EVENT_NONPADDING_RECV = 0,
+  /* A non-padding cell was sent. */
+  CIRCPAD_EVENT_NONPADDING_SENT = 1,
+  /* A padding cell (RELAY_COMMAND_DROP) was sent. */
+  CIRCPAD_EVENT_PADDING_SENT = 2,
+  /* A padding cell was received. */
+  CIRCPAD_EVENT_PADDING_RECV = 3,
+  /* We tried to schedule padding but we ended up picking the infinity bin
+   * which means that padding was delayed infinitely */
+  CIRCPAD_EVENT_INFINITY = 4,
+  /* All histogram bins are empty (we are out of tokens) */
+  CIRCPAD_EVENT_BINS_EMPTY = 5,
+  /* This state has used up its cell count */
+  CIRCPAD_EVENT_LENGTH_COUNT = 6
+} circpad_event_t;
+#define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1)
+
+/** Boolean type that says if we decided to transition states or not */
+typedef enum {
+  CIRCPAD_STATE_UNCHANGED = 0,
+  CIRCPAD_STATE_CHANGED = 1
+} circpad_decision_t;
+
+/** The type for the things in histogram bins (aka tokens) */
+typedef uint32_t circpad_hist_token_t;
+
+/** The type for histogram indexes (needs to be negative for errors) */
+typedef int8_t circpad_hist_index_t;
+
+/** The type for absolute time, from monotime_absolute_usec() */
+typedef uint64_t circpad_time_t;
+
+/** The type for timer delays, in microseconds */
+typedef uint32_t circpad_delay_t;
+#define CIRCPAD_DELAY_UNITS_PER_SECOND  (1000*1000)
+
+/**
+ * An infinite padding cell delay means don't schedule any padding --
+ * simply wait until a different event triggers a transition.
+ *
+ * This means that the maximum delay we can schedule is UINT32_MAX-1
+ * microseconds, or about 4300 seconds (1.25 hours).
+ * XXX: Is this enough if we want to simulate light, intermittent
+ * activity on an onion service?
+ */
+#define CIRCPAD_DELAY_INFINITE  (UINT32_MAX)
+
+/**
+ * This is the maximum delay that the circuit padding system can have, in
+ * seconds.
+ */
+#define CIRCPAD_DELAY_MAX_SECS   \
+    ((CIRCPAD_DELAY_INFINITE/CIRCPAD_DELAY_UNITS_PER_SECOND)+1)
+
+/**
+ * Macro to clarify when we're checking the infinity bin.
+ *
+ * Works with either circpad_state_t or circpad_machine_runtime_t
+ */
+#define CIRCPAD_INFINITY_BIN(mi)  ((mi)->histogram_len-1)
+
+/**
+ * These constants form a bitfield that specifies when a state machine
+ * should be applied to a circuit.
+ *
+ * If any of these elements is set, then the circuit will be tested against
+ * that specific condition. If an element is unset, then we don't test it.
+ * (E.g., if neither NO_STREAMS or STREAMS are set, then we will not care
+ * whether a circuit has streams attached when we apply a state machine.)
+ *
+ * The helper function circpad_circuit_state() converts circuit state
+ * flags into this more compact representation.
+ */
+typedef enum {
+  /* Only apply machine if the circuit is still building */
+  CIRCPAD_CIRC_BUILDING = 1<<0,
+  /* Only apply machine if the circuit is open */
+  CIRCPAD_CIRC_OPENED = 1<<1,
+  /* Only apply machine if the circuit has no attached streams */
+  CIRCPAD_CIRC_NO_STREAMS = 1<<2,
+  /* Only apply machine if the circuit has attached streams */
+  CIRCPAD_CIRC_STREAMS = 1<<3,
+  /* Only apply machine if the circuit still allows RELAY_EARLY cells */
+  CIRCPAD_CIRC_HAS_RELAY_EARLY = 1<<4,
+  /* Only apply machine if the circuit has depleted its RELAY_EARLY cells
+   * allowance. */
+  CIRCPAD_CIRC_HAS_NO_RELAY_EARLY = 1<<5
+} circpad_circuit_state_t;
+
+/** Bitmask that says "apply this machine to all states" */
+#define CIRCPAD_STATE_ALL   \
+    (CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED| \
+     CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_NO_STREAMS| \
+     CIRCPAD_CIRC_HAS_RELAY_EARLY|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY)
+
+/**
+ * A compact circuit purpose bitfield mask that allows us to compactly
+ * specify which circuit purposes a machine should apply to.
+ *
+ * The helper function circpad_circ_purpose_to_mask() converts circuit
+ * purposes into bit positions in this bitmask.
+ */
+typedef uint32_t circpad_purpose_mask_t;
+
+/** Bitmask that says "apply this machine to all purposes". */
+#define CIRCPAD_PURPOSE_ALL (0xFFFFFFFF)
+
+/**
+ * This type specifies all of the conditions that must be met before
+ * a client decides to initiate padding on a circuit.
+ *
+ * A circuit must satisfy every sub-field in this type in order
+ * to be considered to match the conditions.
+ */
+typedef struct circpad_machine_conditions_t {
+  /** Only apply the machine *if* the circuit has at least this many hops */
+  unsigned min_hops : 3;
+
+  /** Only apply the machine *if* vanguards are enabled */
+  unsigned requires_vanguards : 1;
+
+  /**
+   * This machine is ok to use if reduced padding is set in consensus
+   * or torrc. This machine will still be applied even if reduced padding
+   * is not set; this flag only acts to exclude machines that don't have
+   * it set when reduced padding is requested. Therefore, reduced padding
+   * machines should appear at the lowest priority in the padding machine
+   * lists (aka first in the list), so that non-reduced padding machines
+   * for the same purpose are given a chance to apply when reduced padding
+   * is not requested. */
+  unsigned reduced_padding_ok : 1;
+
+  /** Only apply the machine *if* the circuit's state matches any of
+   *  the bits set in this bitmask. */
+  circpad_circuit_state_t state_mask;
+
+  /** Only apply a machine *if* the circuit's purpose matches one
+   *  of the bits set in this bitmask */
+  circpad_purpose_mask_t purpose_mask;
+
+} circpad_machine_conditions_t;
+
+/**
+ * Token removal strategy options.
+ *
+ * The WTF-PAD histograms are meant to specify a target distribution to shape
+ * traffic towards. This is accomplished by removing tokens from the histogram
+ * when either padding or non-padding cells are sent.
+ *
+ * When we see a non-padding cell at a particular time since the last cell, you
+ * remove a token from the corresponding delay bin. These flags specify
+ * which bin to choose if that bin is already empty.
+ */
+typedef enum {
+  /** Don't remove any tokens */
+  CIRCPAD_TOKEN_REMOVAL_NONE = 0,
+  /**
+   * Remove from the first non-zero higher bin index when current is zero.
+   * This is the recommended strategy from the Adaptive Padding paper. */
+  CIRCPAD_TOKEN_REMOVAL_HIGHER = 1,
+  /** Remove from the first non-zero lower bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_LOWER = 2,
+  /** Remove from the closest non-zero bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST = 3,
+  /** Remove from the closest bin by time value (since bins are
+   *  exponentially spaced). */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC = 4,
+  /** Only remove from the exact bin corresponding to this delay. If
+   *  the bin is 0, simply do nothing. Don't pick another bin. */
+  CIRCPAD_TOKEN_REMOVAL_EXACT = 5
+} circpad_removal_t;
+
+/**
+ * Distribution types supported by circpad_distribution_sample().
+ *
+ * These can be used instead of histograms for the inter-packet
+ * timing distribution, or to specify a distribution on the number
+ * of cells that can be sent while in a specific state of the state
+ * machine.
+ *
+ * Each distribution takes up to two parameters which are described below. */
+typedef enum {
+  /* No probability distribution is used */
+  CIRCPAD_DIST_NONE = 0,
+  /* Uniform distribution: param1 is lower bound and param2 is upper bound */
+  CIRCPAD_DIST_UNIFORM = 1,
+  /* Logistic distribution: param1 is Mu, param2 is sigma. */
+  CIRCPAD_DIST_LOGISTIC = 2,
+  /* Log-logistic distribution: param1 is Alpha, param2 is 1.0/Beta */
+  CIRCPAD_DIST_LOG_LOGISTIC = 3,
+  /* Geometric distribution: param1 is 'p' (success probability) */
+  CIRCPAD_DIST_GEOMETRIC = 4,
+  /* Weibull distribution: param1 is k, param2 is Lambda */
+  CIRCPAD_DIST_WEIBULL = 5,
+  /* Generalized Pareto distribution: param1 is sigma, param2 is xi */
+  CIRCPAD_DIST_PARETO = 6
+} circpad_distribution_type_t;
+
+/**
+ * Distribution information.
+ *
+ * This type specifies a specific distribution above, as well as
+ * up to two parameters for that distribution. The specific
+ * per-distribution meaning of these parameters is specified
+ * in circpad_distribution_sample().
+ */
+typedef struct circpad_distribution_t {
+  circpad_distribution_type_t type;
+  double param1;
+  double param2;
+} circpad_distribution_t;
+
+/** State number type. Represents current state of state machine. */
+typedef uint16_t circpad_statenum_t;
+#define  CIRCPAD_STATENUM_MAX   (UINT16_MAX)
+
+/** A histogram can be used to sample padding delays given a machine state.
+ * This constant defines the maximum histogram width (i.e. the max number of
+ * bins).
+ *
+ * The current limit is arbitrary and could be raised if there is a need,
+ * however too many bins will be hard to serialize in the future.
+ *
+ * Memory concerns are not so great here since the corresponding histogram and
+ * histogram_edges arrays are global and not per-circuit.
+ *
+ * If we ever upgrade this to a value that can't be represented by 8-bits we
+ * also need to upgrade circpad_hist_index_t.
+ */
+#define CIRCPAD_MAX_HISTOGRAM_LEN (100)
+
+/**
+ * A state of a padding state machine. The information here are immutable and
+ * represent the initial form of the state; it does not get updated as things
+ * happen. The mutable information that gets updated in runtime are carried in
+ * a circpad_machine_runtime_t.
+ *
+ * This struct describes the histograms and/or probability distributions, as
+ * well as parameters of a single state in the adaptive padding machine.
+ * Instances of this struct exist in global circpad machine definitions that
+ * come from torrc or the consensus.
+ */
+typedef struct circpad_state_t {
+  /**
+   * If a histogram is used for this state, this specifies the number of bins
+   * of this histogram. Histograms must have at least 2 bins.
+   *
+   * In particular, the following histogram:
+   *
+   * Tokens
+   *         +
+   *      10 |    +----+
+   *       9 |    |    |           +---------+
+   *       8 |    |    |           |         |
+   *       7 |    |    |     +-----+         |
+   *       6 +----+ Bin+-----+     |         +---------------+
+   *       5 |    | #1 |     |     |         |               |
+   *         | Bin|    | Bin | Bin |  Bin #4 |    Bin #5     |
+   *         | #0 |    | #2  | #3  |         | (infinity bin)|
+   *         |    |    |     |     |         |               |
+   *         |    |    |     |     |         |               |
+   *       0 +----+----+-----+-----+---------+---------------+
+   *         0   100  200   350   500      1000              ∞  microseconds
+   *
+   * would be specified the following way:
+   *    histogram_len = 6;
+   *    histogram[] =        {   6,  10,   6,  7,    9,     6 }
+   *    histogram_edges[] =  { 0, 100, 200, 350, 500, 1000 }
+   *
+   * The final bin is called the "infinity bin" and if it's chosen we don't
+   * schedule any padding. The infinity bin is strange because its lower edge
+   * is the max value of possible non-infinite delay allowed by this histogram,
+   * and its upper edge is CIRCPAD_DELAY_INFINITE. You can tell if the infinity
+   * bin is chosen by inspecting its bin index or inspecting its upper edge.
+   *
+   * If a delay probability distribution is used for this state, this is set
+   * to 0. */
+  circpad_hist_index_t histogram_len;
+  /** The histogram itself: an array of uint16s of tokens, whose
+   *  widths are exponentially spaced, in microseconds.
+   *
+   *  This array must have histogram_len elements that are strictly
+   *  monotonically increasing. */
+  circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN];
+  /* The histogram bin edges in usec.
+   *
+   * Each element of this array specifies the left edge of the corresponding
+   * bin. The rightmost edge is always infinity and is not specified in this
+   * array.
+   *
+   * This array must have histogram_len elements. */
+  circpad_delay_t histogram_edges[CIRCPAD_MAX_HISTOGRAM_LEN+1];
+  /** Total number of tokens in this histogram. This is a constant and is *not*
+   *  decremented every time we spend a token. It's used for initializing and
+   *  refilling the histogram. */
+  uint32_t histogram_total_tokens;
+
+  /**
+   * Represents a delay probability distribution (aka IAT distribution). It's a
+   * parametrized way of encoding inter-packet delay information in
+   * microseconds. It can be used instead of histograms.
+   *
+   * If it is used, token_removal below must be set to
+   * CIRCPAD_TOKEN_REMOVAL_NONE.
+   *
+   * Start_usec, range_sec, and rtt_estimates are still applied to the
+   * results of sampling from this distribution (range_sec is used as a max).
+   */
+  circpad_distribution_t iat_dist;
+  /*  If a delay probability distribution is used, this is used as the max
+   *  value we can sample from the distribution. However, RTT measurements and
+   *  dist_added_shift gets applied on top of this value to derive the final
+   *  padding delay. */
+  circpad_delay_t dist_max_sample_usec;
+  /*  If a delay probability distribution is used and this is set, we will add
+   *  this value on top of the value sampled from the IAT distribution to
+   *  derive the final padding delay (We also add the RTT measurement if it's
+   *  enabled.). */
+  circpad_delay_t dist_added_shift_usec;
+
+  /**
+   * The length dist is a parameterized way of encoding how long this
+   * state machine runs in terms of sent padding cells or all
+   * sent cells. Values are sampled from this distribution, clamped
+   * to max_len, and then start_len is added to that value.
+   *
+   * It may be specified instead of or in addition to
+   * the infinity bins and bins empty conditions. */
+  circpad_distribution_t length_dist;
+  /** A minimum length value, added to the output of length_dist */
+  uint16_t start_length;
+  /** A cap on the length value that can be sampled from the length_dist */
+  uint64_t max_length;
+
+  /** Should we decrement length when we see a nonpadding packet?
+   * XXX: Are there any machines that actually want to set this to 0? There may
+   * not be. OTOH, it's only a bit.. */
+  unsigned length_includes_nonpadding : 1;
+
+  /**
+   * This is an array that specifies the next state to transition to upon
+   * receipt an event matching the indicated array index.
+   *
+   * This aborts our scheduled packet and switches to the state
+   * corresponding to the index of the array. Tokens are filled upon
+   * this transition.
+   *
+   * States are allowed to transition to themselves, which means re-schedule
+   * a new padding timer. They are also allowed to temporarily "transition"
+   * to the "IGNORE" and "CANCEL" pseudo-states. See #defines below
+   * for details on state behavior and meaning.
+   */
+  circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS];
+
+  /**
+   * If true, estimate the RTT from this relay to the exit/website and add that
+   * to start_usec for use as the histogram bin 0 start delay.
+   *
+   * Right now this is only supported for relay-side state machines.
+   */
+  unsigned use_rtt_estimate : 1;
+
+  /** This specifies the token removal strategy to use upon padding and
+   *  non-padding activity. */
+  circpad_removal_t token_removal;
+} circpad_state_t;
+
+/**
+ * The start state for this machine.
+ *
+ * In the original WTF-PAD, this is only used for transition to/from
+ * the burst state. All other fields are not used. But to simplify the
+ * code we've made it a first-class state. This has no performance
+ * consequences, but may make naive serialization of the state machine
+ * large, if we're not careful about how we represent empty fields.
+ */
+#define  CIRCPAD_STATE_START       0
+
+/**
+ * The burst state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD
+ * (https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf and
+ * https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/), the burst
+ * state serves to detect bursts in traffic. This is done by using longer
+ * delays in its histogram, which represent the expected delays between
+ * bursts of packets in the target stream. If this delay expires without a
+ * real packet being sent, the burst state sends a padding packet and then
+ * immediately transitions to the gap state, which is used to generate
+ * a synthetic padding packet train. In this implementation, this transition
+ * needs to be explicitly specified in the burst state's transition events.
+ *
+ * Because of this flexibility, other padding mechanisms can transition
+ * between these two states arbitrarily, to encode other dynamics of
+ * target traffic.
+ */
+#define  CIRCPAD_STATE_BURST       1
+
+/**
+ * The gap state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD, the gap
+ * state serves to simulate an artificial packet train composed of padding
+ * packets. It does this by specifying much lower inter-packet delays than
+ * the burst state, and transitioning back to itself after padding is sent
+ * if these timers expire before real traffic is sent. If real traffic is
+ * sent, it transitions back to the burst state.
+ *
+ * Again, in this implementation, these transitions must be specified
+ * explicitly, and other transitions are also permitted.
+ */
+#define  CIRCPAD_STATE_GAP         2
+
+/**
+ * End is a pseudo-state that causes the machine to go completely
+ * idle, and optionally get torn down (depending on the
+ * value of circpad_machine_spec_t.should_negotiate_end)
+ *
+ * End MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_END         CIRCPAD_STATENUM_MAX
+
+/**
+ * "Ignore" is a pseudo-state that means "do not react to this
+ * event".
+ *
+ * "Ignore" MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_IGNORE         (CIRCPAD_STATENUM_MAX-1)
+
+/**
+ * "Cancel" is a pseudo-state that means "cancel pending timers,
+ * but remain in your current state".
+ *
+ * Cancel MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_CANCEL         (CIRCPAD_STATENUM_MAX-2)
+
+/**
+ * Since we have 3 pseudo-states, the max state array length is
+ * up to one less than cancel's statenum.
+ */
+#define CIRCPAD_MAX_MACHINE_STATES  (CIRCPAD_STATE_CANCEL-1)
+
+/**
+ * Mutable padding machine info.
+ *
+ * This structure contains mutable information about a padding
+ * machine. The mutable information must be kept separate because
+ * it exists per-circuit, where as the machines themselves are global.
+ * This separation is done to conserve space in the circuit structure.
+ *
+ * This is the per-circuit state that changes regarding the global state
+ * machine. Some parts of it are optional (ie NULL).
+ *
+ * XXX: Play with layout to minimize space on x64 Linux (most common relay).
+ */
+typedef struct circpad_machine_runtime_t {
+  /** The callback pointer for the padding callbacks.
+   *
+   *  These timers stick around the machineinfo until the machineinfo's circuit
+   *  is closed, at which point the timer is cancelled. For this reason it's
+   *  safe to assume that the machineinfo exists if this timer gets
+   *  triggered. */
+  tor_timer_t *padding_timer;
+
+  /** The circuit for this machine */
+  struct circuit_t *on_circ;
+
+  /** A mutable copy of the histogram for the current state.
+   *  NULL if remove_tokens is false for that state */
+  circpad_hist_token_t *histogram;
+  /** Length of the above histogram.
+   * XXX: This field *could* be removed at the expense of added
+   * complexity+overhead for reaching back into the immutable machine
+   * state every time we need to inspect the histogram. It's only a byte,
+   * though, so it seemed worth it.
+   */
+  circpad_hist_index_t histogram_len;
+  /** Remove token from this index upon sending padding */
+  circpad_hist_index_t chosen_bin;
+
+  /** Stop padding/transition if this many cells sent */
+  uint64_t state_length;
+#define CIRCPAD_STATE_LENGTH_INFINITE UINT64_MAX
+
+  /** A scaled count of padding packets sent, used to limit padding overhead.
+   * When this reaches UINT16_MAX, we cut it and nonpadding_sent in half. */
+  uint16_t padding_sent;
+  /** A scaled count of non-padding packets sent, used to limit padding
+   *  overhead. When this reaches UINT16_MAX, we cut it and padding_sent in
+   *  half. */
+  uint16_t nonpadding_sent;
+
+  /**
+   * Timestamp of the most recent cell event (sent, received, padding,
+   * non-padding), in seconds from approx_time().
+   *
+   * Used as an emergency break to stop holding padding circuits open.
+   */
+  time_t last_cell_time_sec;
+
+  /**
+   * EWMA estimate of the RTT of the circuit from this hop
+   * to the exit end, in microseconds. */
+  circpad_delay_t rtt_estimate_usec;
+
+  /**
+   * The last time we got an event relevant to estimating
+   * the RTT. Monotonic time in microseconds since system
+   * start.
+   */
+  circpad_time_t last_received_time_usec;
+
+  /**
+   * The time at which we scheduled a non-padding packet,
+   * or selected an infinite delay.
+   *
+   * Monotonic time in microseconds since system start.
+   * This is 0 if we haven't chosen a padding delay.
+   */
+  circpad_time_t padding_scheduled_at_usec;
+
+  /** What state is this machine in? */
+  circpad_statenum_t current_state;
+
+  /**
+   * True if we have scheduled a timer for padding.
+   *
+   * This is 1 if a timer is pending. It is 0 if
+   * no timer is scheduled. (It can be 0 even when
+   * padding_was_scheduled_at_usec is non-zero).
+   */
+  unsigned is_padding_timer_scheduled : 1;
+
+  /**
+   * If this is true, we have seen full duplex behavior.
+   * Stop updating the RTT.
+   */
+  unsigned stop_rtt_update : 1;
+
+/** Max number of padding machines on each circuit. If changed,
+ * also ensure the machine_index bitwith supports the new size. */
+#define CIRCPAD_MAX_MACHINES    (2)
+  /** Which padding machine index was this for.
+   * (make sure changes to the bitwidth can support the
+   * CIRCPAD_MAX_MACHINES define). */
+  unsigned machine_index : 1;
+
+} circpad_machine_runtime_t;
+
+/** Helper macro to get an actual state machine from a machineinfo */
+#define CIRCPAD_GET_MACHINE(machineinfo) \
+    ((machineinfo)->on_circ->padding_machine[(machineinfo)->machine_index])
+
+/**
+ * This specifies a particular padding machine to use after negotiation.
+ *
+ * The constants for machine_num_t are in trunnel.
+ * We want to be able to define extra numbers in the consensus/torrc, though.
+ */
+typedef uint8_t circpad_machine_num_t;
+
+/** Global state machine structure from the consensus */
+typedef struct circpad_machine_spec_t {
+  /* Just a user-friendly machine name for logs */
+  const char *name;
+
+  /** Global machine number */
+  circpad_machine_num_t machine_num;
+
+  /** Which machine index slot should this machine go into in
+   *  the array on the circuit_t */
+  unsigned machine_index : 1;
+
+  /** Send a padding negotiate to shut down machine at end state? */
+  unsigned should_negotiate_end : 1;
+
+  // These next three fields are origin machine-only...
+  /** Origin side or relay side */
+  unsigned is_origin_side : 1;
+
+  /** Which hop in the circuit should we send padding to/from?
+   *  1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */
+  unsigned target_hopnum : 3;
+
+  /** If this flag is enabled, don't close circuits that use this machine even
+   *  if another part of Tor wants to close this circuit.
+   *
+   *  If this flag is set, the circuitpadding subsystem will close circuits the
+   *  moment the machine transitions to the END state, and only if the circuit
+   *  has already been asked to be closed by another part of Tor.
+   *
+   *  Circuits that should have been closed but were kept open by a padding
+   *  machine are re-purposed to CIRCUIT_PURPOSE_C_CIRCUIT_PADDING, hence
+   *  machines should take that purpose into account if they are filtering
+   *  circuits by purpose. */
+  unsigned manage_circ_lifetime : 1;
+
+  /** This machine only kills fascists if the following conditions are met. */
+  circpad_machine_conditions_t conditions;
+
+  /** How many padding cells can be sent before we apply overhead limits?
+   * XXX: Note that we can only allow up to 64k of padding cells on an
+   * otherwise quiet circuit. Is this enough? It's 33MB. */
+  uint16_t allowed_padding_count;
+
+  /** Padding percent cap: Stop padding if we exceed this percent overhead.
+   * 0 means no limit. Overhead is defined as percent of total traffic, so
+   * that we can use 0..100 here. This is the same definition as used in
+   * Prop#265. */
+  uint8_t max_padding_percent;
+
+  /** State array: indexed by circpad_statenum_t */
+  circpad_state_t *states;
+
+  /**
+   * Number of states this machine has (ie: length of the states array).
+   * XXX: This field is not needed other than for safety. */
+  circpad_statenum_t num_states;
+} circpad_machine_spec_t;
+
+void circpad_new_consensus_params(const networkstatus_t *ns);
+
+int circpad_marked_circuit_for_padding(circuit_t *circ, int reason);
+
+/**
+ * The following are event call-in points that are of interest to
+ * the state machines. They are called during cell processing. */
+void circpad_deliver_unrecognized_cell_events(struct circuit_t *circ,
+                                              cell_direction_t dir);
+void circpad_deliver_sent_relay_cell_events(struct circuit_t *circ,
+                                            uint8_t relay_command);
+void circpad_deliver_recognized_relay_cell_events(struct circuit_t *circ,
+                                                  uint8_t relay_command,
+                                                  crypt_path_t *layer_hint);
+
+/** Cell events are delivered by the above delivery functions */
+void circpad_cell_event_nonpadding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_nonpadding_received(struct circuit_t *on_circ);
+void circpad_cell_event_padding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_padding_received(struct circuit_t *on_circ);
+
+/** Internal events are events the machines send to themselves */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi);
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *);
+circpad_decision_t circpad_internal_event_state_length_up(
+                                  circpad_machine_runtime_t *);
+
+/** Machine creation events are events that cause us to set up or
+ *  tear down padding state machines. */
+void circpad_machine_event_circ_added_hop(struct origin_circuit_t *on_circ);
+void circpad_machine_event_circ_built(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_purpose_changed(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_streams(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_no_streams(struct origin_circuit_t *circ);
+void
+circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ);
+
+void circpad_machines_init(void);
+void circpad_machines_free(void);
+void circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                      smartlist_t *machine_list);
+
+void circpad_machine_states_init(circpad_machine_spec_t *machine,
+                                 circpad_statenum_t num_states);
+
+void circpad_circuit_free_all_machineinfos(struct circuit_t *circ);
+
+bool circpad_padding_is_from_expected_hop(struct circuit_t *circ,
+                                         crypt_path_t *from_hop);
+
+/** Serializaton functions for writing to/from torrc and consensus */
+char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine);
+const circpad_machine_spec_t *circpad_string_to_machine(const char *str);
+
+/* Padding negotiation between client and middle */
+signed_error_t circpad_handle_padding_negotiate(struct circuit_t *circ,
+                                      struct cell_t *cell);
+signed_error_t circpad_handle_padding_negotiated(struct circuit_t *circ,
+                                      struct cell_t *cell,
+                                      crypt_path_t *layer_hint);
+signed_error_t circpad_negotiate_padding(struct origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command);
+bool circpad_padding_negotiated(struct circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response);
+
+circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose);
+
+int circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                                crypt_path_t *layer_hint,
+                                const relay_header_t *rh);
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *));
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_spec_transition, (circpad_machine_runtime_t *mi,
+                             circpad_event_t event));
+
+circpad_decision_t circpad_send_padding_cell_for_callback(
+                                 circpad_machine_runtime_t *mi);
+
+void circpad_free_all(void);
+
+#ifdef CIRCUITPADDING_PRIVATE
+STATIC void  machine_spec_free_(circpad_machine_spec_t *m);
+#define machine_spec_free(chan) \
+  FREE_AND_NULL(circpad_machine_spec_t,machine_spec_free_, (m))
+
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi);
+
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi);
+
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi);
+
+STATIC void circpad_machine_remove_token(circpad_machine_runtime_t *mi);
+
+STATIC circpad_hist_index_t circpad_histogram_usec_to_bin(
+                                       const circpad_machine_runtime_t *mi,
+                                       circpad_delay_t us);
+
+STATIC circpad_machine_runtime_t *circpad_circuit_machineinfo_new(
+                                               struct circuit_t *on_circ,
+                                               int machine_index);
+STATIC void circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us,
+                                         bool use_usec);
+STATIC void circpad_machine_setup_tokens(circpad_machine_runtime_t *mi);
+
+MOCK_DECL(STATIC signed_error_t,
+circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len));
+
+MOCK_DECL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop));
+
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl);
+
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *origin_padding_machines;
+extern smartlist_t *relay_padding_machines;
+
+#endif
+
+#endif /* defined(CIRCUITPADDING_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_H) */
diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c
new file mode 100644
index 0000000000..7220d657fc
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.c
@@ -0,0 +1,456 @@
+/* Copyright (c) 2019 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.c
+ * \brief Circuit padding state machines
+ *
+ * \detail
+ *
+ * Introduce circuit padding machines that will be used by Tor circuits, as
+ * specified by proposal 302 "Hiding onion service clients using padding".
+ *
+ * Right now this file introduces two machines that aim to hide the client-side
+ * of onion service circuits against naive classifiers like the ones from the
+ * "Circuit Fingerprinting Attacks: Passive Deanonymization of Tor Hidden
+ * Services" paper from USENIX. By naive classifiers we mean classifiers that
+ * use basic features like "circuit construction circuits" and "incoming and
+ * outgoing cell counts" and "duration of activity".
+ *
+ * In particular, these machines aim to be lightweight and protect against
+ * these basic classifiers. They don't aim to protect against more advanced
+ * attacks that use deep learning or even correlate various circuit
+ * construction events together. Machines that fool such advanced classifiers
+ * are also possible, but they can't be so lightweight and might require more
+ * WTF-PAD features. So for now we opt for the following two machines:
+ *
+ * Client-side introduction circuit hiding machine:
+ *
+ *    This machine hides client-side introduction circuits by making their
+ *    circuit consruction sequence look like normal general circuits that
+ *    download directory information. Furthermore, the circuits are kept open
+ *    until all the padding has been sent, since intro circuits are usually
+ *    very short lived and this act as a distinguisher. For more info see
+ *    circpad_machine_client_hide_intro_circuits() and the sec.
+ *
+ * Client-side rendezvous circuit hiding machine:
+ *
+ *    This machine hides client-side rendezvous circuits by making their
+ *    circuit construction sequence look like normal general circuits. For more
+ *    details see circpad_machine_client_hide_rend_circuits() and the spec.
+ *
+ * TODO: These are simple machines that carefully manipulate the cells of the
+ *   initial circuit setup procedure to make them look like general
+ *   circuits. In the future, more states can be baked into their state machine
+ *   to do more advanced obfuscation.
+ **/
+
+#define CIRCUITPADDING_MACHINES_PRIVATE
+
+#include "core/or/or.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/circuitlist.h"
+
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitpadding.h"
+
+/** Create a client-side padding machine that aims to hide IP circuits. In
+ *  particular, it keeps intro circuits alive until a bunch of fake traffic has
+ *  been pushed through.
+ */
+void
+circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_ip_circ";
+
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad introduction circuits, and we want to start padding
+   * only after the INTRODUCE1 cell has been sent, so set the purposes
+   * appropriately.
+   *
+   * In particular we want introduction circuits to blend as much as possible
+   * with general circuits. Most general circuits have the following initial
+   * relay cell sequence (outgoing cells marked in [brackets]):
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * Whereas normal introduction circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2
+   *   -> [INTRO1] -> INTRODUCE_ACK
+   *
+   * This means that up to the sixth cell (first line of each sequence above),
+   * both general and intro circuits have identical cell sequences. After that
+   * we want to mimic the second line sequence of
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * We achieve this by starting padding INTRODUCE1 has been sent. With padding
+   * negotiation cells, in the common case of the second line looks like:
+   *   -> [INTRO1] -> [PADDING_NEGOTIATE] -> PADDING_NEGOTIATED -> INTRO_ACK
+   *
+   * Then, the middle node will send between INTRO_MACHINE_MINIMUM_PADDING and
+   * INTRO_MACHINE_MAXIMUM_PADDING cells, to match the "...(inbound data cells
+   * continue)" portion of the trace (aka the rest of an HTTPS response body).
+   */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Keep the circuit alive even after the introduction has been finished,
+   * otherwise the short-term lifetime of the circuit will blow our cover */
+  client_machine->manage_circ_lifetime = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* For the origin-side machine, we transition to OBFUSCATE_CIRC_SETUP after
+   * sending PADDING_NEGOTIATE, and we stay there (without sending any padding)
+   * until we receive a STOP from the other side. */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* origin-side machine has no event reactions while in
+   * CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP, so no more state transitions here. */
+
+  /* The client side should never send padding, so it does not need
+   * to specify token removal, or a histogram definition or state lengths.
+   * That is all controlled by the middle node. */
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client intro point hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits. See
+ *  comments on the function above for more details on the workings of the
+ *  machine. */
+void
+circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_ip_circ";
+
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* We want to negotiate END from this side after all our padding is done, so
+   * that the origin-side machine goes into END state, and eventually closes
+   * the circuit. */
+  relay_machine->should_negotiate_end = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* For the relay-side machine, we want to transition
+   * START -> OBFUSCATE_CIRC_SETUP upon first non-padding
+   * cell sent (PADDING_NEGOTIATED in this case).  */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* For the relay-side, we want to transition from OBFUSCATE_CIRC_SETUP to END
+   * state when the length finishes. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Now let's define the OBF -> OBF transitions that maintain our padding
+   * flow:
+   *
+   * For the relay-side machine, we want to keep on sending padding bytes even
+   * when nothing else happens on this circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_PADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+  /* For the relay-side machine, we need this transition so that we re-enter
+     the state, after PADDING_NEGOTIATED is sent. Otherwise, the remove token
+     function will disable the timer, and nothing will restart it since there
+     is no other motion on an intro circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* Token removal strategy for OBFUSCATE_CIRC_SETUP state: Don't
+   * remove any tokens.
+   *
+   * We rely on the state length sampling and not token removal, to avoid
+   * the mallocs required to copy the histograms for token removal,
+   * and to avoid monotime calls needed to determine histogram
+   * bins for token removal. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Figure out the length of the OBFUSCATE_CIRC_SETUP state so that it's
+   * randomized. The relay side will send between INTRO_MACHINE_MINIMUM_PADDING
+   * and INTRO_MACHINE_MAXIMUM_PADDING padding cells towards the client. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = INTRO_MACHINE_MINIMUM_PADDING;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = INTRO_MACHINE_MAXIMUM_PADDING;
+
+  /* Configure histogram */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+     histogram_len = 2;
+
+  /* For the relay-side machine we want to batch padding instantly to pretend
+   * its an incoming directory download. So set the histogram edges tight:
+   * (1, 10ms, infinity). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 1000;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 10000;
+
+  /* We put all our tokens in bin 0, which means we want 100% probability
+   * for choosing a inter-packet delay of between 1000 and 10000 microseconds
+   * (1 to 10ms). Since we only have 1 bin, it doesn't matter how many tokens
+   * there are, 1000 out of 1000 is 100% */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1000;
+
+  /* just one bin, so setup the total tokens */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens =
+      relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].histogram[0];
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay intro circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
+
+/************************** Rendezvous-circuit machine ***********************/
+
+/** Create a client-side padding machine that aims to hide rendezvous
+ *  circuits.*/
+void
+circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad rendezvous circuits, and we want to start padding only
+   * after the rendezvous circuit has been established.
+   *
+   * Following a similar argument as for intro circuits, we are aiming for
+   * padded rendezvous circuits to blend in with the initial cell sequence of
+   * general circuits which usually look like this:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *    -> [DATA] -> [DATA] -> DATA -> DATA...(incoming cells continue)
+   *
+   * Whereas normal rendezvous circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> REND2 -> [BEGIN]
+   *
+   * This means that up to the sixth cell (in the first line), both general and
+   * rend circuits have identical cell sequences.
+   *
+   * After that we want to mimic a [DATA] -> [DATA] -> DATA -> DATA sequence.
+   *
+   * With padding negotiation right after the REND_ESTABLISHED, the sequence
+   * becomes:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP...
+   *
+   * After which normal application DATA cells continue on the circuit.
+   *
+   * Hence this way we make rendezvous circuits look like general circuits up
+   * till the end of the circuit setup. */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_JOINED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = 1;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATE) */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that we send our outgoing [DROP] before the PADDING_NEGOTIATED comes
+   * back from the relay side. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client rendezvous circuit hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits.
+ *
+ *  This is meant to follow the client-side machine.
+ */
+void
+circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+    = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  relay_machine->conditions.min_hops = 2;
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = 1;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATED) */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_END;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that the outgoing DROP cell is sent immediately after the
+   * PADDING_NEGOTIATED. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay rendezvous circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
diff --git a/src/core/or/circuitpadding_machines.h b/src/core/or/circuitpadding_machines.h
new file mode 100644
index 0000000000..3c9798d42d
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.h
+ * \brief Header file for circuitpadding_machines.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_MACHINES_H
+#define TOR_CIRCUITPADDING_MACHINES_H
+
+void circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl);
+
+#ifdef CIRCUITPADDING_MACHINES_PRIVATE
+
+/** State of the padding machines that actually sends padding */
+#define CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP CIRCPAD_STATE_BURST
+
+/** Constants defining the amount of padding that a machine will send to hide
+ *  HS circuits. The actual value is sampled uniformly random between the
+ *  min/max values.
+ */
+
+/** Minimum number of relay-side padding cells to be sent by this machine */
+#define INTRO_MACHINE_MINIMUM_PADDING 7
+/** Maximum number of relay-side padding cells to be sent by this machine.
+ *  The actual value will be sampled between the min and max.*/
+#define INTRO_MACHINE_MAXIMUM_PADDING 10
+
+#endif /* defined(CIRCUITPADDING_MACHINES_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_MACHINES_H) */
diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c
index 2cde21fa1f..7a7f3ca600 100644
--- a/src/core/or/circuitstats.c
+++ b/src/core/or/circuitstats.c
@@ -29,8 +29,8 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
-#include "feature/control/control.h"
+#include "lib/confmgt/confparse.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/networkstatus.h"
@@ -44,6 +44,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/relay/relay_periodic.h"
 
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -639,9 +640,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n)
 void
 circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ,
-                               CIRC_EVENT_FAILED,
-                               END_CIRC_REASON_TIMEOUT);
+  circuit_event_status(circ,
+                       CIRC_EVENT_FAILED,
+                       END_CIRC_REASON_TIMEOUT);
   circuit_change_purpose(TO_CIRCUIT(circ),
                          CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
   /* Record this event to check for too many timeouts
@@ -1420,6 +1421,7 @@ void
 circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 {
   time_t now = approx_time();
+  // XXXX this should use pubsub
   if (cbt->liveness.nonlive_timeouts > 0) {
     time_t time_since_live = now - cbt->liveness.network_last_live;
     log_notice(LD_CIRC,
diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c
index 7273be3510..284e315f9c 100644
--- a/src/core/or/circuituse.c
+++ b/src/core/or/circuituse.c
@@ -35,13 +35,14 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_client.h"
@@ -69,7 +70,7 @@
 #include "core/or/origin_circuit_st.h"
 #include "core/or/socks_request_st.h"
 
-static void circuit_expire_old_circuits_clientside(void);
+STATIC void circuit_expire_old_circuits_clientside(void);
 static void circuit_increment_failure_count(void);
 
 /** Check whether the hidden service destination of the stream at
@@ -177,7 +178,6 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
       purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
       purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
     tor_addr_t addr;
-    const int family = tor_addr_parse(&addr, conn->socks_request->address);
     if (!exitnode && !build_state->onehop_tunnel) {
       log_debug(LD_CIRC,"Not considering circuit with unknown router.");
       return 0; /* this circuit is screwed and doesn't know it yet,
@@ -198,6 +198,8 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
           return 0; /* this is a circuit to somewhere else */
         if (tor_digest_is_zero(digest)) {
           /* we don't know the digest; have to compare addr:port */
+          const int family = tor_addr_parse(&addr,
+                                            conn->socks_request->address);
           if (family < 0 ||
               !tor_addr_eq(&build_state->chosen_exit->addr, &addr) ||
               build_state->chosen_exit->port != conn->socks_request->port)
@@ -210,12 +212,14 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
         return 0;
       }
     }
-    if (origin_circ->prepend_policy && family != -1) {
-      int r = compare_tor_addr_to_addr_policy(&addr,
-                                              conn->socks_request->port,
-                                              origin_circ->prepend_policy);
-      if (r == ADDR_POLICY_REJECTED)
-        return 0;
+    if (origin_circ->prepend_policy) {
+      if (tor_addr_parse(&addr, conn->socks_request->address) != -1) {
+        int r = compare_tor_addr_to_addr_policy(&addr,
+                                                conn->socks_request->port,
+                                                origin_circ->prepend_policy);
+        if (r == ADDR_POLICY_REJECTED)
+          return 0;
+      }
     }
     if (exitnode && !connection_ap_can_use_exit(conn, exitnode)) {
       /* can't exit from this router */
@@ -1425,6 +1429,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
       if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
         hs_dec_rdv_stream_counter(origin_circ);
       }
+
+      /* If there are no more streams on this circ, tell circpad */
+      if (!origin_circ->p_streams)
+        circpad_machine_event_circ_has_no_streams(origin_circ);
+
       return;
     }
   } else {
@@ -1465,7 +1474,7 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
 /** Find each circuit that has been unused for too long, or dirty
  * for too long and has no streams on it: mark it for close.
  */
-static void
+STATIC void
 circuit_expire_old_circuits_clientside(void)
 {
   struct timeval cutoff, now;
@@ -1505,6 +1514,7 @@ circuit_expire_old_circuits_clientside(void)
                 circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT ||
                 circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
                 circ->purpose == CIRCUIT_PURPOSE_TESTING ||
+                circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING ||
                 (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
                 circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) ||
                 circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
@@ -1674,7 +1684,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop)
 void
 circuit_has_opened(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0);
+  circuit_event_status(circ, CIRC_EVENT_BUILT, 0);
 
   /* Remember that this circuit has finished building. Now if we start
    * it building again later (e.g. by extending it), we will know not
@@ -2523,8 +2533,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
           circ->rend_data = rend_data_dup(edge_conn->rend_data);
         } else if (edge_conn->hs_ident) {
           circ->hs_ident =
-            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk,
-                                 HS_IDENT_CIRCUIT_INTRO);
+            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk);
         }
         if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
             circ->base_.state == CIRCUIT_STATE_OPEN)
@@ -2596,6 +2605,12 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   /* add it into the linked list of streams on this circuit */
   log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.",
             (unsigned)circ->base_.n_circ_id);
+
+  /* If this is the first stream on this circuit, tell circpad
+   * that streams are attached */
+  if (!circ->p_streams)
+    circpad_machine_event_circ_has_streams(circ);
+
   /* reset it, so we can measure circ timeouts */
   ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL);
   ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams;
@@ -3080,6 +3095,8 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose)
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ),
                                           old_purpose);
+
+    circpad_machine_event_circ_purpose_changed(TO_ORIGIN_CIRCUIT(circ));
   }
 }
 
@@ -3113,7 +3130,9 @@ circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
 {
   if (!circ) return;
 
-  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+  tor_assertf_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE,
+                       "Wrong relay_body_len: %d (should be at most %d)",
+                       relay_body_len, RELAY_PAYLOAD_SIZE);
 
   circ->n_delivered_written_circ_bw =
       tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len);
diff --git a/src/core/or/command.c b/src/core/or/command.c
index 5fb6640c22..1c97437769 100644
--- a/src/core/or/command.c
+++ b/src/core/or/command.c
@@ -49,7 +49,7 @@
 #include "core/or/dos.h"
 #include "core/or/onion.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
@@ -182,7 +182,7 @@ command_process_cell(channel_t *chan, cell_t *cell)
     command_time_process_cell(cl, cn, & tp ## time ,            \
                               command_process_ ## tp ## _cell);  \
   } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
+#else /* !defined(KEEP_TIMING_STATS) */
 #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn)
 #endif /* defined(KEEP_TIMING_STATS) */
 
diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c
index 67a772be08..4f5f038e6c 100644
--- a/src/core/or/connection_edge.c
+++ b/src/core/or/connection_edge.c
@@ -62,21 +62,24 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/sendme.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/dnsserv.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
@@ -97,7 +100,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_util.h"
 
 #include "core/or/cell_st.h"
@@ -300,6 +303,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial)
       }
       return 0;
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
       if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) {
         /* (We already sent an end cell if possible) */
@@ -754,8 +762,13 @@ connection_edge_flushed_some(edge_connection_t *conn)
 {
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       break;
   }
   return 0;
@@ -779,7 +792,7 @@ connection_edge_finished_flushing(edge_connection_t *conn)
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       return 0;
     case AP_CONN_STATE_SOCKS_WAIT:
     case AP_CONN_STATE_NATD_WAIT:
@@ -1211,7 +1224,7 @@ connection_ap_rescan_and_attach_pending(void)
     entry_conn->marked_pending_circ_line = 0;   \
     entry_conn->marked_pending_circ_file = 0;   \
   } while (0)
-#else /* !(defined(DEBUGGING_17659)) */
+#else /* !defined(DEBUGGING_17659) */
 #define UNMARK() do { } while (0)
 #endif /* defined(DEBUGGING_17659) */
 
@@ -2803,6 +2816,31 @@ connection_ap_process_natd(entry_connection_t *conn)
   return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
 }
 
+static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 405 Method Not Allowed\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>This is an HTTP CONNECT tunnel, not a full HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>This is an HTTP CONNECT tunnel, not an HTTP proxy.</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use this Tor port as\n"
+  "an HTTP proxy.\n"
+  "</p><p>\n"
+  "This is not correct: This port is configured as a CONNECT tunnel, not\n"
+  "an HTTP proxy. Please configure your client accordingly.  You can also\n"
+  "use HTTPS; then the client should automatically use HTTP CONNECT."
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
 /** Called on an HTTP CONNECT entry connection when some bytes have arrived,
  * but we have not yet received a full HTTP CONNECT request.  Try to parse an
  * HTTP CONNECT request from the connection's inbuf.  On success, set up the
@@ -2843,7 +2881,7 @@ connection_ap_process_http_connect(entry_connection_t *conn)
   tor_assert(command);
   tor_assert(addrport);
   if (strcasecmp(command, "connect")) {
-    errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n";
+    errmsg = HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG;
     goto err;
   }
 
@@ -3706,6 +3744,10 @@ handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn)
   /* Link the circuit and the connection crypt path. */
   conn->cpath_layer = origin_circ->cpath->prev;
 
+  /* If this is the first stream on this circuit, tell circpad */
+  if (!origin_circ->p_streams)
+    circpad_machine_event_circ_has_streams(origin_circ);
+
   /* Add it into the linked list of p_streams on this circuit */
   conn->next_stream = origin_circ->p_streams;
   origin_circ->p_streams = conn;
@@ -3796,6 +3838,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
 
   if (! bcell.is_begindir) {
     /* Steal reference */
+    tor_assert(bcell.address);
     address = bcell.address;
     port = bcell.port;
 
@@ -4528,6 +4571,25 @@ circuit_clear_isolation(origin_circuit_t *circ)
   circ->socks_username_len = circ->socks_password_len = 0;
 }
 
+/** Send an END and mark for close the given edge connection conn using the
+ * given reason that has to be a stream reason.
+ *
+ * Note: We don't unattached the AP connection (if applicable) because we
+ * don't want to flush the remaining data. This function aims at ending
+ * everything quickly regardless of the connection state.
+ *
+ * This function can't fail and does nothing if conn is NULL. */
+void
+connection_edge_end_close(edge_connection_t *conn, uint8_t reason)
+{
+  if (!conn) {
+    return;
+  }
+
+  connection_edge_end(conn, reason);
+  connection_mark_for_close(TO_CONN(conn));
+}
+
 /** Free all storage held in module-scoped variables for connection_edge.c */
 void
 connection_edge_free_all(void)
diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h
index 68d8b19a11..e82b6bd765 100644
--- a/src/core/or/connection_edge.h
+++ b/src/core/or/connection_edge.h
@@ -80,6 +80,7 @@ int connection_edge_process_inbuf(edge_connection_t *conn,
 int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn);
 int connection_edge_end(edge_connection_t *conn, uint8_t reason);
 int connection_edge_end_errno(edge_connection_t *conn);
+void connection_edge_end_close(edge_connection_t *conn, uint8_t reason);
 int connection_edge_flushed_some(edge_connection_t *conn);
 int connection_edge_finished_flushing(edge_connection_t *conn);
 int connection_edge_finished_connecting(edge_connection_t *conn);
diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c
index 67157d8d0b..4c93351e31 100644
--- a/src/core/or/connection_or.c
+++ b/src/core/or/connection_or.c
@@ -22,13 +22,14 @@
  **/
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
 #define TOR_CHANNEL_INTERNAL_
 #define CONNECTION_OR_PRIVATE
+#define ORCONN_EVENT_PRIVATE
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitbuild.h"
@@ -38,7 +39,7 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/reachability.h"
@@ -46,6 +47,7 @@
 #include "lib/geoip/geoip.h"
 #include "core/mainloop/mainloop.h"
 #include "trunnel/link_handshake.h"
+#include "trunnel/netinfo.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -78,6 +80,8 @@
 #include "lib/tls/tortls.h"
 #include "lib/tls/x509.h"
 
+#include "core/or/orconn_event.h"
+
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_launch_v3_or_handshake(or_connection_t *conn);
 static int connection_or_process_cells_from_inbuf(or_connection_t *conn);
@@ -400,6 +404,55 @@ connection_or_report_broken_states(int severity, int domain)
   smartlist_free(items);
 }
 
+/**
+ * Helper function to publish an OR connection status event
+ *
+ * Publishes a messages to subscribers of ORCONN messages, and sends
+ * the control event.
+ **/
+void
+connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp,
+                           int reason)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  msg->status = tp;
+  msg->reason = reason;
+  orconn_status_publish(msg);
+  control_event_or_conn_status(conn, tp, reason);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * connection_or_change_state() calls this to notify subscribers about
+ * a change of an OR connection state.
+ **/
+static void
+connection_or_state_publish(const or_connection_t *conn, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  if (conn->is_pt) {
+    /* Do extra decoding because conn->proxy_type indicates the proxy
+     * protocol that tor uses to talk with the transport plugin,
+     * instead of PROXY_PLUGGABLE. */
+    tor_assert_nonfatal(conn->proxy_type != PROXY_NONE);
+    msg->proxy_type = PROXY_PLUGGABLE;
+  } else {
+    msg->proxy_type = conn->proxy_type;
+  }
+  msg->state = state;
+  if (conn->chan) {
+    msg->chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier;
+  } else {
+    msg->chan = 0;
+  }
+  orconn_state_publish(msg);
+}
+
 /** Call this to change or_connection_t states, so the owning channel_tls_t can
  * be notified.
  */
@@ -407,16 +460,13 @@ connection_or_report_broken_states(int severity, int domain)
 static void
 connection_or_change_state(or_connection_t *conn, uint8_t state)
 {
-  uint8_t old_state;
-
   tor_assert(conn);
 
-  old_state = conn->base_.state;
   conn->base_.state = state;
 
+  connection_or_state_publish(conn, state);
   if (conn->chan)
-    channel_tls_handle_state_change_on_orconn(conn->chan, conn,
-                                              old_state, state);
+    channel_tls_handle_state_change_on_orconn(conn->chan, conn, state);
 }
 
 /** Return the number of circuits using an or_connection_t; this used to
@@ -707,8 +757,6 @@ connection_or_finished_connecting(or_connection_t *or_conn)
 
   log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.",
             conn->address,conn->port);
-  control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  control_event_boot_first_orconn();
 
   if (proxy_type != PROXY_NONE) {
     /* start proxy handshake */
@@ -758,8 +806,8 @@ connection_or_about_to_close(or_connection_t *or_conn)
       entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan));
       if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) {
         int reason = tls_error_to_orconn_end_reason(or_conn->tls_error);
-        control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED,
-                                     reason);
+        connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED,
+                                   reason);
         if (!authdir_mode_tests_reachability(options))
           control_event_bootstrap_prob_or(
                 orconn_end_reason_to_control_string(reason),
@@ -769,10 +817,10 @@ connection_or_about_to_close(or_connection_t *or_conn)
   } else if (conn->hold_open_until_flushed) {
     /* We only set hold_open_until_flushed when we're intentionally
      * closing a connection. */
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   } else if (!tor_digest_is_zero(or_conn->identity_digest)) {
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   }
 }
@@ -1364,7 +1412,7 @@ void
 connection_or_connect_failed(or_connection_t *conn,
                              int reason, const char *msg)
 {
-  control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason);
+  connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason);
   if (!authdir_mode_tests_reachability(get_options()))
     control_event_bootstrap_prob_or(msg, reason, conn);
   note_or_connect_failed(conn);
@@ -1430,7 +1478,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
   int r;
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt = 0;
 
   tor_assert(_addr);
   tor_assert(id_digest);
@@ -1471,21 +1519,27 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
     return NULL;
   }
 
-  connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
-
   conn->is_outgoing = 1;
 
   /* If we are using a proxy server, find it and use it. */
-  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, TO_CONN(conn));
+  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         TO_CONN(conn));
   if (r == 0) {
     conn->proxy_type = proxy_type;
     if (proxy_type != PROXY_NONE) {
       tor_addr_copy(&addr, &proxy_addr);
       port = proxy_port;
       conn->base_.proxy_state = PROXY_INFANT;
+      conn->is_pt = is_pt;
     }
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
   } else {
+    /* This duplication of state change calls is necessary in case we
+     * run into an error condition below */
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
+
     /* get_proxy_addrport() might fail if we have a Bridge line that
        references a transport, but no ClientTransportPlugin lines
        defining its transport proxy. If this is the case, let's try to
@@ -1981,8 +2035,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
 
     /* Tell the new guard API about the channel failure */
     entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan));
-    control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED,
-                                 END_OR_CONN_REASON_OR_IDENTITY);
+    connection_or_event_status(conn, OR_CONN_EVENT_FAILED,
+                               END_OR_CONN_REASON_OR_IDENTITY);
     if (!authdir_mode_tests_reachability(options))
       control_event_bootstrap_prob_or(
                                 "Unexpected identity in router certificate",
@@ -2222,7 +2276,7 @@ int
 connection_or_set_state_open(or_connection_t *conn)
 {
   connection_or_change_state(conn, OR_CONN_STATE_OPEN);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0);
 
   /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection
    * that uses an earlier link protocol should not be treated as a relay. */
@@ -2252,6 +2306,8 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   cell_pack(&networkcell, cell, conn->wide_circ_ids);
 
+  /* We need to count padding cells from this non-packed code path
+   * since they are sent via chan->write_cell() (which is not packed) */
   rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   if (cell->command == CELL_PADDING)
     rep_hist_padding_count_write(PADDING_TYPE_CELL);
@@ -2262,7 +2318,7 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
   if (conn->chan) {
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
-    if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) {
+    if (TLS_CHAN_TO_BASE(conn->chan)->padding_enabled) {
       rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
       if (cell->command == CELL_PADDING)
         rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
@@ -2292,6 +2348,7 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0);
 
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   /* Touch the channel's active timestamp if there is one */
   if (conn->chan)
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
@@ -2428,6 +2485,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus)
   return 0;
 }
 
+static netinfo_addr_t *
+netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr)
+{
+  sa_family_t addr_family = tor_addr_family(tor_addr);
+
+  if (BUG(addr_family != AF_INET && addr_family != AF_INET6))
+    return NULL;
+
+  netinfo_addr_t *netinfo_addr = netinfo_addr_new();
+
+  if (addr_family == AF_INET) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4);
+    netinfo_addr_set_len(netinfo_addr, 4);
+    netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr));
+  } else if (addr_family == AF_INET6) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6);
+    netinfo_addr_set_len(netinfo_addr, 16);
+    uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr);
+    memcpy(ipv6_buf, in6_addr, 16);
+  }
+
+  return netinfo_addr;
+}
+
 /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know
  * about their address, our address, and the current time. */
 MOCK_IMPL(int,
@@ -2436,8 +2518,7 @@ connection_or_send_netinfo,(or_connection_t *conn))
   cell_t cell;
   time_t now = time(NULL);
   const routerinfo_t *me;
-  int len;
-  uint8_t *out;
+  int r = -1;
 
   tor_assert(conn->handshake_state);
 
@@ -2450,20 +2531,21 @@ connection_or_send_netinfo,(or_connection_t *conn))
   memset(&cell, 0, sizeof(cell_t));
   cell.command = CELL_NETINFO;
 
+  netinfo_cell_t *netinfo_cell = netinfo_cell_new();
+
   /* Timestamp, if we're a relay. */
   if (public_server_mode(get_options()) || ! conn->is_outgoing)
-    set_uint32(cell.payload, htonl((uint32_t)now));
+    netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now);
 
   /* Their address. */
-  out = cell.payload + 4;
+  const tor_addr_t *remote_tor_addr =
+    !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr;
   /* We use &conn->real_addr below, unless it hasn't yet been set. If it
    * hasn't yet been set, we know that base_.addr hasn't been tampered with
    * yet either. */
-  len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr)
-                                       ? &conn->real_addr : &conn->base_.addr);
-  if (len<0)
-    return -1;
-  out += len;
+  netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr);
+
+  netinfo_cell_set_other_addr(netinfo_cell, their_addr);
 
   /* My address -- only include it if I'm a public relay, or if I'm a
    * bridge and this is an incoming connection. If I'm a bridge and this
@@ -2471,28 +2553,42 @@ connection_or_send_netinfo,(or_connection_t *conn))
   if ((public_server_mode(get_options()) || !conn->is_outgoing) &&
       (me = router_get_my_routerinfo())) {
     tor_addr_t my_addr;
-    *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr);
-
     tor_addr_from_ipv4h(&my_addr, me->addr);
-    len = append_address_to_payload(out, &my_addr);
-    if (len < 0)
-      return -1;
-    out += len;
+
+    uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr);
+    netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs);
+
+    netinfo_cell_add_my_addrs(netinfo_cell,
+                              netinfo_addr_from_tor_addr(&my_addr));
 
     if (!tor_addr_is_null(&me->ipv6_addr)) {
-      len = append_address_to_payload(out, &me->ipv6_addr);
-      if (len < 0)
-        return -1;
+      netinfo_cell_add_my_addrs(netinfo_cell,
+                                netinfo_addr_from_tor_addr(&me->ipv6_addr));
     }
-  } else {
-    *out = 0;
+  }
+
+  const char *errmsg = NULL;
+  if ((errmsg = netinfo_cell_check(netinfo_cell))) {
+    log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s",
+                    errmsg);
+    goto cleanup;
+  }
+
+  if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                          netinfo_cell) < 0) {
+    log_warn(LD_OR, "Failed generating NETINFO cell");
+    goto cleanup;
   }
 
   conn->handshake_state->digest_sent_data = 0;
   conn->handshake_state->sent_netinfo = 1;
   connection_or_write_cell_to_buf(&cell, conn);
 
-  return 0;
+  r = 0;
+ cleanup:
+  netinfo_cell_free(netinfo_cell);
+
+  return r;
 }
 
 /** Helper used to add an encoded certs to a cert cell */
diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h
index 817bcdd317..272f536b83 100644
--- a/src/core/or/connection_or.h
+++ b/src/core/or/connection_or.h
@@ -17,32 +17,7 @@ struct ed25519_keypair_t;
 
 or_connection_t *TO_OR_CONN(connection_t *);
 
-#define OR_CONN_STATE_MIN_ 1
-/** State for a connection to an OR: waiting for connect() to finish. */
-#define OR_CONN_STATE_CONNECTING 1
-/** State for a connection to an OR: waiting for proxy handshake to complete */
-#define OR_CONN_STATE_PROXY_HANDSHAKING 2
-/** State for an OR connection client: SSL is handshaking, not done
- * yet. */
-#define OR_CONN_STATE_TLS_HANDSHAKING 3
-/** State for a connection to an OR: We're doing a second SSL handshake for
- * renegotiation purposes. (V2 handshake only.) */
-#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
-/** State for a connection at an OR: We're waiting for the client to
- * renegotiate (to indicate a v2 handshake) or send a versions cell (to
- * indicate a v3 handshake) */
-#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
-/** State for an OR connection: We're done with our SSL handshake, we've done
- * renegotiation, but we haven't yet negotiated link protocol versions and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
-/** State for an OR connection: We're done with our SSL handshake, but we
- * haven't yet negotiated link protocol versions, done a V3 handshake, and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
-/** State for an OR connection: Ready to send/receive cells. */
-#define OR_CONN_STATE_OPEN 8
-#define OR_CONN_STATE_MAX_ 8
+#include "core/or/orconn_event.h"
 
 void connection_or_clear_identity(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
@@ -81,6 +56,9 @@ MOCK_DECL(void,connection_or_close_for_error,
 
 void connection_or_report_broken_states(int severity, int domain);
 
+void connection_or_event_status(or_connection_t *conn,
+                                or_conn_status_event_t tp, int reason);
+
 MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn,
                                               int receiving));
 int connection_tls_continue_handshake(or_connection_t *conn);
diff --git a/src/core/or/connection_st.h b/src/core/or/connection_st.h
index d1430eda14..1c42a56d6b 100644
--- a/src/core/or/connection_st.h
+++ b/src/core/or/connection_st.h
@@ -146,4 +146,4 @@ struct connection_t {
  * directory connection. */
 #define DIR_CONN_IS_SERVER(conn) ((conn)->purpose == DIR_PURPOSE_SERVER)
 
-#endif
+#endif /* !defined(CONNECTION_ST_H) */
diff --git a/src/core/or/cpath_build_state_st.h b/src/core/or/cpath_build_state_st.h
index dbe596d851..4572a10430 100644
--- a/src/core/or/cpath_build_state_st.h
+++ b/src/core/or/cpath_build_state_st.h
@@ -34,5 +34,5 @@ struct cpath_build_state_t {
   time_t expiry_time;
 };
 
-#endif
+#endif /* !defined(CIRCUIT_BUILD_STATE_ST_ST_H) */
 
diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c
new file mode 100644
index 0000000000..9b7efc28bf
--- /dev/null
+++ b/src/core/or/crypt_path.c
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypt_path.c
+ *
+ * \brief Functions dealing with layered circuit encryption. This file aims to
+ *   provide an API around the crypt_path_t structure which holds crypto
+ *   information about a specific hop of a circuit.
+ *
+ * TODO: We should eventually move all functions dealing and manipulating
+ *   crypt_path_t to this file, so that eventually we encapsulate more and more
+ *   of crypt_path_t. Here are some more functions that can be moved here with
+ *   some more effort:
+ *
+ *   - circuit_list_path_impl()
+ *   - Functions dealing with cpaths in HSv2 create_rend_cpath() and
+ *     create_rend_cpath_legacy()
+ *   - The cpath related parts of rend_service_receive_introduction() and
+ *     rend_client_send_introduction().
+ **/
+
+#define CRYPT_PATH_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/crypt_path.h"
+
+#include "core/crypto/relay_crypto.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/cell_st.h"
+
+/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
+ * This function is used to extend cpath by another hop.
+ */
+void
+cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop)
+{
+  if (*head_ptr) {
+    new_hop->next = (*head_ptr);
+    new_hop->prev = (*head_ptr)->prev;
+    (*head_ptr)->prev->next = new_hop;
+    (*head_ptr)->prev = new_hop;
+  } else {
+    *head_ptr = new_hop;
+    new_hop->prev = new_hop->next = new_hop;
+  }
+}
+
+/** Create a new hop, annotate it with information about its
+ * corresponding router <b>choice</b>, and append it to the
+ * end of the cpath <b>head_ptr</b>. */
+int
+cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
+{
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+
+  /* link hop into the cpath, at the end. */
+  cpath_extend_linked_list(head_ptr, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_CLOSED;
+
+  hop->extend_info = extend_info_dup(choice);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  return 0;
+}
+
+/** Verify that cpath <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_ok(const crypt_path_t *cp)
+{
+  const crypt_path_t *start = cp;
+
+  do {
+    cpath_assert_layer_ok(cp);
+    /* layers must be in sequence of: "open* awaiting? closed*" */
+    if (cp != start) {
+      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      } else if (cp->state == CPATH_STATE_OPEN) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      }
+    }
+    cp = cp->next;
+    tor_assert(cp);
+  } while (cp != start);
+}
+
+/** Verify that cpath layer <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_layer_ok(const crypt_path_t *cp)
+{
+//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
+//  tor_assert(cp->port);
+  tor_assert(cp);
+  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
+  switch (cp->state)
+    {
+    case CPATH_STATE_OPEN:
+      relay_crypto_assert_ok(&cp->pvt_crypto);
+      FALLTHROUGH;
+    case CPATH_STATE_CLOSED:
+      /*XXXX Assert that there's no handshake_state either. */
+      tor_assert(!cp->rend_dh_handshake_state);
+      break;
+    case CPATH_STATE_AWAITING_KEYS:
+      /* tor_assert(cp->dh_handshake_state); */
+      break;
+    default:
+      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
+      tor_assert(0);
+    }
+  tor_assert(cp->package_window >= 0);
+  tor_assert(cp->deliver_window >= 0);
+}
+
+/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
+ *   - 20 to initialize f_digest
+ *   - 20 to initialize b_digest
+ *   - 16 to key f_crypto
+ *   - 16 to key b_crypto
+ *
+ * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
+ */
+int
+cpath_init_circuit_crypto(crypt_path_t *cpath,
+                          const char *key_data, size_t key_data_len,
+                          int reverse, int is_hs_v3)
+{
+
+  tor_assert(cpath);
+  return relay_crypto_init(&cpath->pvt_crypto, key_data, key_data_len,
+                           reverse, is_hs_v3);
+}
+
+/** Deallocate space associated with the cpath node <b>victim</b>. */
+void
+cpath_free(crypt_path_t *victim)
+{
+  if (!victim)
+    return;
+
+  relay_crypto_clear(&victim->pvt_crypto);
+  onion_handshake_state_release(&victim->handshake_state);
+  crypto_dh_free(victim->rend_dh_handshake_state);
+  extend_info_free(victim->extend_info);
+
+  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
+  tor_free(victim);
+}
+
+/********************** cpath crypto API *******************************/
+
+/** Encrypt or decrypt <b>payload</b> using the crypto of <b>cpath</b>. Actual
+ *  operation decided by <b>is_decrypt</b>.  */
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt)
+{
+  if (is_decrypt) {
+    relay_crypt_one_payload(cpath->pvt_crypto.b_crypto, payload);
+  } else {
+    relay_crypt_one_payload(cpath->pvt_crypto.f_crypto, payload);
+  }
+}
+
+/** Getter for the incoming digest of <b>cpath</b>. */
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath)
+{
+  return cpath->pvt_crypto.b_digest;
+}
+
+/** Set the right integrity digest on the outgoing <b>cell</b> based on the
+ *  cell payload and update the forward digest of <b>cpath</b>. */
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell)
+{
+  relay_set_digest(cpath->pvt_crypto.f_digest, cell);
+}
+
+/************ cpath sendme API ***************************/
+
+/** Return the sendme_digest of this <b>cpath</b>. */
+uint8_t *
+cpath_get_sendme_digest(crypt_path_t *cpath)
+{
+  return relay_crypto_get_sendme_digest(&cpath->pvt_crypto);
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+cpath_sendme_record_cell_digest(crypt_path_t *cpath, bool is_foward_digest)
+{
+  tor_assert(cpath);
+  relay_crypto_record_sendme_digest(&cpath->pvt_crypto, is_foward_digest);
+}
+
+/************ other cpath functions ***************************/
+
+/** Return the first non-open hop in cpath, or return NULL if all
+ * hops are open. */
+crypt_path_t *
+cpath_get_next_non_open_hop(crypt_path_t *cpath)
+{
+  crypt_path_t *hop = cpath;
+  do {
+    if (hop->state != CPATH_STATE_OPEN)
+      return hop;
+    hop = hop->next;
+  } while (hop != cpath);
+  return NULL;
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/** Unittest helper function: Count number of hops in cpath linked list. */
+unsigned int
+cpath_get_n_hops(crypt_path_t **head_ptr)
+{
+  unsigned int n_hops = 0;
+  crypt_path_t *tmp;
+
+  if (!*head_ptr) {
+    return 0;
+  }
+
+  tmp = *head_ptr;
+  do {
+    n_hops++;
+    tmp = tmp->next;
+  } while (tmp != *head_ptr);
+
+  return n_hops;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h
new file mode 100644
index 0000000000..7a95fec2b4
--- /dev/null
+++ b/src/core/or/crypt_path.h
@@ -0,0 +1,46 @@
+/**
+ * \file crypt_path.h
+ * \brief Header file for crypt_path.c.
+ **/
+
+#ifndef CRYPT_PATH_H
+#define CRYPT_PATH_H
+
+void cpath_assert_layer_ok(const crypt_path_t *cp);
+
+void cpath_assert_ok(const crypt_path_t *cp);
+
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+
+int cpath_init_circuit_crypto(crypt_path_t *cpath,
+                              const char *key_data, size_t key_data_len,
+                              int reverse, int is_hs_v3);
+
+void
+cpath_free(crypt_path_t *victim);
+
+void cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop);
+
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt);
+
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath);
+
+void cpath_sendme_record_cell_digest(crypt_path_t *cpath,
+                                     bool is_foward_digest);
+
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell);
+
+crypt_path_t *cpath_get_next_non_open_hop(crypt_path_t *cpath);
+
+void cpath_sendme_circuit_record_inbound_cell(crypt_path_t *cpath);
+
+uint8_t *cpath_get_sendme_digest(crypt_path_t *cpath);
+
+#if defined(TOR_UNIT_TESTS)
+unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(CRYPT_PATH_H) */
diff --git a/src/core/or/crypt_path_reference_st.h b/src/core/or/crypt_path_reference_st.h
index 3d79f26c1c..1827022b4e 100644
--- a/src/core/or/crypt_path_reference_st.h
+++ b/src/core/or/crypt_path_reference_st.h
@@ -19,5 +19,5 @@ struct crypt_path_reference_t {
   crypt_path_t *cpath;
 };
 
-#endif
+#endif /* !defined(CRYPT_PATH_REFERENCE_ST_H) */
 
diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h
index 429480f8ab..249ac6aaa3 100644
--- a/src/core/or/crypt_path_st.h
+++ b/src/core/or/crypt_path_st.h
@@ -24,15 +24,24 @@ struct onion_handshake_state_t {
   } u;
 };
 
+/** Macro to encapsulate private members of a struct.
+ *
+ *  Renames 'x' to 'x_crypt_path_private_field'.
+ */
+#define CRYPT_PATH_PRIV_FIELD(x) x ## _crypt_path_private_field
+
+#ifdef CRYPT_PATH_PRIVATE
+
+/* Helper macro to access private members of a struct. */
+#define pvt_crypto CRYPT_PATH_PRIV_FIELD(crypto)
+
+#endif /* defined(CRYPT_PATH_PRIVATE) */
+
 /** Holds accounting information for a single step in the layered encryption
  * performed by a circuit.  Used only at the client edge of a circuit. */
 struct crypt_path_t {
   uint32_t magic;
 
-  /** Cryptographic state used for encrypting and authenticating relay
-   * cells to and from this hop. */
-  relay_crypto_t crypto;
-
   /** Current state of the handshake as performed with the OR at this
    * step. */
   onion_handshake_state_t handshake_state;
@@ -65,6 +74,12 @@ struct crypt_path_t {
                        * at this step? */
   int deliver_window; /**< How many cells are we willing to deliver originating
                        * at this step? */
+
+  /*********************** Private members ****************************/
+
+  /** Private member: Cryptographic state used for encrypting and
+   * authenticating relay cells to and from this hop. */
+  relay_crypto_t CRYPT_PATH_PRIV_FIELD(crypto);
 };
 
-#endif
+#endif /* !defined(CRYPT_PATH_ST_H) */
diff --git a/src/core/or/destroy_cell_queue_st.h b/src/core/or/destroy_cell_queue_st.h
index 56630670ba..e917afc700 100644
--- a/src/core/or/destroy_cell_queue_st.h
+++ b/src/core/or/destroy_cell_queue_st.h
@@ -23,5 +23,5 @@ struct destroy_cell_queue_t {
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
+#endif /* !defined(DESTROY_CELL_QUEUE_ST_H) */
 
diff --git a/src/core/or/dos.h b/src/core/or/dos.h
index 058b7afce6..a46f65d767 100644
--- a/src/core/or/dos.h
+++ b/src/core/or/dos.h
@@ -135,7 +135,7 @@ MOCK_DECL(STATIC unsigned int, get_param_cc_enabled,
 MOCK_DECL(STATIC unsigned int, get_param_conn_enabled,
           (const networkstatus_t *ns));
 
-#endif /* TOR_DOS_PRIVATE */
+#endif /* defined(DOS_PRIVATE) */
 
-#endif /* TOR_DOS_H */
+#endif /* !defined(TOR_DOS_H) */
 
diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h
index 1665b8589f..8922a3a9cf 100644
--- a/src/core/or/edge_connection_st.h
+++ b/src/core/or/edge_connection_st.h
@@ -73,5 +73,5 @@ struct edge_connection_t {
   uint64_t dirreq_id;
 };
 
-#endif
+#endif /* !defined(EDGE_CONNECTION_ST_H) */
 
diff --git a/src/core/or/entry_connection_st.h b/src/core/or/entry_connection_st.h
index 45621fadbf..e65c545d17 100644
--- a/src/core/or/entry_connection_st.h
+++ b/src/core/or/entry_connection_st.h
@@ -96,5 +96,5 @@ struct entry_connection_t {
 /** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
 #define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
 
-#endif
+#endif /* !defined(ENTRY_CONNECTION_ST_H) */
 
diff --git a/src/core/or/entry_port_cfg_st.h b/src/core/or/entry_port_cfg_st.h
index 87dfb331e5..b84838d44f 100644
--- a/src/core/or/entry_port_cfg_st.h
+++ b/src/core/or/entry_port_cfg_st.h
@@ -50,5 +50,5 @@ struct entry_port_cfg_t {
 
 };
 
-#endif
+#endif /* !defined(ENTRY_PORT_CFG_ST_H) */
 
diff --git a/src/core/or/extend_info_st.h b/src/core/or/extend_info_st.h
index bc7a77b1b2..7704ff16b5 100644
--- a/src/core/or/extend_info_st.h
+++ b/src/core/or/extend_info_st.h
@@ -27,4 +27,4 @@ struct extend_info_t {
   curve25519_public_key_t curve25519_onion_key;
 };
 
-#endif
+#endif /* !defined(EXTEND_INFO_ST_H) */
diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h
index d4617be108..1fe47ad3f1 100644
--- a/src/core/or/half_edge_st.h
+++ b/src/core/or/half_edge_st.h
@@ -30,5 +30,5 @@ typedef struct half_edge_t {
   int connected_pending : 1;
 } half_edge_t;
 
-#endif
+#endif /* !defined(HALF_EDGE_ST_H) */
 
diff --git a/src/core/or/listener_connection_st.h b/src/core/or/listener_connection_st.h
index 8989a39dc8..1250d9c9b4 100644
--- a/src/core/or/listener_connection_st.h
+++ b/src/core/or/listener_connection_st.h
@@ -21,5 +21,5 @@ struct listener_connection_t {
 
 };
 
-#endif
+#endif /* !defined(LISTENER_CONNECTION_ST_H) */
 
diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c
new file mode 100644
index 0000000000..3cb9147134
--- /dev/null
+++ b/src/core/or/ocirc_event.c
@@ -0,0 +1,128 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.c
+ * \brief Publish state change messages for origin circuits
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of origin circuits.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/ocirc_event_sys.h"
+#include "core/or/origin_circuit_st.h"
+#include "lib/subsys/subsys.h"
+
+DECLARE_PUBLISH(ocirc_state);
+DECLARE_PUBLISH(ocirc_chan);
+DECLARE_PUBLISH(ocirc_cevent);
+
+static void
+ocirc_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+ocirc_state_fmt(msg_aux_data_t u)
+{
+  ocirc_state_msg_t *msg = (ocirc_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" state=%d onehop=%d>",
+               msg->gid, msg->state, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_chan_fmt(msg_aux_data_t u)
+{
+  ocirc_chan_msg_t *msg = (ocirc_chan_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" chan=%"PRIu64" onehop=%d>",
+               msg->gid, msg->chan, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_cevent_fmt(msg_aux_data_t u)
+{
+  ocirc_cevent_msg_t *msg = (ocirc_cevent_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" evtype=%d reason=%d onehop=%d>",
+               msg->gid, msg->evtype, msg->reason, msg->onehop);
+  return s;
+}
+
+static dispatch_typefns_t ocirc_state_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_state_fmt,
+};
+
+static dispatch_typefns_t ocirc_chan_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_chan_fmt,
+};
+
+static dispatch_typefns_t ocirc_cevent_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_cevent_fmt,
+};
+
+static int
+ocirc_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_state, &ocirc_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_chan, &ocirc_chan_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_cevent, &ocirc_cevent_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_state))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  return 0;
+}
+
+void
+ocirc_state_publish(ocirc_state_msg_t *msg)
+{
+  PUBLISH(ocirc_state, msg);
+}
+
+void
+ocirc_chan_publish(ocirc_chan_msg_t *msg)
+{
+  PUBLISH(ocirc_chan, msg);
+}
+
+void
+ocirc_cevent_publish(ocirc_cevent_msg_t *msg)
+{
+  PUBLISH(ocirc_cevent, msg);
+}
+
+const subsys_fns_t sys_ocirc_event = {
+  .name = "ocirc_event",
+  .supported = true,
+  .level = -32,
+  .add_pubsub = ocirc_add_pubsub,
+};
diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h
new file mode 100644
index 0000000000..8e9494874f
--- /dev/null
+++ b/src/core/or/ocirc_event.h
@@ -0,0 +1,72 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.h
+ * \brief Header file for ocirc_event.c
+ **/
+
+#ifndef TOR_OCIRC_EVENT_H
+#define TOR_OCIRC_EVENT_H
+
+#include <stdbool.h>
+
+#include "lib/cc/torint.h"
+#include "lib/pubsub/pubsub.h"
+
+/** Used to indicate the type of a circuit event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum circuit_status_event_t {
+  CIRC_EVENT_LAUNCHED = 0,
+  CIRC_EVENT_BUILT    = 1,
+  CIRC_EVENT_EXTENDED = 2,
+  CIRC_EVENT_FAILED   = 3,
+  CIRC_EVENT_CLOSED   = 4,
+} circuit_status_event_t;
+
+/** Message for origin circuit state update */
+typedef struct ocirc_state_msg_t {
+  uint32_t gid;       /**< global ID (only origin circuits have them) */
+  int state;          /**< new circuit state */
+  bool onehop;        /**< one-hop circuit? */
+} ocirc_state_msg_t;
+
+DECLARE_MESSAGE(ocirc_state, ocirc_state, ocirc_state_msg_t *);
+
+/**
+ * Message when a channel gets associated to a circuit.
+ *
+ * This doesn't always correspond to something in circuitbuild.c
+ * setting the n_chan field in the circuit.  For some reason, if
+ * circuit_handle_first_hop() launches a new circuit, it doesn't set
+ * the n_chan field.
+ */
+typedef struct ocirc_chan_msg_t {
+  uint32_t gid;                 /**< global ID */
+  uint64_t chan;                /**< channel ID */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_chan_msg_t;
+
+DECLARE_MESSAGE(ocirc_chan, ocirc_chan, ocirc_chan_msg_t *);
+
+/**
+ * Message for origin circuit status event
+ *
+ * This contains information that ends up in CIRC control protocol events.
+ */
+typedef struct ocirc_cevent_msg_t {
+  uint32_t gid;                 /**< global ID */
+  int evtype;                   /**< event type */
+  int reason;                   /**< reason */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_cevent_msg_t;
+
+DECLARE_MESSAGE(ocirc_cevent, ocirc_cevent, ocirc_cevent_msg_t *);
+
+#ifdef OCIRC_EVENT_PRIVATE
+void ocirc_state_publish(ocirc_state_msg_t *msg);
+void ocirc_chan_publish(ocirc_chan_msg_t *msg);
+void ocirc_cevent_publish(ocirc_cevent_msg_t *msg);
+#endif
+
+#endif /* !defined(TOR_OCIRC_EVENT_H) */
diff --git a/src/core/or/ocirc_event_sys.h b/src/core/or/ocirc_event_sys.h
new file mode 100644
index 0000000000..61180496da
--- /dev/null
+++ b/src/core/or/ocirc_event_sys.h
@@ -0,0 +1,13 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+
+/**
+ * \file ocirc_event_sys.h
+ * \brief Declare subsystem object for the origin circuit event module.
+ **/
+
+#ifndef TOR_OCIRC_EVENT_SYS_H
+#define TOR_OCIRC_EVENT_SYS_H
+
+extern const struct subsys_fns_t sys_ocirc_event;
+
+#endif /* !defined(TOR_OCIRC_EVENT_SYS_H) */
diff --git a/src/core/or/or.h b/src/core/or/or.h
index 7c601e49b3..990cfacbc0 100644
--- a/src/core/or/or.h
+++ b/src/core/or/or.h
@@ -26,7 +26,7 @@
 #include "lib/cc/compat_compiler.h"
 #include "lib/cc/torint.h"
 #include "lib/container/map.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/container/smartlist.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rsa.h"
@@ -97,6 +97,8 @@ struct curve25519_public_key_t;
 #define SIGNEWNYM 129
 #define SIGCLEARDNSCACHE 130
 #define SIGHEARTBEAT 131
+#define SIGACTIVE 132
+#define SIGDORMANT 133
 
 #if (SIZEOF_CELL_T != 0)
 /* On Irix, stdlib.h defines a cell_t type, so we need to make sure
@@ -205,6 +207,9 @@ struct curve25519_public_key_t;
 #define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39
 #define RELAY_COMMAND_INTRODUCE_ACK 40
 
+#define RELAY_COMMAND_PADDING_NEGOTIATE 41
+#define RELAY_COMMAND_PADDING_NEGOTIATED 42
+
 /* Reasons why an OR connection is closed. */
 #define END_OR_CONN_REASON_DONE           1
 #define END_OR_CONN_REASON_REFUSED        2 /* connection refused */
@@ -834,6 +839,14 @@ typedef struct protover_summary_flags_t {
    * service rendezvous point supporting version 3 as seen in proposal 224.
    * This requires HSRend=2. */
   unsigned int supports_v3_rendezvous_point: 1;
+
+  /** True iff this router has a protocol list that allows clients to
+   * negotiate hs circuit setup padding. Requires Padding>=2. */
+  unsigned int supports_hs_setup_padding : 1;
+
+  /** True iff this router has a protocol list that allows it to support the
+   * ESTABLISH_INTRO DoS cell extension. Requires HSIntro>=5. */
+  unsigned int supports_establish_intro_dos_extension : 1;
 } protover_summary_flags_t;
 
 typedef struct routerinfo_t routerinfo_t;
diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h
index 6b6feb9d89..f3eb861613 100644
--- a/src/core/or/or_circuit_st.h
+++ b/src/core/or/or_circuit_st.h
@@ -12,6 +12,8 @@
 #include "core/or/circuit_st.h"
 #include "core/or/crypt_path_st.h"
 
+#include "lib/evloop/token_bucket.h"
+
 struct onion_queue_t;
 
 /** An or_circuit_t holds information needed to implement a circuit at an
@@ -33,11 +35,6 @@ struct or_circuit_t {
   cell_queue_t p_chan_cells;
   /** The channel that is previous in this circuit. */
   channel_t *p_chan;
-  /**
-   * Circuit mux associated with p_chan to which this circuit is attached;
-   * NULL if we have no p_chan.
-   */
-  circuitmux_t *p_mux;
   /** Linked list of Exit streams associated with this circuit. */
   edge_connection_t *n_streams;
   /** Linked list of Exit streams associated with this circuit that are
@@ -74,7 +71,16 @@ struct or_circuit_t {
    * exit-ward queues of this circuit; reset every time when writing
    * buffer stats to disk. */
   uint64_t total_cell_waiting_time;
+
+  /** If set, the DoS defenses are enabled on this circuit meaning that the
+   * introduce2_bucket is initialized and used. */
+  unsigned int introduce2_dos_defense_enabled : 1;
+
+  /** INTRODUCE2 cell bucket controlling how much can go on this circuit. Only
+   * used if this is a service introduction circuit at the intro point
+   * (purpose = CIRCUIT_PURPOSE_INTRO_POINT). */
+  token_bucket_ctr_t introduce2_bucket;
 };
 
-#endif
+#endif /* !defined(OR_CIRCUIT_ST_H) */
 
diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h
index d5db5e8694..051fcd00d3 100644
--- a/src/core/or/or_connection_st.h
+++ b/src/core/or/or_connection_st.h
@@ -67,6 +67,8 @@ struct or_connection_t {
    * geoip cache and handled by the DoS mitigation subsystem. We use this to
    * insure we have a coherent count of concurrent connection. */
   unsigned int tracked_for_dos_mitigation : 1;
+  /** True iff this connection is using a pluggable transport */
+  unsigned int is_pt : 1;
 
   uint16_t link_proto; /**< What protocol version are we using? 0 for
                         * "none negotiated yet." */
@@ -89,4 +91,4 @@ struct or_connection_t {
   uint64_t bytes_xmitted, bytes_xmitted_by_tls;
 };
 
-#endif
+#endif /* !defined(OR_CONNECTION_ST_H) */
diff --git a/src/core/or/or_handshake_certs_st.h b/src/core/or/or_handshake_certs_st.h
index a93b7104aa..9deb6d6d59 100644
--- a/src/core/or/or_handshake_certs_st.h
+++ b/src/core/or/or_handshake_certs_st.h
@@ -37,4 +37,4 @@ struct or_handshake_certs_t {
   size_t ed_rsa_crosscert_len;
 };
 
-#endif
+#endif /* !defined(OR_HANDSHAKE_CERTS_ST) */
diff --git a/src/core/or/or_handshake_state_st.h b/src/core/or/or_handshake_state_st.h
index 09a8a34179..472ce8a302 100644
--- a/src/core/or/or_handshake_state_st.h
+++ b/src/core/or/or_handshake_state_st.h
@@ -74,5 +74,5 @@ struct or_handshake_state_t {
   or_handshake_certs_t *certs;
 };
 
-#endif
+#endif /* !defined(OR_HANDSHAKE_STATE_ST) */
 
diff --git a/src/core/or/or_periodic.c b/src/core/or/or_periodic.c
new file mode 100644
index 0000000000..fe28c99192
--- /dev/null
+++ b/src/core/or/or_periodic.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.c
+ * @brief Periodic callbacks for the onion routing subsystem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+
+#include "core/or/channel.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_periodic.h"
+
+#include "feature/relay/routermode.h"
+
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+
+#define CHANNEL_CHECK_INTERVAL (60*60)
+static int
+check_canonical_channels_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (public_server_mode(options))
+    channel_check_for_duplicates();
+
+  return CHANNEL_CHECK_INTERVAL;
+}
+
+DECLARE_EVENT(check_canonical_channels, RELAY, FL(NEED_NET));
+
+/**
+ * Periodic callback: as a server, see if we have any old unused circuits
+ * that should be expired */
+static int
+expire_old_circuits_serverside_callback(time_t now,
+                                        const or_options_t *options)
+{
+  (void)options;
+  /* every 11 seconds, so not usually the same second as other such events */
+  circuit_expire_old_circuits_serverside(now);
+  return 11;
+}
+
+DECLARE_EVENT(expire_old_circuits_serverside, ROUTER, FL(NEED_NET));
+
+void
+or_register_periodic_events(void)
+{
+  // These are router-only events, but they're owned by the OR subsystem. */
+  periodic_events_register(&check_canonical_channels_event);
+  periodic_events_register(&expire_old_circuits_serverside_event);
+}
diff --git a/src/core/or/or_periodic.h b/src/core/or/or_periodic.h
new file mode 100644
index 0000000000..c2f47cf5ef
--- /dev/null
+++ b/src/core/or/or_periodic.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.h
+ * @brief Header for core/or/or_periodic.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_PERIODIC_H
+#define TOR_CORE_OR_OR_PERIODIC_H
+
+void or_register_periodic_events(void);
+
+#endif /* !defined(TOR_CORE_OR_OR_PERIODIC_H) */
diff --git a/src/core/or/or_sys.c b/src/core/or/or_sys.c
new file mode 100644
index 0000000000..6f8c81a4df
--- /dev/null
+++ b/src/core/or/or_sys.c
@@ -0,0 +1,43 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.c
+ * @brief Subsystem definitions for OR module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/or/or_periodic.h"
+#include "core/or/or_sys.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/versions.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_or_initialize(void)
+{
+  or_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_or_shutdown(void)
+{
+  protover_free_all();
+  protover_summary_cache_free_all();
+  policies_free_all();
+}
+
+const struct subsys_fns_t sys_or = {
+  .name = "or",
+  .supported = true,
+  .level = 20,
+  .initialize = subsys_or_initialize,
+  .shutdown = subsys_or_shutdown,
+};
diff --git a/src/core/or/or_sys.h b/src/core/or/or_sys.h
new file mode 100644
index 0000000000..c37ef01858
--- /dev/null
+++ b/src/core/or/or_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.h
+ * @brief Header for core/or/or_sys.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_SYS_H
+#define TOR_CORE_OR_OR_SYS_H
+
+extern const struct subsys_fns_t sys_or;
+
+#endif /* !defined(TOR_CORE_OR_OR_SYS_H) */
diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c
new file mode 100644
index 0000000000..86f112fc09
--- /dev/null
+++ b/src/core/or/orconn_event.c
@@ -0,0 +1,99 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.c
+ * \brief Publish state change messages for OR connections
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of OR connections.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/orconn_event.h"
+#include "core/or/orconn_event_sys.h"
+
+DECLARE_PUBLISH(orconn_state);
+DECLARE_PUBLISH(orconn_status);
+
+static void
+orconn_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+orconn_state_fmt(msg_aux_data_t u)
+{
+  orconn_state_msg_t *msg = (orconn_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" chan=%"PRIu64" proxy_type=%d state=%d>",
+               msg->gid, msg->chan, msg->proxy_type, msg->state);
+  return s;
+}
+
+static char *
+orconn_status_fmt(msg_aux_data_t u)
+{
+  orconn_status_msg_t *msg = (orconn_status_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" status=%d reason=%d>",
+               msg->gid, msg->status, msg->reason);
+  return s;
+}
+
+static dispatch_typefns_t orconn_state_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_state_fmt,
+};
+
+static dispatch_typefns_t orconn_status_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_status_fmt,
+};
+
+static int
+orconn_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_state, &orconn_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_status, &orconn_status_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_state) != 0)
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_status) != 0)
+    return -1;
+  return 0;
+}
+
+void
+orconn_state_publish(orconn_state_msg_t *msg)
+{
+  PUBLISH(orconn_state, msg);
+}
+
+void
+orconn_status_publish(orconn_status_msg_t *msg)
+{
+  PUBLISH(orconn_status, msg);
+}
+
+const subsys_fns_t sys_orconn_event = {
+  .name = "orconn_event",
+  .supported = true,
+  .level = -33,
+  .add_pubsub = orconn_add_pubsub,
+};
diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h
new file mode 100644
index 0000000000..fb67a7d183
--- /dev/null
+++ b/src/core/or/orconn_event.h
@@ -0,0 +1,103 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.h
+ * \brief Header file for orconn_event.c
+ *
+ * The OR_CONN_STATE_* symbols are here to make it easier for
+ * subscribers to make decisions based on the messages that they
+ * receive.
+ **/
+
+#ifndef TOR_ORCONN_EVENT_H
+#define TOR_ORCONN_EVENT_H
+
+#include "lib/pubsub/pubsub.h"
+
+/**
+ * @name States of OR connections
+ *
+ * These must be in a partial ordering such that usually no OR
+ * connection will transition from a higher-numbered state to a
+ * lower-numbered one.  Code such as bto_update_best() depends on this
+ * ordering to determine the best state it's seen so far.
+ * @{ */
+#define OR_CONN_STATE_MIN_ 1
+/** State for a connection to an OR: waiting for connect() to finish. */
+#define OR_CONN_STATE_CONNECTING 1
+/** State for a connection to an OR: waiting for proxy handshake to complete */
+#define OR_CONN_STATE_PROXY_HANDSHAKING 2
+/** State for an OR connection client: SSL is handshaking, not done
+ * yet. */
+#define OR_CONN_STATE_TLS_HANDSHAKING 3
+/** State for a connection to an OR: We're doing a second SSL handshake for
+ * renegotiation purposes. (V2 handshake only.) */
+#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
+/** State for a connection at an OR: We're waiting for the client to
+ * renegotiate (to indicate a v2 handshake) or send a versions cell (to
+ * indicate a v3 handshake) */
+#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
+/** State for an OR connection: We're done with our SSL handshake, we've done
+ * renegotiation, but we haven't yet negotiated link protocol versions and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
+/** State for an OR connection: We're done with our SSL handshake, but we
+ * haven't yet negotiated link protocol versions, done a V3 handshake, and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
+/** State for an OR connection: Ready to send/receive cells. */
+#define OR_CONN_STATE_OPEN 8
+#define OR_CONN_STATE_MAX_ 8
+/** @} */
+
+/** Used to indicate the type of an OR connection event passed to the
+ * controller.  The various types are defined in control-spec.txt */
+typedef enum or_conn_status_event_t {
+  OR_CONN_EVENT_LAUNCHED     = 0,
+  OR_CONN_EVENT_CONNECTED    = 1,
+  OR_CONN_EVENT_FAILED       = 2,
+  OR_CONN_EVENT_CLOSED       = 3,
+  OR_CONN_EVENT_NEW          = 4,
+} or_conn_status_event_t;
+
+/**
+ * Message for orconn state update
+ *
+ * This contains information about internal state changes of
+ * or_connection_t objects.  The chan and proxy_type fields are
+ * additional information that a subscriber may need to make
+ * decisions.
+ **/
+typedef struct orconn_state_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  uint64_t chan;                /**< associated channel ID */
+  int proxy_type;               /**< connection's proxy type */
+  uint8_t state;                /**< new connection state */
+} orconn_state_msg_t;
+
+DECLARE_MESSAGE(orconn_state, orconn_state, orconn_state_msg_t *);
+
+/**
+ * Message for orconn status event
+ *
+ * This contains information that ends up in ORCONN control protocol
+ * events.
+ **/
+typedef struct orconn_status_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  int status;                   /**< or_conn_status_event_t */
+  int reason;                   /**< reason */
+} orconn_status_msg_t;
+
+DECLARE_MESSAGE(orconn_status, orconn_status, orconn_status_msg_t *);
+
+#ifdef ORCONN_EVENT_PRIVATE
+void orconn_state_publish(orconn_state_msg_t *);
+void orconn_status_publish(orconn_status_msg_t *);
+#endif
+
+#endif /* !defined(TOR_ORCONN_EVENT_H) */
diff --git a/src/core/or/orconn_event_sys.h b/src/core/or/orconn_event_sys.h
new file mode 100644
index 0000000000..9703b2e3d1
--- /dev/null
+++ b/src/core/or/orconn_event_sys.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+
+/**
+ * \file orconn_event_sys.h
+ * \brief Declare subsystem object for the OR connection event module.
+ **/
+#ifndef TOR_ORCONN_EVENT_SYS_H
+#define TOR_ORCONN_EVENT_SYS_H
+
+extern const struct subsys_fns_t sys_orconn_event;
+
+#endif /* !defined(TOR_ORCONN_EVENT_SYS_H) */
diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h
index f55416db14..01bbc84ae2 100644
--- a/src/core/or/origin_circuit_st.h
+++ b/src/core/or/origin_circuit_st.h
@@ -161,6 +161,10 @@ struct origin_circuit_t {
    * connections to this circuit. */
   unsigned int unusable_for_new_conns : 1;
 
+  /* If this flag is set (due to padding negotiation failure), we should
+   * not try to negotiate further circuit padding. */
+  unsigned padding_negotiation_failed : 1;
+
   /**
    * Tristate variable to guard against pathbias miscounting
    * due to circuit purpose transitions changing the decision
@@ -291,4 +295,4 @@ struct origin_circuit_t {
 
 };
 
-#endif
+#endif /* !defined(ORIGIN_CIRCUIT_ST_H) */
diff --git a/src/core/or/policies.c b/src/core/or/policies.c
index 3ed282d785..39a0817c85 100644
--- a/src/core/or/policies.c
+++ b/src/core/or/policies.c
@@ -29,7 +29,9 @@
 #include "feature/relay/routermode.h"
 #include "lib/geoip/geoip.h"
 #include "ht.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/encoding/confline.h"
+#include "trunnel/ed25519_cert.h"
 
 #include "core/or/addr_policy_st.h"
 #include "feature/dirclient/dir_server_st.h"
@@ -461,7 +463,8 @@ fascist_firewall_use_ipv6(const or_options_t *options)
    * ClientPreferIPv6DirPort is deprecated, but check it anyway. */
   return (options->ClientUseIPv6 == 1 || options->ClientUseIPv4 == 0 ||
           options->ClientPreferIPv6ORPort == 1 ||
-          options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1);
+          options->ClientPreferIPv6DirPort == 1 || options->UseBridges == 1 ||
+          options->ClientAutoIPv6ORPort == 1);
 }
 
 /** Do we prefer to connect to IPv6, ignoring ClientPreferIPv6ORPort and
@@ -488,6 +491,15 @@ fascist_firewall_prefer_ipv6_impl(const or_options_t *options)
   return -1;
 }
 
+/* Choose whether we prefer IPv4 or IPv6 by randomly choosing an address
+ * family. Return 0 for IPv4, and 1 for IPv6. */
+MOCK_IMPL(int,
+fascist_firewall_rand_prefer_ipv6_addr, (void))
+{
+  /* TODO: Check for failures, and infer our preference based on this. */
+  return crypto_rand_int(2);
+}
+
 /** Do we prefer to connect to IPv6 ORPorts?
  * Use node_ipv6_or_preferred() whenever possible: it supports bridge client
  * per-node IPv6 preferences.
@@ -502,7 +514,10 @@ fascist_firewall_prefer_ipv6_orport(const or_options_t *options)
   }
 
   /* We can use both IPv4 and IPv6 - which do we prefer? */
-  if (options->ClientPreferIPv6ORPort == 1) {
+  if (options->ClientAutoIPv6ORPort == 1) {
+    /* If ClientAutoIPv6ORPort is 1, we prefer IPv4 or IPv6 at random. */
+    return fascist_firewall_rand_prefer_ipv6_addr();
+  } else if (options->ClientPreferIPv6ORPort == 1) {
     return 1;
   }
 
@@ -1001,6 +1016,83 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs,
   }
 }
 
+/** Like fascist_firewall_choose_address_base(), but takes in a smartlist
+ * <b>lspecs</b> consisting of one or more link specifiers. We assume
+ * fw_connection is FIREWALL_OR_CONNECTION as link specifiers cannot
+ * contain DirPorts.
+ */
+void
+fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                   int pref_only, tor_addr_port_t* ap)
+{
+  int have_v4 = 0, have_v6 = 0;
+  uint16_t port_v4 = 0, port_v6 = 0;
+  tor_addr_t addr_v4, addr_v6;
+
+  tor_assert(ap);
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Unknown or missing link specifiers");
+    return;
+  }
+  if (smartlist_len(lspecs) == 0) {
+    log_warn(LD_PROTOCOL, "Link specifiers are empty");
+    return;
+  }
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
+  tor_addr_make_null(&addr_v4, AF_INET);
+  tor_addr_make_null(&addr_v6, AF_INET6);
+
+  SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
+    switch (link_specifier_get_ls_type(ls)) {
+    case LS_IPV4:
+      /* Skip if we already seen a v4. */
+      if (have_v4) continue;
+      tor_addr_from_ipv4h(&addr_v4,
+                          link_specifier_get_un_ipv4_addr(ls));
+      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      have_v4 = 1;
+      break;
+    case LS_IPV6:
+      /* Skip if we already seen a v6, or deliberately skip it if we're not a
+       * direct connection. */
+      if (have_v6) continue;
+      tor_addr_from_ipv6_bytes(&addr_v6,
+          (const char *) link_specifier_getconstarray_un_ipv6_addr(ls));
+      port_v6 = link_specifier_get_un_ipv6_port(ls);
+      have_v6 = 1;
+      break;
+    default:
+      /* Ignore unknown. */
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* If we don't have IPv4 or IPv6 in link specifiers, log a bug and return. */
+  if (!have_v4 && !have_v6) {
+    if (!have_v6) {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4 or IPv6");
+    } else {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4");
+    }
+    return;
+  }
+
+  /* Here, don't check for DirPorts as link specifiers are only used for
+   * ORPorts. */
+  const or_options_t *options = get_options();
+  int pref_ipv6 = fascist_firewall_prefer_ipv6_orport(options);
+  /* Assume that the DirPorts are zero as link specifiers only use ORPorts. */
+  fascist_firewall_choose_address_base(&addr_v4, port_v4, 0,
+                                       &addr_v6, port_v6, 0,
+                                       FIREWALL_OR_CONNECTION,
+                                       pref_only, pref_ipv6,
+                                       ap);
+}
+
 /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and
  * looks up the node's IPv6 preference rather than taking an argument
  * for pref_ipv6. */
@@ -1150,6 +1242,15 @@ authdir_policy_badexit_address(uint32_t addr, uint16_t port)
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END
 
+/** Check <b>or_options</b> to determine whether or not we are using the
+ * default options for exit policy. Return true if so, false otherwise. */
+static int
+policy_using_default_exit_options(const or_options_t *or_options)
+{
+  return (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 &&
+          or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0);
+}
+
 /** Config helper: If there's any problem with the policy configuration
  * options in <b>options</b>, return -1 and set <b>msg</b> to a newly
  * allocated description of the error. Else return 0. */
@@ -1168,9 +1269,8 @@ validate_addr_policies(const or_options_t *options, char **msg)
 
   static int warned_about_nonexit = 0;
 
-  if (public_server_mode(options) &&
-      !warned_about_nonexit && options->ExitPolicy == NULL &&
-      options->ExitRelay == -1 && options->ReducedExitPolicy == 0) {
+  if (public_server_mode(options) && !warned_about_nonexit &&
+      policy_using_default_exit_options(options)) {
     warned_about_nonexit = 1;
     log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. "
                "If you want to be an exit relay, "
@@ -2127,9 +2227,9 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options,
   int rv = 0;
 
   /* Short-circuit for non-exit relays, or for relays where we didn't specify
-   * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */
-  if (or_options->ExitRelay == 0 || (or_options->ExitPolicy == NULL &&
-      or_options->ExitRelay == -1 && or_options->ReducedExitPolicy == 0)) {
+   * ExitPolicy or ReducedExitPolicy or IPv6Exit and ExitRelay is auto. */
+  if (or_options->ExitRelay == 0 ||
+      policy_using_default_exit_options(or_options)) {
     append_exit_policy_string(result, "reject *4:*");
     append_exit_policy_string(result, "reject *6:*");
     return 0;
@@ -2706,7 +2806,7 @@ parse_short_policy(const char *summary)
   int is_accept;
   int n_entries;
   short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */
-  const char *next;
+  char *next;
 
   if (!strcmpstart(summary, "accept ")) {
     is_accept = 1;
@@ -2721,57 +2821,56 @@ parse_short_policy(const char *summary)
 
   n_entries = 0;
   for ( ; *summary; summary = next) {
-    const char *comma = strchr(summary, ',');
-    unsigned low, high;
-    char dummy;
-    char ent_buf[32];
-    size_t len;
-
-    next = comma ? comma+1 : strchr(summary, '\0');
-    len = comma ? (size_t)(comma - summary) : strlen(summary);
-
     if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) {
       log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s",
              escaped(orig_summary));
       return NULL;
     }
 
-    if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) {
-      /* unrecognized entry format. skip it. */
-      continue;
-    }
-    if (len < 1) {
-      /* empty; skip it. */
-      /* XXX This happens to be unreachable, since if len==0, then *summary is
-       * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */
-      continue;
+    unsigned low, high;
+    int ok;
+    low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next);
+    if (!ok) {
+      if (! TOR_ISDIGIT(*summary) || *summary == ',') {
+        /* Unrecognized format: skip it. */
+        goto skip_ent;
+      } else {
+        goto bad_ent;
+      }
     }
 
-    memcpy(ent_buf, summary, len);
-    ent_buf[len] = '\0';
+    switch (*next) {
+      case ',':
+        ++next;
+        FALLTHROUGH;
+      case '\0':
+        high = low;
+        break;
+      case '-':
+        high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next);
+        if (!ok)
+          goto bad_ent;
 
-    if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) {
-      if (low<1 || low>65535 || high<1 || high>65535 || low>high) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-    } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) {
-      if (low<1 || low>65535) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-      high = low;
-    } else {
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
-             escaped(orig_summary));
-      return NULL;
+        if (*next == ',')
+          ++next;
+        else if (*next != '\0')
+          goto bad_ent;
+
+        break;
+      default:
+        goto bad_ent;
     }
 
     entries[n_entries].min_port = low;
     entries[n_entries].max_port = high;
     n_entries++;
+
+    continue;
+  skip_ent:
+    next = strchr(next, ',');
+    if (!next)
+      break;
+    ++next;
   }
 
   if (n_entries == 0) {
@@ -2792,6 +2891,11 @@ parse_short_policy(const char *summary)
   result->n_entries = n_entries;
   memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries);
   return result;
+
+ bad_ent:
+  log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
+         escaped(orig_summary));
+  return NULL;
 }
 
 /** Write <b>policy</b> back out into a string. */
diff --git a/src/core/or/policies.h b/src/core/or/policies.h
index 2c38de362f..3c46363c04 100644
--- a/src/core/or/policies.h
+++ b/src/core/or/policies.h
@@ -70,6 +70,7 @@ typedef struct short_policy_t {
 int firewall_is_fascist_or(void);
 int firewall_is_fascist_dir(void);
 int fascist_firewall_use_ipv6(const or_options_t *options);
+MOCK_DECL(int, fascist_firewall_rand_prefer_ipv6_addr, (void));
 int fascist_firewall_prefer_ipv6_orport(const or_options_t *options);
 int fascist_firewall_prefer_ipv6_dirport(const or_options_t *options);
 
@@ -91,6 +92,8 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds,
 void fascist_firewall_choose_address_rs(const routerstatus_t *rs,
                                         firewall_connection_t fw_connection,
                                         int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                        int pref_only, tor_addr_port_t* ap);
 void fascist_firewall_choose_address_node(const node_t *node,
                                           firewall_connection_t fw_connection,
                                           int pref_only, tor_addr_port_t* ap);
diff --git a/src/core/or/port_cfg_st.h b/src/core/or/port_cfg_st.h
index b67091ce32..e9e82bb1de 100644
--- a/src/core/or/port_cfg_st.h
+++ b/src/core/or/port_cfg_st.h
@@ -31,5 +31,5 @@ struct port_cfg_t {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
+#endif /* !defined(PORT_CFG_ST_H) */
 
diff --git a/src/core/or/protover.c b/src/core/or/protover.c
index 17979d04ea..905c5e9ed3 100644
--- a/src/core/or/protover.c
+++ b/src/core/or/protover.c
@@ -39,6 +39,9 @@ static int protocol_list_contains(const smartlist_t *protos,
 static const struct {
   protocol_type_t protover_type;
   const char *name;
+/* If you add a new protocol here, you probably also want to add
+ * parsing for it in routerstatus_parse_entry_from_string() so that
+ * it is set in routerstatus_t */
 } PROTOCOL_NAMES[] = {
   { PRT_LINK, "Link" },
   { PRT_LINKAUTH, "LinkAuth" },
@@ -49,7 +52,9 @@ static const struct {
   { PRT_HSREND, "HSRend" },
   { PRT_DESC, "Desc" },
   { PRT_MICRODESC, "Microdesc"},
-  { PRT_CONS, "Cons" }
+  { PRT_PADDING, "Padding"},
+  { PRT_CONS, "Cons" },
+  { PRT_FLOWCTRL, "FlowCtrl"},
 };
 
 #define N_PROTOCOL_NAMES ARRAY_LENGTH(PROTOCOL_NAMES)
@@ -387,7 +392,7 @@ protover_get_supported_protocols(void)
     "Desc=1-2 "
     "DirCache=1-2 "
     "HSDir=1-2 "
-    "HSIntro=3-4 "
+    "HSIntro=3-5 "
     "HSRend=1-2 "
     "Link=1-5 "
 #ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
@@ -396,7 +401,9 @@ protover_get_supported_protocols(void)
     "LinkAuth=3 "
 #endif
     "Microdesc=1-2 "
-    "Relay=1-2";
+    "Relay=1-2 "
+    "Padding=2 "
+    "FlowCtrl=1";
 }
 
 /** The protocols from protover_get_supported_protocols(), as parsed into a
@@ -815,6 +822,8 @@ protover_all_supported(const char *s, char **missing_out)
        * ones and, if so, add them to unsupported->ranges. */
       if (versions->low != 0 && versions->high != 0) {
         smartlist_add(unsupported->ranges, versions);
+      } else {
+        tor_free(versions);
       }
       /* Finally, if we had something unsupported, add it to the list of
        * missing_some things and mark that there was something missing. */
@@ -823,7 +832,6 @@ protover_all_supported(const char *s, char **missing_out)
         all_supported = 0;
       } else {
         proto_entry_free(unsupported);
-        tor_free(versions);
       }
     } SMARTLIST_FOREACH_END(range);
 
diff --git a/src/core/or/protover.h b/src/core/or/protover.h
index 7e181ba97a..af45a31aeb 100644
--- a/src/core/or/protover.h
+++ b/src/core/or/protover.h
@@ -28,21 +28,25 @@ struct smartlist_t;
 #define PROTOVER_HS_INTRO_V3 4
 /** The protover version number that signifies HSv3 rendezvous point support */
 #define PROTOVER_HS_RENDEZVOUS_POINT_V3 2
+/** The protover that signals support for HS circuit setup padding machines */
+#define PROTOVER_HS_SETUP_PADDING 2
 
 /** List of recognized subprotocols. */
 /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust`
 /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto`
 typedef enum protocol_type_t {
-  PRT_LINK,
-  PRT_LINKAUTH,
-  PRT_RELAY,
-  PRT_DIRCACHE,
-  PRT_HSDIR,
-  PRT_HSINTRO,
-  PRT_HSREND,
-  PRT_DESC,
-  PRT_MICRODESC,
-  PRT_CONS,
+  PRT_LINK      = 0,
+  PRT_LINKAUTH  = 1,
+  PRT_RELAY     = 2,
+  PRT_DIRCACHE  = 3,
+  PRT_HSDIR     = 4,
+  PRT_HSINTRO   = 5,
+  PRT_HSREND    = 6,
+  PRT_DESC      = 7,
+  PRT_MICRODESC = 8,
+  PRT_CONS      = 9,
+  PRT_PADDING   = 10,
+  PRT_FLOWCTRL  = 11,
 } protocol_type_t;
 
 bool protover_contains_long_protocol_names(const char *s);
diff --git a/src/core/or/relay.c b/src/core/or/relay.c
index f5fc1cfbb3..f83d2365e4 100644
--- a/src/core/or/relay.c
+++ b/src/core/or/relay.c
@@ -49,18 +49,19 @@
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "feature/client/circpathbias.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "lib/compress/compress.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircommon/directory.h"
@@ -80,7 +81,6 @@
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/routerlist.h"
 #include "core/or/scheduler.h"
-#include "feature/stats/rephist.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/cell_queue_st.h"
@@ -93,15 +93,12 @@
 #include "core/or/origin_circuit_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "core/or/socks_request_st.h"
-
-#include "lib/intmath/weakrng.h"
+#include "core/or/sendme.h"
 
 static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell,
                                             cell_direction_t cell_direction,
                                             crypt_path_t *layer_hint);
 
-static void circuit_consider_sending_sendme(circuit_t *circ,
-                                            crypt_path_t *layer_hint);
 static void circuit_resume_edge_reading(circuit_t *circ,
                                         crypt_path_t *layer_hint);
 static int circuit_resume_edge_reading_helper(edge_connection_t *conn,
@@ -134,9 +131,6 @@ uint64_t stats_n_relay_cells_delivered = 0;
  * reached (see append_cell_to_circuit_queue()) */
 uint64_t stats_n_circ_max_cell_reached = 0;
 
-/** Used to tell which stream to read from first on a circuit. */
-static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT;
-
 /**
  * Update channel usage state based on the type of relay cell and
  * circuit properties.
@@ -253,6 +247,10 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   if (recognized) {
     edge_connection_t *conn = NULL;
 
+    /* Recognized cell, the cell digest has been updated, we'll record it for
+     * the SENDME if need be. */
+    sendme_record_received_cell_digest(circ, layer_hint);
+
     if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
       if (pathbias_check_probe_response(circ, cell) == -1) {
         pathbias_count_valid_cells(circ, cell);
@@ -267,8 +265,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_OUT) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending away from origin.");
-      if ((reason=connection_edge_process_relay_cell(cell, circ, conn, NULL))
-          < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn, NULL);
+      if (reason < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "connection_edge_process_relay_cell (away from origin) "
                "failed.");
@@ -278,8 +276,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_IN) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending to origin.");
-      if ((reason = connection_edge_process_relay_cell(cell, circ, conn,
-                                                       layer_hint)) < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn,
+                                                  layer_hint);
+      if (reason < 0) {
         /* If a client is trying to connect to unknown hidden service port,
          * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit.
          * Do not log warn as this is an expected behavior for a service. */
@@ -293,7 +292,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     return 0;
   }
 
-  /* not recognized. pass it on. */
+  /* not recognized. inform circpad and pass it on. */
+  circpad_deliver_unrecognized_cell_events(circ, cell_direction);
+
   if (cell_direction == CELL_DIRECTION_OUT) {
     cell->circ_id = circ->n_circ_id; /* switch it */
     chan = circ->n_chan;
@@ -353,11 +354,11 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
  *  - Encrypt it to the right layer
  *  - Append it to the appropriate cell_queue on <b>circ</b>.
  */
-static int
-circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
+MOCK_IMPL(int,
+circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
                            cell_direction_t cell_direction,
                            crypt_path_t *layer_hint, streamid_t on_stream,
-                           const char *filename, int lineno)
+                           const char *filename, int lineno))
 {
   channel_t *chan; /* where to send the cell */
 
@@ -524,6 +525,8 @@ relay_command_to_string(uint8_t command)
     case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK";
     case RELAY_COMMAND_EXTEND2: return "EXTEND2";
     case RELAY_COMMAND_EXTENDED2: return "EXTENDED2";
+    case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE";
+    case RELAY_COMMAND_PADDING_NEGOTIATED: return "PADDING_NEGOTIATED";
     default:
       tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u",
                    (unsigned)command);
@@ -531,6 +534,64 @@ relay_command_to_string(uint8_t command)
   }
 }
 
+/** When padding a cell with randomness, leave this many zeros after the
+ * payload. */
+#define CELL_PADDING_GAP 4
+
+/** Return the offset where the padding should start. The <b>data_len</b> is
+ * the relay payload length expected to be put in the cell. It can not be
+ * bigger than RELAY_PAYLOAD_SIZE else this function assert().
+ *
+ * Value will always be smaller than CELL_PAYLOAD_SIZE because this offset is
+ * for the entire cell length not just the data payload length. Zero is
+ * returned if there is no room for padding.
+ *
+ * This function always skips the first 4 bytes after the payload because
+ * having some unused zero bytes has saved us a lot of times in the past. */
+
+STATIC size_t
+get_pad_cell_offset(size_t data_len)
+{
+  /* This is never supposed to happen but in case it does, stop right away
+   * because if tor is tricked somehow into not adding random bytes to the
+   * payload with this function returning 0 for a bad data_len, the entire
+   * authenticated SENDME design can be bypassed leading to bad denial of
+   * service attacks. */
+  tor_assert(data_len <= RELAY_PAYLOAD_SIZE);
+
+  /* If the offset is larger than the cell payload size, we return an offset
+   * of zero indicating that no padding needs to be added. */
+  size_t offset = RELAY_HEADER_SIZE + data_len + CELL_PADDING_GAP;
+  if (offset >= CELL_PAYLOAD_SIZE) {
+    return 0;
+  }
+  return offset;
+}
+
+/* Add random bytes to the unused portion of the payload, to foil attacks
+ * where the other side can predict all of the bytes in the payload and thus
+ * compute the authenticated SENDME cells without seeing the traffic. See
+ * proposal 289. */
+static void
+pad_cell_payload(uint8_t *cell_payload, size_t data_len)
+{
+  size_t pad_offset, pad_len;
+
+  tor_assert(cell_payload);
+
+  pad_offset = get_pad_cell_offset(data_len);
+  if (pad_offset == 0) {
+    /* We can't add padding so we are done. */
+    return;
+  }
+
+  /* Remember here that the cell_payload is the length of the header and
+   * payload size so we offset it using the full length of the cell. */
+  pad_len = CELL_PAYLOAD_SIZE - pad_offset;
+  crypto_fast_rng_getbytes(get_thread_fast_rng(),
+                           cell_payload + pad_offset, pad_len);
+}
+
 /** Make a relay cell out of <b>relay_command</b> and <b>payload</b>, and send
  * it onto the open circuit <b>circ</b>. <b>stream_id</b> is the ID on
  * <b>circ</b> for the stream that's sending the relay cell, or 0 if it's a
@@ -574,11 +635,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
   if (payload_len)
     memcpy(cell.payload+RELAY_HEADER_SIZE, payload, payload_len);
 
+  /* Add random padding to the cell if we can. */
+  pad_cell_payload(cell.payload, payload_len);
+
   log_debug(LD_OR,"delivering %d cell %s.", relay_command,
             cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward");
 
-  if (relay_command == RELAY_COMMAND_DROP)
-    rep_hist_padding_count_write(PADDING_TYPE_DROP);
+  /* Tell circpad we're sending a relay cell */
+  circpad_deliver_sent_relay_cell_events(circ, relay_command);
 
   /* If we are sending an END cell and this circuit is used for a tunneled
    * directory request, advance its state. */
@@ -602,7 +666,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
        * one of them.  Don't worry about the conn protocol version:
        * append_cell_to_circuit_queue will fix it up. */
       cell.command = CELL_RELAY_EARLY;
-      --origin_circ->remaining_relay_early_cells;
+      /* If we're out of relay early cells, tell circpad */
+      if (--origin_circ->remaining_relay_early_cells == 0)
+        circpad_machine_event_circ_has_no_relay_early(origin_circ);
       log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.",
                 (int)origin_circ->remaining_relay_early_cells);
       /* Memorize the command that is sent as RELAY_EARLY cell; helps debug
@@ -639,6 +705,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
     circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL);
     return -1;
   }
+
+  /* If applicable, note the cell digest for the SENDME version 1 purpose if
+   * we need to. This call needs to be after the circuit_package_relay_cell()
+   * because the cell digest is set within that function. */
+  if (relay_command == RELAY_COMMAND_DATA) {
+    sendme_record_cell_digest_on_circ(circ, cpath_layer);
+  }
+
   return 0;
 }
 
@@ -1428,85 +1502,108 @@ connection_edge_process_relay_cell_not_open(
 //  return -1;
 }
 
-/** An incoming relay cell has arrived on circuit <b>circ</b>. If
- * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
- * destined for <b>conn</b>.
+/** Process a SENDME cell that arrived on <b>circ</b>. If it is a stream level
+ * cell, it is destined for the given <b>conn</b>. If it is a circuit level
+ * cell, it is destined for the <b>layer_hint</b>. The <b>domain</b> is the
+ * logging domain that should be used.
  *
- * If <b>layer_hint</b> is defined, then we're the origin of the
- * circuit, and it specifies the hop that packaged <b>cell</b>.
- *
- * Return -reason if you want to warn and tear down the circuit, else 0.
- */
-STATIC int
-connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
-                                   edge_connection_t *conn,
-                                   crypt_path_t *layer_hint)
+ * Return 0 if everything went well or a negative value representing a circuit
+ * end reason on error for which the caller is responsible for closing it. */
+static int
+process_sendme_cell(const relay_header_t *rh, const cell_t *cell,
+                    circuit_t *circ, edge_connection_t *conn,
+                    crypt_path_t *layer_hint, int domain)
 {
-  static int num_seen=0;
-  relay_header_t rh;
-  unsigned domain = layer_hint?LD_APP:LD_EXIT;
-  int reason;
-  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
-                            * that's in the EXIT_CONN_STATE_RESOLVING
-                            * or EXIT_CONN_STATE_CONNECTING states. */
-
-  tor_assert(cell);
-  tor_assert(circ);
+  int ret;
 
-  relay_header_unpack(&rh, cell->payload);
-//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
-  num_seen++;
-  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
-            num_seen, rh.command, rh.stream_id);
+  tor_assert(rh);
 
-  if (rh.length > RELAY_PAYLOAD_SIZE) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Relay cell length field too long. Closing circuit.");
-    return - END_CIRC_REASON_TORPROTOCOL;
+  if (!rh->stream_id) {
+    /* Circuit level SENDME cell. */
+    ret = sendme_process_circuit_level(layer_hint, circ,
+                                       cell->payload + RELAY_HEADER_SIZE,
+                                       rh->length);
+    if (ret < 0) {
+      return ret;
+    }
+    /* Resume reading on any streams now that we've processed a valid
+     * SENDME cell that updated our package window. */
+    circuit_resume_edge_reading(circ, layer_hint);
+    /* We are done, the rest of the code is for the stream level. */
+    return 0;
   }
 
-  if (rh.stream_id == 0) {
-    switch (rh.command) {
-      case RELAY_COMMAND_BEGIN:
-      case RELAY_COMMAND_CONNECTED:
-      case RELAY_COMMAND_END:
-      case RELAY_COMMAND_RESOLVE:
-      case RELAY_COMMAND_RESOLVED:
-      case RELAY_COMMAND_BEGIN_DIR:
-        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
-               "stream_id. Dropping.", (int)rh.command);
-        return 0;
-      default:
-        ;
+  /* No connection, might be half edge state. We are done if so. */
+  if (!conn) {
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+      if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                               rh->stream_id)) {
+        circuit_read_valid_data(ocirc, rh->length);
+        log_info(domain, "Sendme cell on circ %u valid on half-closed "
+                         "stream id %d",
+                 ocirc->global_identifier, rh->stream_id);
+      }
     }
+
+    log_info(domain, "SENDME cell dropped, unknown stream (streamid %d).",
+             rh->stream_id);
+    return 0;
   }
 
-  /* either conn is NULL, in which case we've got a control cell, or else
-   * conn points to the recognized stream. */
+  /* Stream level SENDME cell. */
+  ret = sendme_process_stream_level(conn, circ, rh->length);
+  if (ret < 0) {
+    /* Means we need to close the circuit with reason ret. */
+    return ret;
+  }
 
-  if (conn && !connection_state_is_open(TO_CONN(conn))) {
-    if (conn->base_.type == CONN_TYPE_EXIT &&
-        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
-         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
-        rh.command == RELAY_COMMAND_DATA) {
-      /* Allow DATA cells to be delivered to an exit node in state
-       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
-       * This speeds up HTTP, for example. */
-      optimistic_data = 1;
-    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
-      log_warn(LD_BUG, "Somehow I had a connection that matched a "
-               "data cell with stream ID 0.");
-    } else {
-      return connection_edge_process_relay_cell_not_open(
-               &rh, cell, circ, conn, layer_hint);
-    }
+  /* We've now processed properly a SENDME cell, all windows have been
+   * properly updated, we'll read on the edge connection to see if we can
+   * get data out towards the end point (Exit or client) since we are now
+   * allowed to deliver more cells. */
+
+  if (circuit_queue_streams_are_blocked(circ)) {
+    /* Still waiting for queue to flush; don't touch conn */
+    return 0;
+  }
+  connection_start_reading(TO_CONN(conn));
+  /* handle whatever might still be on the inbuf */
+  if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
+    /* (We already sent an end cell if possible) */
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
   }
+  return 0;
+}
 
-  switch (rh.command) {
-    case RELAY_COMMAND_DROP:
-      rep_hist_padding_count_read(PADDING_TYPE_DROP);
-//      log_info(domain,"Got a relay-level padding cell. Dropping.");
-      return 0;
+/** A helper for connection_edge_process_relay_cell(): Actually handles the
+ *  cell that we received on the connection.
+ *
+ *  The arguments are the same as in the parent function
+ *  connection_edge_process_relay_cell(), plus the relay header <b>rh</b> as
+ *  unpacked by the parent function, and <b>optimistic_data</b> as set by the
+ *  parent function.
+ */
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data)
+{
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int reason;
+
+  tor_assert(rh);
+
+  /* First pass the cell to the circuit padding subsystem, in case it's a
+   * padding cell or circuit that should be handled there. */
+  if (circpad_check_received_cell(cell, circ, layer_hint, rh) == 0) {
+    log_debug(domain, "Cell handled as circuit padding");
+    return 0;
+  }
+
+  /* Now handle all the other commands */
+  switch (rh->command) {
     case RELAY_COMMAND_BEGIN:
     case RELAY_COMMAND_BEGIN_DIR:
       if (layer_hint &&
@@ -1527,7 +1624,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "Begin cell for known stream. Dropping.");
         return 0;
       }
-      if (rh.command == RELAY_COMMAND_BEGIN_DIR &&
+      if (rh->command == RELAY_COMMAND_BEGIN_DIR &&
           circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) {
         /* Assign this circuit and its app-ward OR connection a unique ID,
          * so that we can measure download times. The local edge and dir
@@ -1540,24 +1637,21 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       return connection_exit_begin_conn(cell, circ);
     case RELAY_COMMAND_DATA:
       ++stats_n_data_cells_received;
-      if (( layer_hint && --layer_hint->deliver_window < 0) ||
-          (!layer_hint && --circ->deliver_window < 0)) {
+
+      /* Update our circuit-level deliver window that we received a DATA cell.
+       * If the deliver window goes below 0, we end the circuit and stream due
+       * to a protocol failure. */
+      if (sendme_circuit_data_received(circ, layer_hint) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) circ deliver_window below 0. Killing.");
-        if (conn) {
-          /* XXXX Do we actually need to do this?  Will killing the circuit
-           * not send an END and mark the stream for close as appropriate? */
-          connection_edge_end(conn, END_STREAM_REASON_TORPROTOCOL);
-          connection_mark_for_close(TO_CONN(conn));
-        }
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
-      log_debug(domain,"circ deliver_window now %d.", layer_hint ?
-                layer_hint->deliver_window : circ->deliver_window);
 
-      circuit_consider_sending_sendme(circ, layer_hint);
+      /* Consider sending a circuit-level SENDME cell. */
+      sendme_circuit_consider_sending(circ, layer_hint);
 
-      if (rh.stream_id == 0) {
+      if (rh->stream_id == 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero "
                "stream_id. Dropping.");
         return 0;
@@ -1565,32 +1659,37 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (connection_half_edge_is_valid_data(ocirc->half_streams,
-                                                 rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
+                                                 rh->stream_id)) {
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "data cell on circ %u valid on half-closed "
-                     "stream id %d", ocirc->global_identifier, rh.stream_id);
+                     "stream id %d", ocirc->global_identifier, rh->stream_id);
           }
         }
 
         log_info(domain,"data cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
+                 rh->stream_id);
         return 0;
       }
 
-      if (--conn->deliver_window < 0) { /* is it below 0 after decrement? */
+      /* Update our stream-level deliver window that we just received a DATA
+       * cell. Going below 0 means we have a protocol level error so the
+       * stream and circuit are closed. */
+
+      if (sendme_stream_data_received(conn) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) conn deliver_window below 0. Killing.");
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
       /* Total all valid application bytes delivered */
-      if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      if (CIRCUIT_IS_ORIGIN(circ) && rh->length > 0) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
 
-      stats_n_data_bytes_received += rh.length;
+      stats_n_data_bytes_received += rh->length;
       connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE),
-                              rh.length, TO_CONN(conn));
+                              rh->length, TO_CONN(conn));
 
 #ifdef MEASUREMENTS_21206
       /* Count number of RELAY_DATA cells received on a linked directory
@@ -1606,25 +1705,25 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         /* Only send a SENDME if we're not getting optimistic data; otherwise
          * a SENDME could arrive before the CONNECTED.
          */
-        connection_edge_consider_sending_sendme(conn);
+        sendme_connection_edge_consider_sending(conn);
       }
 
       return 0;
     case RELAY_COMMAND_END:
-      reason = rh.length > 0 ?
+      reason = rh->length > 0 ?
         get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC;
       if (!conn) {
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (connection_half_edge_is_valid_end(ocirc->half_streams,
-                                                rh.stream_id)) {
+                                                rh->stream_id)) {
 
-            circuit_read_valid_data(ocirc, rh.length);
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "end cell (%s) on circ %u valid on half-closed "
                      "stream id %d",
                      stream_end_reason_to_string(reason),
-                     ocirc->global_identifier, rh.stream_id);
+                     ocirc->global_identifier, rh->stream_id);
             return 0;
           }
         }
@@ -1656,7 +1755,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
 
         /* Total all valid application bytes delivered */
         if (CIRCUIT_IS_ORIGIN(circ)) {
-          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
         }
       }
       return 0;
@@ -1664,7 +1763,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_EXTEND2: {
       static uint64_t total_n_extend=0, total_nonearly=0;
       total_n_extend++;
-      if (rh.stream_id) {
+      if (rh->stream_id) {
         log_fn(LOG_PROTOCOL_WARN, domain,
                "'extend' cell received for non-zero stream. Dropping.");
         return 0;
@@ -1705,9 +1804,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_debug(domain,"Got an extended cell! Yay.");
       {
         extended_cell_t extended_cell;
-        if (extended_cell_parse(&extended_cell, rh.command,
+        if (extended_cell_parse(&extended_cell, rh->command,
                         (const uint8_t*)cell->payload+RELAY_HEADER_SIZE,
-                        rh.length)<0) {
+                        rh->length)<0) {
           log_warn(LD_PROTOCOL,
                    "Can't parse EXTENDED cell; killing circuit.");
           return -END_CIRC_REASON_TORPROTOCOL;
@@ -1725,7 +1824,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       }
       /* Total all valid bytes delivered. */
       if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
       return 0;
     case RELAY_COMMAND_TRUNCATE:
@@ -1769,7 +1868,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
        * circuit is being torn down anyway, though.  */
       if (CIRCUIT_IS_ORIGIN(circ)) {
         circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
+                                rh->length);
       }
       circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
                         get_uint8(cell->payload + RELAY_HEADER_SIZE));
@@ -1784,11 +1883,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       if (CIRCUIT_IS_ORIGIN(circ)) {
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (connection_half_edge_is_valid_connected(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "connected cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1796,102 +1895,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_info(domain,
                "'connected' received on circid %u for streamid %d, "
                "no conn attached anymore. Ignoring.",
-               (unsigned)circ->n_circ_id, rh.stream_id);
+               (unsigned)circ->n_circ_id, rh->stream_id);
       return 0;
     case RELAY_COMMAND_SENDME:
-      if (!rh.stream_id) {
-        if (layer_hint) {
-          if (layer_hint->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from exit relay. "
-                   "Closing circ.");
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          layer_hint->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.",
-                    layer_hint->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-
-          /* We count circuit-level sendme's as valid delivered data because
-           * they are rate limited.
-           */
-          if (CIRCUIT_IS_ORIGIN(circ)) {
-            circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                    rh.length);
-          }
-
-        } else {
-          if (circ->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&client_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from client. "
-                   "Closing circ (window %d).",
-                   circ->package_window);
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          circ->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,
-                    "circ-level sendme at non-origin, packagewindow %d.",
-                    circ->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-        }
-        return 0;
-      }
-      if (!conn) {
-        if (CIRCUIT_IS_ORIGIN(circ)) {
-          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-          if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
-                                                   rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
-            log_info(domain,
-                    "sendme cell on circ %u valid on half-closed "
-                    "stream id %d", ocirc->global_identifier, rh.stream_id);
-          }
-        }
-
-        log_info(domain,"sendme cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
-        return 0;
-      }
-
-      /* Don't allow the other endpoint to request more than our maximum
-       * (i.e. initial) stream SENDME window worth of data. Well-behaved
-       * stock clients will not request more than this max (as per the check
-       * in the while loop of connection_edge_consider_sending_sendme()).
-       */
-      if (conn->package_window + STREAMWINDOW_INCREMENT >
-          STREAMWINDOW_START_MAX) {
-        static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
-        log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
-               "Unexpected stream sendme cell. Closing circ (window %d).",
-               conn->package_window);
-        return -END_CIRC_REASON_TORPROTOCOL;
-      }
-
-      /* At this point, the stream sendme is valid */
-      if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
-      }
-
-      conn->package_window += STREAMWINDOW_INCREMENT;
-      log_debug(domain,"stream-level sendme, packagewindow now %d.",
-                conn->package_window);
-      if (circuit_queue_streams_are_blocked(circ)) {
-        /* Still waiting for queue to flush; don't touch conn */
-        return 0;
-      }
-      connection_start_reading(TO_CONN(conn));
-      /* handle whatever might still be on the inbuf */
-      if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
-        /* (We already sent an end cell if possible) */
-        connection_mark_for_close(TO_CONN(conn));
-        return 0;
-      }
-      return 0;
+      return process_sendme_cell(rh, cell, circ, conn, layer_hint, domain);
     case RELAY_COMMAND_RESOLVE:
       if (layer_hint) {
         log_fn(LOG_PROTOCOL_WARN, LD_APP,
@@ -1919,11 +1926,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       if (CIRCUIT_IS_ORIGIN(circ)) {
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (connection_half_edge_is_valid_resolved(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "resolved cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1941,17 +1948,96 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_INTRO_ESTABLISHED:
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       rend_process_relay_cell(circ, layer_hint,
-                              rh.command, rh.length,
+                              rh->command, rh->length,
                               cell->payload+RELAY_HEADER_SIZE);
       return 0;
   }
   log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Received unknown relay command %d. Perhaps the other side is using "
          "a newer version of Tor? Dropping.",
-         rh.command);
+         rh->command);
   return 0; /* for forward compatibility, don't kill the circuit */
 }
 
+/** An incoming relay cell has arrived on circuit <b>circ</b>. If
+ * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
+ * destined for <b>conn</b>.
+ *
+ * If <b>layer_hint</b> is defined, then we're the origin of the
+ * circuit, and it specifies the hop that packaged <b>cell</b>.
+ *
+ * Return -reason if you want to warn and tear down the circuit, else 0.
+ */
+STATIC int
+connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
+                                   edge_connection_t *conn,
+                                   crypt_path_t *layer_hint)
+{
+  static int num_seen=0;
+  relay_header_t rh;
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
+                            * that's in the EXIT_CONN_STATE_RESOLVING
+                            * or EXIT_CONN_STATE_CONNECTING states. */
+
+  tor_assert(cell);
+  tor_assert(circ);
+
+  relay_header_unpack(&rh, cell->payload);
+//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
+  num_seen++;
+  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
+            num_seen, rh.command, rh.stream_id);
+
+  if (rh.length > RELAY_PAYLOAD_SIZE) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Relay cell length field too long. Closing circuit.");
+    return - END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  if (rh.stream_id == 0) {
+    switch (rh.command) {
+      case RELAY_COMMAND_BEGIN:
+      case RELAY_COMMAND_CONNECTED:
+      case RELAY_COMMAND_END:
+      case RELAY_COMMAND_RESOLVE:
+      case RELAY_COMMAND_RESOLVED:
+      case RELAY_COMMAND_BEGIN_DIR:
+        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
+               "stream_id. Dropping.", (int)rh.command);
+        return 0;
+      default:
+        ;
+    }
+  }
+
+  /* Tell circpad that we've received a recognized cell */
+  circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint);
+
+  /* either conn is NULL, in which case we've got a control cell, or else
+   * conn points to the recognized stream. */
+  if (conn && !connection_state_is_open(TO_CONN(conn))) {
+    if (conn->base_.type == CONN_TYPE_EXIT &&
+        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
+         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
+        rh.command == RELAY_COMMAND_DATA) {
+      /* Allow DATA cells to be delivered to an exit node in state
+       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
+       * This speeds up HTTP, for example. */
+      optimistic_data = 1;
+    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
+      log_warn(LD_BUG, "Somehow I had a connection that matched a "
+               "data cell with stream ID 0.");
+    } else {
+      return connection_edge_process_relay_cell_not_open(
+               &rh, cell, circ, conn, layer_hint);
+    }
+  }
+
+  return handle_relay_cell_command(cell, circ, conn, layer_hint,
+                              &rh, optimistic_data);
+}
+
 /** How many relay_data cells have we built, ever? */
 uint64_t stats_n_data_cells_packaged = 0;
 /** How many bytes of data have we put in relay_data cells have we built,
@@ -1965,6 +2051,84 @@ uint64_t stats_n_data_cells_received = 0;
  * ever received were completely full of data. */
 uint64_t stats_n_data_bytes_received = 0;
 
+/**
+ * Called when initializing a circuit, or when we have reached the end of the
+ * window in which we need to send some randomness so that incoming sendme
+ * cells will be unpredictable.  Resets the flags and picks a new window.
+ */
+void
+circuit_reset_sendme_randomness(circuit_t *circ)
+{
+  circ->have_sent_sufficiently_random_cell = 0;
+  circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 +
+    crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2);
+}
+
+/**
+ * Any relay data payload containing fewer than this many real bytes is
+ * considered to have enough randomness to.
+ **/
+#define RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES \
+  (RELAY_PAYLOAD_SIZE - CELL_PADDING_GAP - 16)
+
+/**
+ * Helper. Return the number of bytes that should be put into a cell from a
+ * given edge connection on which <b>n_available</b> bytes are available.
+ */
+STATIC size_t
+connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                           int package_partial,
+                                           circuit_t *on_circuit)
+{
+  if (!n_available)
+    return 0;
+
+  /* Do we need to force this payload to have space for randomness? */
+  const bool force_random_bytes =
+    (on_circuit->send_randomness_after_n_cells == 0) &&
+    (! on_circuit->have_sent_sufficiently_random_cell);
+
+  /* At most how much would we like to send in this cell? */
+  size_t target_length;
+  if (force_random_bytes) {
+    target_length = RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES;
+  } else {
+    target_length = RELAY_PAYLOAD_SIZE;
+  }
+
+  /* Decide how many bytes we will actually put into this cell. */
+  size_t package_length;
+  if (n_available >= target_length) { /* A full payload is available. */
+    package_length = target_length;
+  } else { /* not a full payload available */
+    if (package_partial)
+      package_length = n_available; /* just take whatever's available now */
+    else
+      return 0; /* nothing to do until we have a full payload */
+  }
+
+  /* If we reach this point, we will be definitely sending the cell. */
+  tor_assert_nonfatal(package_length > 0);
+
+  if (package_length <= RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES) {
+    /* This cell will have enough randomness in the padding to make a future
+     * sendme cell unpredictable. */
+    on_circuit->have_sent_sufficiently_random_cell = 1;
+  }
+
+  if (on_circuit->send_randomness_after_n_cells == 0) {
+    /* Either this cell, or some previous cell, had enough padding to
+     * ensure sendme unpredictability. */
+    tor_assert_nonfatal(on_circuit->have_sent_sufficiently_random_cell);
+    /* Pick a new interval in which we need to send randomness. */
+    circuit_reset_sendme_randomness(on_circuit);
+  }
+
+  --on_circuit->send_randomness_after_n_cells;
+
+  return package_length;
+}
+
 /** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or
  * <b>package_partial</b> is true), and the appropriate package windows aren't
  * empty, grab a cell and send it down the circuit.
@@ -2037,17 +2201,14 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     bytes_to_process = connection_get_inbuf_len(TO_CONN(conn));
   }
 
-  if (!bytes_to_process)
+  length = connection_edge_get_inbuf_bytes_to_package(bytes_to_process,
+                                                      package_partial, circ);
+  if (!length)
     return 0;
 
-  if (!package_partial && bytes_to_process < RELAY_PAYLOAD_SIZE)
-    return 0;
+  /* If we reach this point, we will definitely be packaging bytes into
+   * a cell. */
 
-  if (bytes_to_process > RELAY_PAYLOAD_SIZE) {
-    length = RELAY_PAYLOAD_SIZE;
-  } else {
-    length = bytes_to_process;
-  }
   stats_n_data_bytes_packaged += length;
   stats_n_data_cells_packaged += 1;
 
@@ -2082,15 +2243,17 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     return 0;
   }
 
-  if (!cpath_layer) { /* non-rendezvous exit */
-    tor_assert(circ->package_window > 0);
-    circ->package_window--;
-  } else { /* we're an AP, or an exit on a rendezvous circ */
-    tor_assert(cpath_layer->package_window > 0);
-    cpath_layer->package_window--;
+  /* Handle the circuit-level SENDME package window. */
+  if (sendme_note_circuit_data_packaged(circ, cpath_layer) < 0) {
+    /* Package window has gone under 0. Protocol issue. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Circuit package window is below 0. Closing circuit.");
+    conn->end_reason = END_STREAM_REASON_TORPROTOCOL;
+    return -1;
   }
 
-  if (--conn->package_window <= 0) { /* is it 0 after decrement? */
+  /* Handle the stream-level SENDME package window. */
+  if (sendme_note_stream_data_packaged(conn) < 0) {
     connection_stop_reading(TO_CONN(conn));
     log_debug(domain,"conn->package_window reached 0.");
     circuit_consider_stop_edge_reading(circ, cpath_layer);
@@ -2108,42 +2271,6 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
   goto repeat_connection_edge_package_raw_inbuf;
 }
 
-/** Called when we've just received a relay data cell, when
- * we've just finished flushing all bytes to stream <b>conn</b>,
- * or when we've flushed *some* bytes to the stream <b>conn</b>.
- *
- * If conn->outbuf is not too full, and our deliver window is
- * low, send back a suitable number of stream-level sendme cells.
- */
-void
-connection_edge_consider_sending_sendme(edge_connection_t *conn)
-{
-  circuit_t *circ;
-
-  if (connection_outbuf_too_full(TO_CONN(conn)))
-    return;
-
-  circ = circuit_get_by_edge_conn(conn);
-  if (!circ) {
-    /* this can legitimately happen if the destroy has already
-     * arrived and torn down the circuit */
-    log_info(LD_APP,"No circuit associated with conn. Skipping.");
-    return;
-  }
-
-  while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) {
-    log_debug(conn->base_.type == CONN_TYPE_AP ?LD_APP:LD_EXIT,
-              "Outbuf %d, Queuing stream sendme.",
-              (int)conn->base_.outbuf_flushlen);
-    conn->deliver_window += STREAMWINDOW_INCREMENT;
-    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
-                                     NULL, 0) < 0) {
-      log_warn(LD_APP,"connection_edge_send_command failed. Skipping.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The circuit <b>circ</b> has received a circuit-level sendme
  * (on hop <b>layer_hint</b>, if we're the OP). Go through all the
  * attached streams and let them resume reading and packaging, if
@@ -2166,12 +2293,6 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
                                        circ, layer_hint);
 }
 
-void
-stream_choice_seed_weak_rng(void)
-{
-  crypto_seed_weak_rng(&stream_choice_rng);
-}
-
 /** A helper function for circuit_resume_edge_reading() above.
  * The arguments are the same, except that <b>conn</b> is the head
  * of a linked list of edge streams that should each be considered.
@@ -2223,7 +2344,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn,
     int num_streams = 0;
     for (conn = first_conn; conn; conn = conn->next_stream) {
       num_streams++;
-      if (tor_weak_random_one_in_n(&stream_choice_rng, num_streams)) {
+
+      if (crypto_fast_rng_one_in_n(get_thread_fast_rng(), num_streams)) {
         chosen_stream = conn;
       }
       /* Invariant: chosen_stream has been chosen uniformly at random from
@@ -2365,33 +2487,6 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
   return 0;
 }
 
-/** Check if the deliver_window for circuit <b>circ</b> (at hop
- * <b>layer_hint</b> if it's defined) is low enough that we should
- * send a circuit-level sendme back down the circuit. If so, send
- * enough sendmes that the window would be overfull if we sent any
- * more.
- */
-static void
-circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint)
-{
-//  log_fn(LOG_INFO,"Considering: layer_hint is %s",
-//         layer_hint ? "defined" : "null");
-  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
-          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
-    log_debug(LD_CIRC,"Queuing circuit sendme.");
-    if (layer_hint)
-      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
-    else
-      circ->deliver_window += CIRCWINDOW_INCREMENT;
-    if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
-                                     NULL, 0, layer_hint) < 0) {
-      log_warn(LD_CIRC,
-               "relay_send_command_from_edge failed. Circuit's closed.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The total number of cells we have allocated. */
 static size_t total_cells_allocated = 0;
 
@@ -2766,7 +2861,7 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan,
 }
 
 /** Extract the command from a packed cell. */
-static uint8_t
+uint8_t
 packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids)
 {
   if (wide_circ_ids) {
diff --git a/src/core/or/relay.h b/src/core/or/relay.h
index 7cc3c43e43..99f7553013 100644
--- a/src/core/or/relay.h
+++ b/src/core/or/relay.h
@@ -42,6 +42,7 @@ int connection_edge_package_raw_inbuf(edge_connection_t *conn,
                                       int package_partial,
                                       int *max_cells);
 void connection_edge_consider_sending_sendme(edge_connection_t *conn);
+void circuit_reset_sendme_randomness(circuit_t *circ);
 
 extern uint64_t stats_n_data_cells_packaged;
 extern uint64_t stats_n_data_bytes_packaged;
@@ -78,6 +79,11 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue,
 void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out);
 MOCK_DECL(int, channel_flush_from_first_active_circuit,
           (channel_t *chan, int max));
+MOCK_DECL(int, circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno));
+
 void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
                              const char *file, int lineno);
 #define update_circuit_on_cmux(circ, direction) \
@@ -89,11 +95,15 @@ const uint8_t *decode_address_from_payload(tor_addr_t *addr_out,
                                         int payload_len);
 void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan);
 
-void stream_choice_seed_weak_rng(void);
-
 circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids);
+uint8_t packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids);
 
 #ifdef RELAY_PRIVATE
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data);
+
 STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
                          tor_addr_t *addr_out, int *ttl_out);
 /** An address-and-ttl tuple as yielded by resolved_cell_parse */
@@ -117,8 +127,11 @@ STATIC int cell_queues_check_size(void);
 STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                                    edge_connection_t *conn,
                                    crypt_path_t *layer_hint);
+STATIC size_t get_pad_cell_offset(size_t payload_len);
+STATIC size_t connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                                      int package_partial,
+                                                      circuit_t *on_circuit);
 
 #endif /* defined(RELAY_PRIVATE) */
 
 #endif /* !defined(TOR_RELAY_H) */
-
diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h
index dafce257c7..83bbd329a6 100644
--- a/src/core/or/relay_crypto_st.h
+++ b/src/core/or/relay_crypto_st.h
@@ -25,7 +25,9 @@ struct relay_crypto_t {
   /** Digest state for cells heading away from the OR at this step. */
   struct crypto_digest_t *b_digest;
 
+  /** Digest used for the next SENDME cell if any. */
+  uint8_t sendme_digest[DIGEST_LEN];
 };
 #undef crypto_cipher_t
 
-#endif
+#endif /* !defined(RELAY_CRYPTO_ST_H) */
diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c
index 9f1a27d501..0e3f0fe815 100644
--- a/src/core/or/scheduler.c
+++ b/src/core/or/scheduler.c
@@ -9,7 +9,7 @@
 #define SCHEDULER_KIST_PRIVATE
 #include "core/or/scheduler.h"
 #include "core/mainloop/mainloop.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #define TOR_CHANNEL_INTERNAL_
 #include "core/or/channeltls.h"
 #include "lib/evloop/compat_libevent.h"
@@ -267,7 +267,7 @@ select_scheduler(void)
           log_notice(LD_SCHED, "Scheduler type KIST has been disabled by "
                                "the consensus or no kernel support.");
         }
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
         log_info(LD_SCHED, "Scheduler type KIST not built in");
 #endif /* defined(HAVE_KIST_SUPPORT) */
         continue;
diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c
index 79ecb0bc7e..35b613cb8a 100644
--- a/src/core/or/scheduler_kist.c
+++ b/src/core/or/scheduler_kist.c
@@ -4,7 +4,7 @@
 #define SCHEDULER_KIST_PRIVATE
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/nodelist/networkstatus.h"
@@ -104,7 +104,7 @@ static unsigned int kist_lite_mode = 0;
  * changed and it doesn't recognized the values passed to the syscalls needed
  * by KIST. In that case, fallback to the naive approach. */
 static unsigned int kist_no_kernel_support = 0;
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 static unsigned int kist_lite_mode = 1;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -298,7 +298,7 @@ update_socket_info_impl, (socket_table_ent_t *ent))
   }
   return;
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
   goto fallback;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -724,7 +724,7 @@ kist_scheduler_run(void)
     SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
       scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
       if (!smartlist_contains(cp, readd_chan)) {
-        if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+        if (!SCHED_BUG(readd_chan->sched_heap_idx != -1, readd_chan)) {
           /* XXXX Note that the check above is in theory redundant with
            * the smartlist_contains check.  But let's make sure we're
            * not messing anything up, and leave them both for now. */
@@ -833,7 +833,7 @@ scheduler_can_use_kist(void)
   return run_interval > 0;
 }
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 
 int
 scheduler_can_use_kist(void)
diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c
new file mode 100644
index 0000000000..e61cf0c0f8
--- /dev/null
+++ b/src/core/or/sendme.c
@@ -0,0 +1,710 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.c
+ * \brief Code that is related to SENDME cells both in terms of
+ *        creating/parsing cells and handling the content.
+ */
+
+#define SENDME_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/mainloop/connection.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+#include "feature/nodelist/networkstatus.h"
+#include "lib/ctime/di_ops.h"
+#include "trunnel/sendme_cell.h"
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * used when emitting a SENDME cell. */
+STATIC int
+get_emit_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_emit_min_version",
+                                 SENDME_EMIT_MIN_VERSION_DEFAULT,
+                                 SENDME_EMIT_MIN_VERSION_MIN,
+                                 SENDME_EMIT_MIN_VERSION_MAX);
+}
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * accepted when receiving a SENDME cell. */
+STATIC int
+get_accept_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_accept_min_version",
+                                 SENDME_ACCEPT_MIN_VERSION_DEFAULT,
+                                 SENDME_ACCEPT_MIN_VERSION_MIN,
+                                 SENDME_ACCEPT_MIN_VERSION_MAX);
+}
+
+/* Pop the first cell digset on the given circuit from the SENDME last digests
+ * list. NULL is returned if the list is uninitialized or empty.
+ *
+ * The caller gets ownership of the returned digest thus is responsible for
+ * freeing the memory. */
+static uint8_t *
+pop_first_cell_digest(const circuit_t *circ)
+{
+  uint8_t *circ_digest;
+
+  tor_assert(circ);
+
+  if (circ->sendme_last_digests == NULL ||
+      smartlist_len(circ->sendme_last_digests) == 0) {
+    return NULL;
+  }
+
+  /* More cell digest than the SENDME window is never suppose to happen. The
+   * cell should have been rejected before reaching this point due to its
+   * package_window down to 0 leading to a circuit close. Scream loudly but
+   * still pop the element so we don't memory leak. */
+  tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <=
+                      CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT);
+
+  circ_digest = smartlist_get(circ->sendme_last_digests, 0);
+  smartlist_del_keeporder(circ->sendme_last_digests, 0);
+  return circ_digest;
+}
+
+/* Return true iff the given cell digest matches the first digest in the
+ * circuit sendme list. */
+static bool
+v1_digest_matches(const uint8_t *circ_digest, const uint8_t *cell_digest)
+{
+  tor_assert(circ_digest);
+  tor_assert(cell_digest);
+
+  /* Compare the digest with the one in the SENDME. This cell is invalid
+   * without a perfect match. */
+  if (tor_memneq(circ_digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "SENDME v1 cell digest do not match.");
+    return false;
+  }
+
+  /* Digests matches! */
+  return true;
+}
+
+/* Return true iff the given decoded SENDME version 1 cell is valid and
+ * matches the expected digest on the circuit.
+ *
+ * Validation is done by comparing the digest in the cell from the previous
+ * cell we saw which tells us that the other side has in fact seen that cell.
+ * See proposal 289 for more details. */
+static bool
+cell_v1_is_valid(const sendme_cell_t *cell, const uint8_t *circ_digest)
+{
+  tor_assert(cell);
+  tor_assert(circ_digest);
+
+  const uint8_t *cell_digest = sendme_cell_getconstarray_data_v1_digest(cell);
+  return v1_digest_matches(circ_digest, cell_digest);
+}
+
+/* Return true iff the given cell version can be handled or if the minimum
+ * accepted version from the consensus is known to us. */
+STATIC bool
+cell_version_can_be_handled(uint8_t cell_version)
+{
+  int accept_version = get_accept_min_version();
+
+  /* We will first check if the consensus minimum accepted version can be
+   * handled by us and if not, regardless of the cell version we got, we can't
+   * continue. */
+  if (accept_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Unable to accept SENDME version %u (from consensus). "
+           "We only support <= %u. Probably your tor is too old?",
+           accept_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  /* Then, is this version below the accepted version from the consensus? If
+   * yes, we must not handle it. */
+  if (cell_version < accept_version) {
+    log_info(LD_PROTOCOL, "Unacceptable SENDME version %u. Only "
+                          "accepting %u (from consensus). Closing circuit.",
+             cell_version, accept_version);
+    goto invalid;
+  }
+
+  /* Is this cell version supported by us? */
+  if (cell_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_info(LD_PROTOCOL, "SENDME cell version %u is not supported by us. "
+                          "We only support <= %u",
+             cell_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  return true;
+ invalid:
+  return false;
+}
+
+/* Return true iff the encoded SENDME cell in cell_payload of length
+ * cell_payload_len is valid. For each version:
+ *
+ *  0: No validation
+ *  1: Authenticated with last cell digest.
+ *
+ * This is the main critical function to make sure we can continue to
+ * send/recv cells on a circuit. If the SENDME is invalid, the circuit should
+ * be marked for close by the caller. */
+STATIC bool
+sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload,
+                size_t cell_payload_len)
+{
+  uint8_t cell_version;
+  uint8_t *circ_digest = NULL;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* An empty payload means version 0 so skip trunnel parsing. We won't be
+   * able to parse a 0 length buffer into a valid SENDME cell. */
+  if (cell_payload_len == 0) {
+    cell_version = 0;
+  } else {
+    /* First we'll decode the cell so we can get the version. */
+    if (sendme_cell_parse(&cell, cell_payload, cell_payload_len) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unparseable SENDME cell received. Closing circuit.");
+      goto invalid;
+    }
+    cell_version = sendme_cell_get_version(cell);
+  }
+
+  /* Validate that we can handle this cell version. */
+  if (!cell_version_can_be_handled(cell_version)) {
+    goto invalid;
+  }
+
+  /* Pop the first element that was added (FIFO). We do that regardless of the
+   * version so we don't accumulate on the circuit if v0 is used by the other
+   * end point. */
+  circ_digest = pop_first_cell_digest(circ);
+  if (circ_digest == NULL) {
+    /* We shouldn't have received a SENDME if we have no digests. Log at
+     * protocol warning because it can be tricked by sending many SENDMEs
+     * without prior data cell. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "We received a SENDME but we have no cell digests to match. "
+           "Closing circuit.");
+    goto invalid;
+  }
+
+  /* Validate depending on the version now. */
+  switch (cell_version) {
+  case 0x01:
+    if (!cell_v1_is_valid(cell, circ_digest)) {
+      goto invalid;
+    }
+    break;
+  case 0x00:
+    /* Version 0, there is no work to be done on the payload so it is
+     * necessarily valid if we pass the version validation. */
+    break;
+  default:
+    log_warn(LD_PROTOCOL, "Unknown SENDME cell version %d received.",
+             cell_version);
+    tor_assert_nonfatal_unreached();
+    break;
+  }
+
+  /* Valid cell. */
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return true;
+ invalid:
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return false;
+}
+
+/* Build and encode a version 1 SENDME cell into payload, which must be at
+ * least of RELAY_PAYLOAD_SIZE bytes, using the digest for the cell data.
+ *
+ * Return the size in bytes of the encoded cell in payload. A negative value
+ * is returned on encoding failure. */
+STATIC ssize_t
+build_cell_payload_v1(const uint8_t *cell_digest, uint8_t *payload)
+{
+  ssize_t len = -1;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(cell_digest);
+  tor_assert(payload);
+
+  cell = sendme_cell_new();
+
+  /* Building a payload for version 1. */
+  sendme_cell_set_version(cell, 0x01);
+  /* Set the data length field for v1. */
+  sendme_cell_set_data_len(cell, TRUNNEL_SENDME_V1_DIGEST_LEN);
+
+  /* Copy the digest into the data payload. */
+  memcpy(sendme_cell_getarray_data_v1_digest(cell), cell_digest,
+         sendme_cell_get_data_len(cell));
+
+  /* Finally, encode the cell into the payload. */
+  len = sendme_cell_encode(payload, RELAY_PAYLOAD_SIZE, cell);
+
+  sendme_cell_free(cell);
+  return len;
+}
+
+/* Send a circuit-level SENDME on the given circuit using the layer_hint if
+ * not NULL. The digest is only used for version 1.
+ *
+ * Return 0 on success else a negative value and the circuit will be closed
+ * because we failed to send the cell on it. */
+static int
+send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint,
+                          const uint8_t *cell_digest)
+{
+  uint8_t emit_version;
+  uint8_t payload[RELAY_PAYLOAD_SIZE];
+  ssize_t payload_len;
+
+  tor_assert(circ);
+  tor_assert(cell_digest);
+
+  emit_version = get_emit_min_version();
+  switch (emit_version) {
+  case 0x01:
+    payload_len = build_cell_payload_v1(cell_digest, payload);
+    if (BUG(payload_len < 0)) {
+      /* Unable to encode the cell, abort. We can recover from this by closing
+       * the circuit but in theory it should never happen. */
+      return -1;
+    }
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 1 cell.");
+    break;
+  case 0x00:
+    FALLTHROUGH;
+  default:
+    /* Unknown version, fallback to version 0 meaning no payload. */
+    payload_len = 0;
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 0 cell. "
+                           "Consensus emit version is %d", emit_version);
+    break;
+  }
+
+  if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
+                                   (char *) payload, payload_len,
+                                   layer_hint) < 0) {
+    log_warn(LD_CIRC,
+             "SENDME relay_send_command_from_edge failed. Circuit's closed.");
+    return -1; /* the circuit's closed, don't continue */
+  }
+  return 0;
+}
+
+/* Record the cell digest only if the next cell is expected to be a SENDME. */
+static void
+record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest)
+{
+  tor_assert(circ);
+  tor_assert(sendme_digest);
+
+  /* Add the digest to the last seen list in the circuit. */
+  if (circ->sendme_last_digests == NULL) {
+    circ->sendme_last_digests = smartlist_new();
+  }
+  smartlist_add(circ->sendme_last_digests,
+                tor_memdup(sendme_digest, DIGEST_LEN));
+}
+
+/*
+ * Public API
+ */
+
+/** Return true iff the next cell for the given cell window is expected to be
+ * a SENDME.
+ *
+ * We are able to know that because the package or deliver window value minus
+ * one cell (the possible SENDME cell) should be a multiple of the increment
+ * window value. */
+static bool
+circuit_sendme_cell_is_next(int window)
+{
+  /* At the start of the window, no SENDME will be expected. */
+  if (window == CIRCWINDOW_START) {
+    return false;
+  }
+
+  /* Are we at the limit of the increment and if not, we don't expect next
+   * cell is a SENDME.
+   *
+   * We test against the window minus 1 because when we are looking if the
+   * next cell is a SENDME, the window (either package or deliver) hasn't been
+   * decremented just yet so when this is called, we are currently processing
+   * the "window - 1" cell.
+   *
+   * This function is used when recording a cell digest and this is done quite
+   * low in the stack when decrypting or encrypting a cell. The window is only
+   * updated once the cell is actually put in the outbuf. */
+  if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) {
+    return false;
+  }
+
+  /* Next cell is expected to be a SENDME. */
+  return true;
+}
+
+/** Called when we've just received a relay data cell, when we've just
+ * finished flushing all bytes to stream <b>conn</b>, or when we've flushed
+ * *some* bytes to the stream <b>conn</b>.
+ *
+ * If conn->outbuf is not too full, and our deliver window is low, send back a
+ * suitable number of stream-level sendme cells.
+ */
+void
+sendme_connection_edge_consider_sending(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  int log_domain = TO_CONN(conn)->type == CONN_TYPE_AP ? LD_APP : LD_EXIT;
+
+  /* Don't send it if we still have data to deliver. */
+  if (connection_outbuf_too_full(TO_CONN(conn))) {
+    goto end;
+  }
+
+  if (circuit_get_by_edge_conn(conn) == NULL) {
+    /* This can legitimately happen if the destroy has already arrived and
+     * torn down the circuit. */
+    log_info(log_domain, "No circuit associated with edge connection. "
+                         "Skipping sending SENDME.");
+    goto end;
+  }
+
+  while (conn->deliver_window <=
+         (STREAMWINDOW_START - STREAMWINDOW_INCREMENT)) {
+    log_debug(log_domain, "Outbuf %" TOR_PRIuSZ ", queuing stream SENDME.",
+              TO_CONN(conn)->outbuf_flushlen);
+    conn->deliver_window += STREAMWINDOW_INCREMENT;
+    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
+                                     NULL, 0) < 0) {
+      log_warn(LD_BUG, "connection_edge_send_command failed while sending "
+                       "a SENDME. Circuit probably closed, skipping.");
+      goto end; /* The circuit's closed, don't continue */
+    }
+  }
+
+ end:
+  return;
+}
+
+/** Check if the deliver_window for circuit <b>circ</b> (at hop
+ * <b>layer_hint</b> if it's defined) is low enough that we should
+ * send a circuit-level sendme back down the circuit. If so, send
+ * enough sendmes that the window would be overfull if we sent any
+ * more.
+ */
+void
+sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  bool sent_one_sendme = false;
+  const uint8_t *digest;
+
+  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
+          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
+    log_debug(LD_CIRC,"Queuing circuit sendme.");
+    if (layer_hint) {
+      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = cpath_get_sendme_digest(layer_hint);
+    } else {
+      circ->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+    }
+    if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) {
+      return; /* The circuit's closed, don't continue */
+    }
+    /* Current implementation is not suppose to send multiple SENDME at once
+     * because this means we would use the same relay crypto digest for each
+     * SENDME leading to a mismatch on the other side and the circuit to
+     * collapse. Scream loudly if it ever happens so we can address it. */
+    tor_assert_nonfatal(!sent_one_sendme);
+    sent_one_sendme = true;
+  }
+}
+
+/* Process a circuit-level SENDME cell that we just received. The layer_hint,
+ * if not NULL, is the Exit hop of the connection which means that we are a
+ * client. In that case, circ must be an origin circuit. The cell_body_len is
+ * the length of the SENDME cell payload (excluding the header). The
+ * cell_payload is the payload.
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_circuit_level(crypt_path_t *layer_hint,
+                             circuit_t *circ, const uint8_t *cell_payload,
+                             uint16_t cell_payload_len)
+{
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* Validate the SENDME cell. Depending on the version, different validation
+   * can be done. An invalid SENDME requires us to close the circuit. */
+  if (!sendme_is_valid(circ, cell_payload, cell_payload_len)) {
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  /* If we are the origin of the circuit, we are the Client so we use the
+   * layer hint (the Exit hop) for the package window tracking. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* If we are the origin of the circuit, it is impossible to not have a
+     * cpath. Just in case, bug on it and close the circuit. */
+    if (BUG(layer_hint == NULL)) {
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    if ((layer_hint->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from exit relay. "
+                     "Closing circ.");
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    layer_hint->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.",
+              layer_hint->package_window);
+
+    /* We count circuit-level sendme's as valid delivered data because they
+     * are rate limited. */
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len);
+  } else {
+    /* We aren't the origin of this circuit so we are the Exit and thus we
+     * track the package window with the circuit object. */
+    if ((circ->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&client_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from client. "
+                     "Closing circ (window %d).", circ->package_window);
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    circ->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_EXIT, "circ-level sendme at non-origin, packagewindow %d.",
+              circ->package_window);
+  }
+
+  return 0;
+}
+
+/* Process a stream-level SENDME cell that we just received. The conn is the
+ * edge connection (stream) that the circuit circ is associated with. The
+ * cell_body_len is the length of the payload (excluding the header).
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                            uint16_t cell_body_len)
+{
+  tor_assert(conn);
+  tor_assert(circ);
+
+  /* Don't allow the other endpoint to request more than our maximum (i.e.
+   * initial) stream SENDME window worth of data. Well-behaved stock clients
+   * will not request more than this max (as per the check in the while loop
+   * of sendme_connection_edge_consider_sending()). */
+  if ((conn->package_window + STREAMWINDOW_INCREMENT) >
+      STREAMWINDOW_START_MAX) {
+    static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
+    log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                   "Unexpected stream sendme cell. Closing circ (window %d).",
+                   conn->package_window);
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+  /* At this point, the stream sendme is valid */
+  conn->package_window += STREAMWINDOW_INCREMENT;
+
+  /* We count circuit-level sendme's as valid delivered data because they are
+   * rate limited. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_body_len);
+  }
+
+  log_debug(CIRCUIT_IS_ORIGIN(circ) ? LD_APP : LD_EXIT,
+            "stream-level sendme, package_window now %d.",
+            conn->package_window);
+  return 0;
+}
+
+/* Called when a relay DATA cell is received on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the deliver window and return its new value. */
+int
+sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int deliver_window, domain;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    tor_assert(layer_hint);
+    --layer_hint->deliver_window;
+    deliver_window = layer_hint->deliver_window;
+    domain = LD_APP;
+  } else {
+    tor_assert(!layer_hint);
+    --circ->deliver_window;
+    deliver_window = circ->deliver_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit deliver_window now %d.", deliver_window);
+  return deliver_window;
+}
+
+/* Called when a relay DATA cell is received for the given edge connection
+ * conn. Update the deliver window and return its new value. */
+int
+sendme_stream_data_received(edge_connection_t *conn)
+{
+  tor_assert(conn);
+  return --conn->deliver_window;
+}
+
+/* Called when a relay DATA cell is packaged on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the package window and return its new value. */
+int
+sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int package_window, domain;
+
+  tor_assert(circ);
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* Client side. */
+    tor_assert(layer_hint);
+    --layer_hint->package_window;
+    package_window = layer_hint->package_window;
+    domain = LD_APP;
+  } else {
+    /* Exit side. */
+    tor_assert(!layer_hint);
+    --circ->package_window;
+    package_window = circ->package_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit package_window now %d.", package_window);
+  return package_window;
+}
+
+/* Called when a relay DATA cell is packaged for the given edge connection
+ * conn. Update the package window and return its new value. */
+int
+sendme_note_stream_data_packaged(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  --conn->package_window;
+  log_debug(LD_APP, "Stream package_window now %d.", conn->package_window);
+  return conn->package_window;
+}
+
+/* Record the cell digest into the circuit sendme digest list depending on
+ * which edge we are. The digest is recorded only if we expect the next cell
+ * that we will receive is a SENDME so we can match the digest. */
+void
+sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath)
+{
+  int package_window;
+  uint8_t *sendme_digest;
+
+  tor_assert(circ);
+
+  package_window = circ->package_window;
+  if (cpath) {
+    package_window = cpath->package_window;
+  }
+
+  /* Is this the last cell before a SENDME? The idea is that if the
+   * package_window reaches a multiple of the increment, after this cell, we
+   * should expect a SENDME. */
+  if (!circuit_sendme_cell_is_next(package_window)) {
+    return;
+  }
+
+  /* Getting the digest is expensive so we only do it once we are certain to
+   * record it on the circuit. */
+  if (cpath) {
+    sendme_digest = cpath_get_sendme_digest(cpath);
+  } else {
+    sendme_digest =
+      relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+  }
+
+  record_cell_digest_on_circ(circ, sendme_digest);
+}
+
+/* Called once we decrypted a cell and recognized it. Record the cell digest
+ * as the next sendme digest only if the next cell we'll send on the circuit
+ * is expected to be a SENDME. */
+void
+sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window :
+                                           circ->deliver_window)) {
+    return;
+  }
+
+  if (cpath) {
+    /* Record incoming digest. */
+    cpath_sendme_record_cell_digest(cpath, false);
+  } else {
+    /* Record foward digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, true);
+  }
+}
+
+/* Called once we encrypted a cell. Record the cell digest as the next sendme
+ * digest only if the next cell we expect to receive is a SENDME so we can
+ * match the digests. */
+void
+sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window :
+                                           circ->package_window)) {
+    goto end;
+  }
+
+  if (cpath) {
+    /* Record the forward digest. */
+    cpath_sendme_record_cell_digest(cpath, true);
+  } else {
+    /* Record the incoming digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, false);
+  }
+
+ end:
+  return;
+}
diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h
new file mode 100644
index 0000000000..9d757ee435
--- /dev/null
+++ b/src/core/or/sendme.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.h
+ * \brief Header file for sendme.c.
+ **/
+
+#ifndef TOR_SENDME_H
+#define TOR_SENDME_H
+
+#include "core/or/edge_connection_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+
+/* Sending SENDME cell. */
+void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn);
+void sendme_circuit_consider_sending(circuit_t *circ,
+                                     crypt_path_t *layer_hint);
+
+/* Processing SENDME cell. */
+int sendme_process_circuit_level(crypt_path_t *layer_hint,
+                                 circuit_t *circ, const uint8_t *cell_payload,
+                                 uint16_t cell_payload_len);
+int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                                uint16_t cell_body_len);
+
+/* Update deliver window functions. */
+int sendme_stream_data_received(edge_connection_t *conn);
+int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint);
+
+/* Update package window functions. */
+int sendme_note_circuit_data_packaged(circuit_t *circ,
+                                      crypt_path_t *layer_hint);
+int sendme_note_stream_data_packaged(edge_connection_t *conn);
+
+/* Record cell digest on circuit. */
+void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath);
+/* Record cell digest as the SENDME digest. */
+void sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+void sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+
+/* Private section starts. */
+#ifdef SENDME_PRIVATE
+
+/* The maximum supported version. Above that value, the cell can't be
+ * recognized as a valid SENDME. */
+#define SENDME_MAX_SUPPORTED_VERSION 1
+
+/* The cell version constants for when emitting a cell. */
+#define SENDME_EMIT_MIN_VERSION_DEFAULT 1
+#define SENDME_EMIT_MIN_VERSION_MIN 0
+#define SENDME_EMIT_MIN_VERSION_MAX UINT8_MAX
+
+/* The cell version constants for when accepting a cell. */
+#define SENDME_ACCEPT_MIN_VERSION_DEFAULT 0
+#define SENDME_ACCEPT_MIN_VERSION_MIN 0
+#define SENDME_ACCEPT_MIN_VERSION_MAX UINT8_MAX
+
+/*
+ * Unit tests declaractions.
+ */
+#ifdef TOR_UNIT_TESTS
+
+STATIC int get_emit_min_version(void);
+STATIC int get_accept_min_version(void);
+
+STATIC bool cell_version_can_be_handled(uint8_t cell_version);
+
+STATIC ssize_t build_cell_payload_v1(const uint8_t *cell_digest,
+                                     uint8_t *payload);
+STATIC bool sendme_is_valid(const circuit_t *circ,
+                            const uint8_t *cell_payload,
+                            size_t cell_payload_len);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(SENDME_PRIVATE) */
+
+#endif /* !defined(TOR_SENDME_H) */
diff --git a/src/core/or/server_port_cfg_st.h b/src/core/or/server_port_cfg_st.h
index bd026af7ee..0738735c61 100644
--- a/src/core/or/server_port_cfg_st.h
+++ b/src/core/or/server_port_cfg_st.h
@@ -16,5 +16,5 @@ struct server_port_cfg_t {
   unsigned int bind_ipv6_only : 1;
 };
 
-#endif
+#endif /* !defined(SERVER_PORT_CFG_ST_H) */
 
diff --git a/src/core/or/socks_request_st.h b/src/core/or/socks_request_st.h
index 5922870c61..9fb941ff7e 100644
--- a/src/core/or/socks_request_st.h
+++ b/src/core/or/socks_request_st.h
@@ -74,4 +74,4 @@ struct socks_request_t {
   uint8_t socks5_atyp; /* SOCKS5 address type */
 };
 
-#endif
+#endif /* !defined(SOCKS_REQUEST_ST_H) */
diff --git a/src/core/or/tor_version_st.h b/src/core/or/tor_version_st.h
index 716429bd32..c5bdcaf07b 100644
--- a/src/core/or/tor_version_st.h
+++ b/src/core/or/tor_version_st.h
@@ -28,5 +28,5 @@ struct tor_version_t {
   char git_tag[DIGEST_LEN];
 };
 
-#endif
+#endif /* !defined(TOR_VERSION_ST_H) */
 
diff --git a/src/core/or/var_cell_st.h b/src/core/or/var_cell_st.h
index 4287c83f6d..607c0d6c83 100644
--- a/src/core/or/var_cell_st.h
+++ b/src/core/or/var_cell_st.h
@@ -19,5 +19,5 @@ struct var_cell_t {
   uint8_t payload[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
+#endif /* !defined(VAR_CELL_ST_H) */
 
diff --git a/src/core/or/versions.c b/src/core/or/versions.c
index 33273a5294..2c32b529f7 100644
--- a/src/core/or/versions.c
+++ b/src/core/or/versions.c
@@ -16,6 +16,25 @@
 
 #include "core/or/tor_version_st.h"
 
+/**
+ * Return the approximate date when this release came out, or was
+ * scheduled to come out, according to the APPROX_RELEASE_DATE set in
+ * configure.ac
+ **/
+time_t
+tor_get_approx_release_date(void)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  tor_snprintf(tbuf, sizeof(tbuf),
+               "%s 00:00:00", APPROX_RELEASE_DATE);
+  time_t result = 0;
+  int r = parse_iso_time(tbuf, &result);
+  if (BUG(r < 0)) {
+    result = 0;
+  }
+  return result;
+}
+
 /** Return VS_RECOMMENDED if <b>myversion</b> is contained in
  * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
  * entries. Else, return VS_OLD if every member of
@@ -377,6 +396,69 @@ sort_version_list(smartlist_t *versions, int remove_duplicates)
     smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
 }
 
+/** If there are more than this many entries, we're probably under
+ * some kind of weird DoS. */
+static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024;
+
+/**
+ * Map from protover string to protover_summary_flags_t.
+ */
+static strmap_t *protover_summary_map = NULL;
+
+/**
+ * Helper.  Given a non-NULL protover string <b>protocols</b>, set <b>out</b>
+ * to its summary, and memoize the result in <b>protover_summary_map</b>.
+ */
+static void
+memoize_protover_summary(protover_summary_flags_t *out,
+                         const char *protocols)
+{
+  if (!protover_summary_map)
+    protover_summary_map = strmap_new();
+
+  if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) {
+    protover_summary_cache_free_all();
+    tor_assert(protover_summary_map == NULL);
+    protover_summary_map = strmap_new();
+  }
+
+  const protover_summary_flags_t *cached =
+    strmap_get(protover_summary_map, protocols);
+
+  if (cached != NULL) {
+    /* We found a cached entry; no need to parse this one. */
+    memcpy(out, cached, sizeof(protover_summary_flags_t));
+    tor_assert(out->protocols_known);
+    return;
+  }
+
+  memset(out, 0, sizeof(*out));
+  out->protocols_known = 1;
+  out->supports_extend2_cells =
+    protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+  out->supports_ed25519_link_handshake_compat =
+    protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_link_handshake_any =
+    protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_hs_intro =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+  out->supports_v3_hsdir =
+    protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                    PROTOVER_HSDIR_V3);
+  out->supports_v3_rendezvous_point =
+    protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                    PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  out->supports_hs_setup_padding =
+    protocol_list_supports_protocol(protocols, PRT_PADDING,
+                                    PROTOVER_HS_SETUP_PADDING);
+  out->supports_establish_intro_dos_extension =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 5);
+
+  protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out));
+  cached = strmap_set(protover_summary_map, protocols, new_cached);
+  tor_assert(!cached);
+}
+
 /** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
  * falling back or correcting them based on <b>version</b> as appropriate.
  */
@@ -388,21 +470,7 @@ summarize_protover_flags(protover_summary_flags_t *out,
   tor_assert(out);
   memset(out, 0, sizeof(*out));
   if (protocols) {
-    out->protocols_known = 1;
-    out->supports_extend2_cells =
-      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
-    out->supports_ed25519_link_handshake_compat =
-      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_link_handshake_any =
-      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_hs_intro =
-      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
-    out->supports_v3_hsdir =
-      protocol_list_supports_protocol(protocols, PRT_HSDIR,
-                                      PROTOVER_HSDIR_V3);
-    out->supports_v3_rendezvous_point =
-      protocol_list_supports_protocol(protocols, PRT_HSREND,
-                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+    memoize_protover_summary(out, protocols);
   }
   if (version && !strcmpstart(version, "Tor ")) {
     if (!out->protocols_known) {
@@ -420,3 +488,13 @@ summarize_protover_flags(protover_summary_flags_t *out,
     }
   }
 }
+
+/**
+ * Free all space held in the protover_summary_map.
+ */
+void
+protover_summary_cache_free_all(void)
+{
+  strmap_free(protover_summary_map, tor_free_);
+  protover_summary_map = NULL;
+}
diff --git a/src/core/or/versions.h b/src/core/or/versions.h
index 22f3be176f..9aa7a0db87 100644
--- a/src/core/or/versions.h
+++ b/src/core/or/versions.h
@@ -26,6 +26,8 @@ typedef enum version_status_t {
   VS_UNKNOWN, /**< We have no idea. */
 } version_status_t;
 
+time_t tor_get_approx_release_date(void);
+
 version_status_t tor_version_is_obsolete(const char *myversion,
                                          const char *versionlist);
 int tor_version_parse_platform(const char *platform,
@@ -41,4 +43,6 @@ void summarize_protover_flags(protover_summary_flags_t *out,
                               const char *protocols,
                               const char *version);
 
+void protover_summary_cache_free_all(void);
+
 #endif /* !defined(TOR_VERSIONS_H) */
diff --git a/src/core/proto/.may_include b/src/core/proto/.may_include
new file mode 100644
index 0000000000..c1647a5cf9
--- /dev/null
+++ b/src/core/proto/.may_include
@@ -0,0 +1,10 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/buf/*.h
+
+trunnel/*.h
+
+core/proto/*.h
\ No newline at end of file
diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c
index 0442e2c6ee..697fed29e1 100644
--- a/src/core/proto/proto_cell.c
+++ b/src/core/proto/proto_cell.c
@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_cell.h"
 
 #include "core/or/connection_or.h"
diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c
index 21fa328f02..d741f28f09 100644
--- a/src/core/proto/proto_control0.c
+++ b/src/core/proto/proto_control0.c
@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_control0.h"
 
 /** Return 1 iff buf looks more like it has an (obsolete) v0 controller
diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c
index edbc51b10c..4213bc14dd 100644
--- a/src/core/proto/proto_ext_or.c
+++ b/src/core/proto/proto_ext_or.c
@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_ext_or.h"
 
diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c
index 5c86fc4979..88c59ef561 100644
--- a/src/core/proto/proto_http.c
+++ b/src/core/proto/proto_http.c
@@ -6,7 +6,7 @@
 
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 
 /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c
index c7bf13b9f4..c06e6b1a41 100644
--- a/src/core/proto/proto_socks.c
+++ b/src/core/proto/proto_socks.c
@@ -6,9 +6,9 @@
 
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/relay/ext_orport.h"
diff --git a/src/ext/.may_include b/src/ext/.may_include
new file mode 100644
index 0000000000..1eafff2eeb
--- /dev/null
+++ b/src/ext/.may_include
@@ -0,0 +1,10 @@
+
+orconfig.h
+
+lib/err/*.h
+lib/cc/*.h
+
+tinytest*.h
+ext/siphash.h
+ext/byteorder.h
+ext/tor_readpassphrase.h
\ No newline at end of file
diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c
index a65b6fcbe6..e0f5b2e5c9 100644
--- a/src/ext/csiphash.c
+++ b/src/ext/csiphash.c
@@ -30,12 +30,12 @@
 */
 
 #include "lib/cc/torint.h"
-#include "lib/log/util_bug.h"
+#include "lib/err/torerr.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 #include <string.h>
 #include <stdlib.h>
-#include "byteorder.h"
+#include "ext/byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
@@ -87,6 +87,13 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		v0 ^= mi;
 	}
 
+#ifdef __COVERITY__
+	{
+		uint64_t mi = 0;
+		memcpy(&mi, m+i, (src_sz-blocks));
+		last7 = _le64toh(mi) | (uint64_t)(src_sz & 0xff) << 56;
+	}
+#else
 	switch (src_sz - blocks) {
 		case 7: last7 |= (uint64_t)m[i + 6] << 48; FALLTHROUGH;
 		case 6: last7 |= (uint64_t)m[i + 5] << 40; FALLTHROUGH;
@@ -98,6 +105,7 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		case 0:
 		default:;
 	}
+#endif
 	v3 ^= last7;
 	DOUBLE_ROUND(v0,v1,v2,v3);
 	v0 ^= last7;
@@ -112,13 +120,13 @@ static int the_siphash_key_is_set = 0;
 static struct sipkey the_siphash_key;
 
 uint64_t siphash24g(const void *src, unsigned long src_sz) {
-	tor_assert(the_siphash_key_is_set);
+	raw_assert(the_siphash_key_is_set);
 	return siphash24(src, src_sz, &the_siphash_key);
 }
 
 void siphash_set_global_key(const struct sipkey *key)
 {
-	tor_assert(! the_siphash_key_is_set);
+	raw_assert(! the_siphash_key_is_set);
 	the_siphash_key.k0 = key->k0;
 	the_siphash_key.k1 = key->k1;
 	the_siphash_key_is_set = 1;
diff --git a/src/ext/include.am b/src/ext/include.am
index 6bdce2d79e..317e25d78e 100644
--- a/src/ext/include.am
+++ b/src/ext/include.am
@@ -143,6 +143,7 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS)
 LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a
 noinst_LIBRARIES += $(LIBED25519_DONNA)
 
+if BUILD_KECCAK_TINY
 src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\
   @CFLAGS_CONSTTIME@
 
@@ -156,6 +157,7 @@ noinst_HEADERS += $(LIBKECCAK_TINY_HDRS)
 
 LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a
 noinst_LIBRARIES += $(LIBKECCAK_TINY)
+endif
 
 EXTRA_DIST += \
 	src/ext/timeouts/bench/bench-add.lua 		\
diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c
index e0df05d7b7..16611af1e2 100644
--- a/src/ext/readpassphrase.c
+++ b/src/ext/readpassphrase.c
@@ -30,7 +30,7 @@
 #include <signal.h>
 #include <ctype.h>
 #include <fcntl.h>
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include
new file mode 100644
index 0000000000..92c7116555
--- /dev/null
+++ b/src/ext/timeouts/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+
+ext/tor_queue.h
+ext/timeouts/*.h
+ext/timeouts/timeout-bitops.c
diff --git a/src/ext/timeouts/test-timeout.c b/src/ext/timeouts/test-timeout.c
index 8077129376..52d2e31e0c 100644
--- a/src/ext/timeouts/test-timeout.c
+++ b/src/ext/timeouts/test-timeout.c
@@ -4,7 +4,7 @@
 #include <assert.h>
 #include <limits.h>
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #define THE_END_OF_TIME ((timeout_t)-1)
 
diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c
index d4b514d2c5..79fcc168ed 100644
--- a/src/ext/timeouts/timeout.c
+++ b/src/ext/timeouts/timeout.c
@@ -38,16 +38,16 @@
 
 #include <errno.h>     /* errno */
 
-#include "tor_queue.h" /* TAILQ(3) */
+#include "ext/tor_queue.h" /* TAILQ(3) */
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #ifndef TIMEOUT_DEBUG
 #define TIMEOUT_DEBUG 0
 #endif
 
 #if TIMEOUT_DEBUG - 0
-#include "timeout-debug.h"
+#include "ext/timeouts/timeout-debug.h"
 #endif
 
 #ifdef TIMEOUT_DISABLE_RELATIVE_ACCESS
@@ -141,7 +141,7 @@
 #define WHEEL_MASK (WHEEL_LEN - 1)
 #define TIMEOUT_MAX ((TIMEOUT_C(1) << (WHEEL_BIT * WHEEL_NUM)) - 1)
 
-#include "timeout-bitops.c"
+#include "ext/timeouts/timeout-bitops.c"
 
 #if WHEEL_BIT == 6
 #define ctz(n) ctz64(n)
@@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) {
 			timeout = MIN(_timeout, timeout);
 		}
 
-		relmask <<= WHEEL_BIT; 
+		relmask <<= WHEEL_BIT;
 		relmask |= WHEEL_MASK;
 	}
 
@@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) {
 TIMEOUT_PUBLIC int timeout_v_api(void) {
 	return TIMEOUT_V_API;
 } /* timeout_version() */
-
diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h
index b35874e153..1ed309fd08 100644
--- a/src/ext/timeouts/timeout.h
+++ b/src/ext/timeouts/timeout.h
@@ -31,7 +31,7 @@
 
 #include <inttypes.h>   /* PRIu64 PRIx64 PRIX64 uint64_t */
 
-#include "tor_queue.h"  /* TAILQ(3) */
+#include "ext/tor_queue.h"  /* TAILQ(3) */
 
 
 /*
@@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int);
 #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS
 TIMEOUT_PUBLIC bool timeout_pending(struct timeout *);
 /* true if on timing wheel, false otherwise */
- 
+
 TIMEOUT_PUBLIC bool timeout_expired(struct timeout *);
 /* true if on expired queue, false otherwise */
 
diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c
index 16f11e4639..239fdd0a38 100644
--- a/src/ext/tinytest.c
+++ b/src/ext/tinytest.c
@@ -492,6 +492,12 @@ tinytest_set_test_skipped_(void)
 		cur_test_outcome = SKIP;
 }
 
+int
+tinytest_cur_test_has_failed(void)
+{
+	return (cur_test_outcome == FAIL);
+}
+
 char *
 tinytest_format_hex_(const void *val_, unsigned long len)
 {
diff --git a/src/ext/tinytest.h b/src/ext/tinytest.h
index ed07b26bc0..05c2fda052 100644
--- a/src/ext/tinytest.h
+++ b/src/ext/tinytest.h
@@ -72,6 +72,9 @@ struct testlist_alias_t {
 };
 #define END_OF_ALIASES { NULL, NULL }
 
+/** Return true iff the current test has failed. */
+int tinytest_cur_test_has_failed(void);
+
 /** Implementation: called from a test to indicate failure, before logging. */
 void tinytest_set_test_failed_(void);
 /** Implementation: called from a test to indicate that we're skipping. */
diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h
index 15d1c8633e..52afa9ccd4 100644
--- a/src/ext/trunnel/trunnel-impl.h
+++ b/src/ext/trunnel/trunnel-impl.h
@@ -1,4 +1,4 @@
-/* trunnel-impl.h -- copied from Trunnel v1.5.2
+/* trunnel-impl.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c
index 3ae3fe02c8..01a55c5bec 100644
--- a/src/ext/trunnel/trunnel.c
+++ b/src/ext/trunnel/trunnel.c
@@ -1,4 +1,4 @@
-/* trunnel.c -- copied from Trunnel v1.5.2
+/* trunnel.c -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h
index 9b708437b8..87c75f4ec3 100644
--- a/src/ext/trunnel/trunnel.h
+++ b/src/ext/trunnel/trunnel.h
@@ -1,4 +1,4 @@
-/* trunnel.h -- copied from Trunnel v1.5.2
+/* trunnel.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c
index 697397d46b..e270c51ac9 100644
--- a/src/feature/api/tor_api.c
+++ b/src/feature/api/tor_api.c
@@ -40,10 +40,10 @@
 #define raw_socketpair tor_ersatz_socketpair
 #define raw_closesocket closesocket
 #define snprintf _snprintf
-#else
+#else /* !defined(_WIN32) */
 #define raw_socketpair socketpair
 #define raw_closesocket close
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h
index 2bf130c376..cb84853a52 100644
--- a/src/feature/api/tor_api.h
+++ b/src/feature/api/tor_api.h
@@ -55,7 +55,7 @@ typedef SOCKET tor_control_socket_t;
 #else
 typedef int tor_control_socket_t;
 #define INVALID_TOR_CONTROL_SOCKET (-1)
-#endif
+#endif /* defined(_WIN32) */
 
 /** DOCDOC */
 tor_control_socket_t tor_main_configuration_setup_control_socket(
diff --git a/src/feature/client/addressmap.c b/src/feature/client/addressmap.c
index bbe786a6a2..c5a27ce8c6 100644
--- a/src/feature/client/addressmap.c
+++ b/src/feature/client/addressmap.c
@@ -22,7 +22,7 @@
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/relay/dns.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
index 626c5efcae..f517876609 100644
--- a/src/feature/client/bridges.c
+++ b/src/feature/client/bridges.c
@@ -348,7 +348,7 @@ int
 node_is_a_configured_bridge(const node_t *node)
 {
   /* First, let's try searching for a bridge with matching identity. */
-  if (BUG(tor_digest_is_zero(node->identity)))
+  if (BUG(fast_mem_is_zero(node->identity, DIGEST_LEN)))
     return 0;
 
   if (find_bridge_by_digest(node->identity) != NULL)
@@ -844,7 +844,8 @@ rewrite_node_address_for_bridge(const bridge_info_t *bridge, node_t *node)
       }
     }
 
-    if (options->ClientPreferIPv6ORPort == -1) {
+    if (options->ClientPreferIPv6ORPort == -1 ||
+        options->ClientAutoIPv6ORPort == 0) {
       /* Mark which address to use based on which bridge_t we got. */
       node->ipv6_preferred = (tor_addr_family(&bridge->addr) == AF_INET6 &&
                               !tor_addr_is_null(&node->ri->ipv6_addr));
diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c
index 1743ab5a81..60a52664f1 100644
--- a/src/feature/client/circpathbias.c
+++ b/src/feature/client/circpathbias.c
@@ -176,6 +176,7 @@ pathbias_get_scale_threshold(const or_options_t *options)
 static double
 pathbias_get_scale_ratio(const or_options_t *options)
 {
+  (void) options;
   /*
    * The scale factor is the denominator for our scaling
    * of circuit counts for our path bias window.
@@ -185,7 +186,8 @@ pathbias_get_scale_ratio(const or_options_t *options)
    */
   int denominator = networkstatus_get_param(NULL, "pb_scalefactor",
                               2, 2, INT32_MAX);
-  (void) options;
+  tor_assert(denominator > 0);
+
   /**
    * The mult factor is the numerator for our scaling
    * of circuit counts for our path bias window. It
@@ -301,7 +303,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
   return circ->cpath &&
          circ->cpath->next != circ->cpath &&
          circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS;
-#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */
+#else /* !defined(N2N_TAGGING_IS_POSSIBLE) */
   /* If tagging attacks are no longer possible, we probably want to
    * count bias from the first hop. However, one could argue that
    * timing-based tagging is still more useful than per-hop failure.
@@ -369,8 +371,9 @@ pathbias_should_count(origin_circuit_t *circ)
         !circ->build_state->onehop_tunnel) {
       if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) {
         log_info(LD_BUG,
-               "One-hop circuit has length %d. Path state is %s. "
+               "One-hop circuit %d has length %d. Path state is %s. "
                "Circuit is a %s currently %s.%s",
+               circ->global_identifier,
                circ->build_state->desired_path_len,
                pathbias_state_to_string(circ->path_state),
                circuit_purpose_to_string(circ->base_.purpose),
@@ -398,12 +401,13 @@ pathbias_should_count(origin_circuit_t *circ)
   /* Check to see if the shouldcount result has changed due to a
    * unexpected purpose change that would affect our results */
   if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) {
-      log_info(LD_BUG,
-              "Circuit %d is now being counted despite being ignored "
-              "in the past. Purpose is %s, path state is %s",
-              circ->global_identifier,
-              circuit_purpose_to_string(circ->base_.purpose),
-              pathbias_state_to_string(circ->path_state));
+    log_info(LD_CIRC,
+            "Circuit %d is not being counted by pathbias because it was "
+            "ignored in the past. Purpose is %s, path state is %s",
+            circ->global_identifier,
+            circuit_purpose_to_string(circ->base_.purpose),
+            pathbias_state_to_string(circ->path_state));
+    return 0;
   }
   circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED;
 
@@ -434,8 +438,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                                      approx_time()))) {
         log_info(LD_BUG,
-                "Opened circuit is in strange path state %s. "
+                "Opened circuit %d is in strange path state %s. "
                 "Circuit is a %s currently %s.%s",
+                circ->global_identifier,
                 pathbias_state_to_string(circ->path_state),
                 circuit_purpose_to_string(circ->base_.purpose),
                 circuit_state_to_string(circ->base_.state),
@@ -468,8 +473,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
           if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                   approx_time()))) {
             log_info(LD_BUG,
-                   "Unopened circuit has strange path state %s. "
+                   "Unopened circuit %d has strange path state %s. "
                    "Circuit is a %s currently %s.%s",
+                   circ->global_identifier,
                    pathbias_state_to_string(circ->path_state),
                    circuit_purpose_to_string(circ->base_.purpose),
                    circuit_state_to_string(circ->base_.state),
@@ -538,8 +544,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
         if ((rate_msg = rate_limit_log(&success_notice_limit,
                 approx_time()))) {
           log_info(LD_BUG,
-              "Succeeded circuit is in strange path state %s. "
+              "Succeeded circuit %d is in strange path state %s. "
               "Circuit is a %s currently %s.%s",
+              circ->global_identifier,
               pathbias_state_to_string(circ->path_state),
               circuit_purpose_to_string(circ->base_.purpose),
               circuit_state_to_string(circ->base_.state),
@@ -574,8 +581,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&success_notice_limit,
               approx_time()))) {
         log_info(LD_BUG,
-            "Opened circuit is in strange path state %s. "
+            "Opened circuit %d is in strange path state %s. "
             "Circuit is a %s currently %s.%s",
+            circ->global_identifier,
             pathbias_state_to_string(circ->path_state),
             circuit_purpose_to_string(circ->base_.purpose),
             circuit_state_to_string(circ->base_.state),
@@ -601,8 +609,9 @@ pathbias_count_use_attempt(origin_circuit_t *circ)
 
   if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) {
     log_notice(LD_BUG,
-        "Used circuit is in strange path state %s. "
+        "Used circuit %d is in strange path state %s. "
         "Circuit is a %s currently %s.",
+        circ->global_identifier,
         pathbias_state_to_string(circ->path_state),
         circuit_purpose_to_string(circ->base_.purpose),
         circuit_state_to_string(circ->base_.state));
diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c
index cf593cfb68..f9e436e88f 100644
--- a/src/feature/client/dnsserv.c
+++ b/src/feature/client/dnsserv.c
@@ -26,8 +26,9 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/policies.h"
 
 #include "feature/control/control_connection_st.h"
@@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse,
   edge_connection_t *conn;
   char *q_name;
 
+  /* Launching a request for a user counts as user activity. */
+  note_user_activity(approx_time());
+
   /* Make a new dummy AP connection, and attach the request to it. */
   entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
   entry_conn->entry_cfg.dns_request = 1;
@@ -234,7 +238,7 @@ dnsserv_launch_request(const char *name, int reverse,
     TO_CONN(conn)->port = control_conn->base_.port;
     TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
   }
-#else /* !(defined(AF_UNIX)) */
+#else /* !defined(AF_UNIX) */
   TO_CONN(conn)->port = control_conn->base_.port;
   TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
 #endif /* defined(AF_UNIX) */
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
index 8d9230b66b..115d871843 100644
--- a/src/feature/client/entrynodes.c
+++ b/src/feature/client/entrynodes.c
@@ -114,7 +114,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "app/config/statefile.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
@@ -128,7 +128,7 @@
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c
index 0d1342c87b..3f731ac7d4 100644
--- a/src/feature/client/transports.c
+++ b/src/feature/client/transports.c
@@ -100,12 +100,14 @@
 #include "app/config/statefile.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
+#include "lib/process/process.h"
 #include "lib/process/env.h"
-#include "lib/process/subprocess.h"
 
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp);
 
 static inline int proxy_configuration_finished(const managed_proxy_t *mp);
@@ -127,6 +129,8 @@ static void parse_method_error(const char *line, int is_server_method);
 #define PROTO_SMETHODS_DONE "SMETHODS DONE"
 #define PROTO_PROXY_DONE "PROXY DONE"
 #define PROTO_PROXY_ERROR "PROXY-ERROR"
+#define PROTO_LOG "LOG"
+#define PROTO_STATUS "STATUS"
 
 /** The first and only supported - at the moment - configuration
     protocol version. */
@@ -490,8 +494,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
   tor_assert(mp->conf_state == PT_PROTO_COMPLETED);
 
   /* destroy the process handle and terminate the process. */
-  tor_process_handle_destroy(mp->process_handle, 1);
-  mp->process_handle = NULL;
+  process_set_data(mp->process, NULL);
+  process_terminate(mp->process);
 
   /* destroy all its registered transports, since we will no longer
      use them. */
@@ -520,34 +524,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
 static int
 launch_managed_proxy(managed_proxy_t *mp)
 {
-  int retval;
-
-  process_environment_t *env = create_managed_proxy_environment(mp);
-
-#ifdef _WIN32
-  /* Passing NULL as lpApplicationName makes Windows search for the .exe */
-  retval = tor_spawn_background(NULL,
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#else /* !(defined(_WIN32)) */
-  retval = tor_spawn_background(mp->argv[0],
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#endif /* defined(_WIN32) */
-
-  process_environment_free(env);
-
-  if (retval == PROCESS_STATUS_ERROR) {
-    log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.",
+  tor_assert(mp);
+
+  smartlist_t *env = create_managed_proxy_environment(mp);
+
+  /* Configure our process. */
+  process_set_data(mp->process, mp);
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback);
+  process_set_exit_callback(mp->process, managed_proxy_exit_callback);
+  process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE);
+  process_reset_environment(mp->process, env);
+
+  /* Cleanup our env. */
+  SMARTLIST_FOREACH(env, char *, x, tor_free(x));
+  smartlist_free(env);
+
+  /* Skip the argv[0] as we get that from process_new(argv[0]). */
+  for (int i = 1; mp->argv[i] != NULL; ++i)
+    process_append_argument(mp->process, mp->argv[i]);
+
+  if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.",
              mp->argv[0]);
     return -1;
   }
 
-  log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.",
-           mp->argv[0], tor_process_get_pid(mp->process_handle));
-
+  log_info(LD_CONFIG,
+           "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.",
+           mp->argv[0], process_get_pid(mp->process));
   mp->conf_state = PT_PROTO_LAUNCHED;
 
   return 0;
@@ -615,10 +620,6 @@ pt_configure_remaining_proxies(void)
 STATIC int
 configure_proxy(managed_proxy_t *mp)
 {
-  int configuration_finished = 0;
-  smartlist_t *proxy_output = NULL;
-  enum stream_status stream_status = 0;
-
   /* if we haven't launched the proxy yet, do it now */
   if (mp->conf_state == PT_PROTO_INFANT) {
     if (launch_managed_proxy(mp) < 0) { /* launch fail */
@@ -629,45 +630,8 @@ configure_proxy(managed_proxy_t *mp)
   }
 
   tor_assert(mp->conf_state != PT_PROTO_INFANT);
-  tor_assert(mp->process_handle);
-
-  proxy_output =
-    tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle),
-                              &stream_status);
-  if (!proxy_output) { /* failed to get input from proxy */
-    if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */
-      mp->conf_state = PT_PROTO_BROKEN;
-      log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' "
-               "is '%s'. Most probably the managed proxy stopped running. "
-               "This might be a bug of the managed proxy, a bug of Tor, or "
-               "a misconfiguration. Please enable logging on your managed "
-               "proxy and check the logs for errors.",
-               mp->argv[0], stream_status_to_string(stream_status));
-    }
-
-    goto done;
-  }
-
-  /* Handle lines. */
-  SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) {
-    handle_proxy_line(line, mp);
-    if (proxy_configuration_finished(mp))
-      goto done;
-  } SMARTLIST_FOREACH_END(line);
-
- done:
-  /* if the proxy finished configuring, exit the loop. */
-  if (proxy_configuration_finished(mp)) {
-    handle_finished_proxy(mp);
-    configuration_finished = 1;
-  }
-
-  if (proxy_output) {
-    SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp));
-    smartlist_free(proxy_output);
-  }
-
-  return configuration_finished;
+  tor_assert(mp->process);
+  return mp->conf_state == PT_PROTO_COMPLETED;
 }
 
 /** Register server managed proxy <b>mp</b> transports to state */
@@ -748,8 +712,14 @@ managed_proxy_destroy(managed_proxy_t *mp,
   /* free the outgoing proxy URI */
   tor_free(mp->proxy_uri);
 
-  tor_process_handle_destroy(mp->process_handle, also_terminate_process);
-  mp->process_handle = NULL;
+  /* do we want to terminate our process if it's still running? */
+  if (also_terminate_process && mp->process) {
+    /* Note that we do not call process_free(mp->process) here because we let
+     * the exit handler in managed_proxy_exit_callback() return `true` which
+     * makes the process subsystem deallocate the process_t. */
+    process_set_data(mp->process, NULL);
+    process_terminate(mp->process);
+  }
 
   tor_free(mp);
 }
@@ -945,21 +915,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp)
 
     parse_proxy_error(line);
     goto err;
-  } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) {
-    /* managed proxy launch failed: parse error message to learn why. */
-    int retval, child_state, saved_errno;
-    retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x",
-                        &child_state, &saved_errno);
-    if (retval == 2) {
-      log_warn(LD_GENERAL,
-               "Could not launch managed proxy executable at '%s' ('%s').",
-               mp->argv[0], strerror(saved_errno));
-    } else { /* failed to parse error message */
-      log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.",
-               mp->argv[0]);
-    }
 
-    mp->conf_state = PT_PROTO_FAILED_LAUNCH;
+    /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS
+     * string to make sure we can later extend this big if/else-if table with
+     * something that begins with "LOG" without having to get the order right.
+     * */
+  } else if (!strcmpstart(line, PROTO_LOG " ")) {
+    parse_log_line(line, mp);
+    return;
+  } else if (!strcmpstart(line, PROTO_STATUS " ")) {
+    parse_status_line(line, mp);
     return;
   }
 
@@ -1182,6 +1147,121 @@ parse_proxy_error(const char *line)
            line+strlen(PROTO_PROXY_ERROR)+1);
 }
 
+/** Parses a LOG <b>line</b> and emit log events accordingly. */
+STATIC void
+parse_log_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *log_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_LOG) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_LOG);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_LOG) + 1;
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s",
+             mp->argv[0], data);
+    goto done;
+  }
+
+  const config_line_t *severity = config_line_find(values, "SEVERITY");
+  const config_line_t *message = config_line_find(values, "MESSAGE");
+
+  /* Check if we got a message. */
+  if (! message) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "MESSAGE: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Check if severity is there and whether it's valid. */
+  if (! severity) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "SEVERITY: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  int log_severity = managed_proxy_severity_parse(severity->value);
+
+  if (log_severity == -1) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an "
+                    "invalid severity level: %s",
+                    mp->argv[0], severity->value);
+    goto done;
+  }
+
+  tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s",
+          mp->argv[0], message->value);
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  log_message = kvline_encode(values, KV_QUOTED);
+
+  /* Emit control port event. */
+  control_event_pt_log(log_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(log_message);
+}
+
+/** Parses a STATUS <b>line</b> and emit control events accordingly. */
+STATIC void
+parse_status_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *status_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_STATUS) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_STATUS);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_STATUS) + 1;
+
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid "
+             "STATUS message: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* We check if we received the TRANSPORT parameter, which is the only
+   * *required* value. */
+  const config_line_t *type = config_line_find(values, "TRANSPORT");
+
+  if (! type) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without "
+                    "TRANSPORT: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  status_message = kvline_encode(values, KV_QUOTED);
+
+  /* We have checked that TRANSPORT is there, we can now emit the STATUS event
+   * via the control port. */
+  control_event_pt_status(status_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(status_message);
+}
+
 /** Return a newly allocated string that tor should place in
  * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server
  * manged proxy in <b>mp</b>. Return NULL if no such options are found. */
@@ -1257,7 +1337,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp)
 
 /** Return a newly allocated process_environment_t * for <b>mp</b>'s
  * process. */
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp)
 {
   const or_options_t *options = get_options();
@@ -1272,8 +1352,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
   /* The final environment to be passed to mp. */
   smartlist_t *merged_env_vars = get_current_process_environment_variables();
 
-  process_environment_t *env;
-
   {
     char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */
     smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp);
@@ -1346,11 +1424,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     } else {
       smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=");
     }
-
-    /* All new versions of tor will keep stdin open, so PTs can use it
-     * as a reliable termination detection mechanism.
-     */
-    smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
   } else {
     /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the
      * TOR_PT_PROXY line.
@@ -1361,19 +1434,19 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     }
   }
 
+  /* All new versions of tor will keep stdin open, so PTs can use it
+   * as a reliable termination detection mechanism.
+   */
+  smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
+
   SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) {
     set_environment_variable_in_smartlist(merged_env_vars, env_var,
                                           tor_free_, 1);
   } SMARTLIST_FOREACH_END(env_var);
 
-  env = process_environment_make(merged_env_vars);
-
   smartlist_free(envs);
 
-  SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x));
-  smartlist_free(merged_env_vars);
-
-  return env;
+  return merged_env_vars;
 }
 
 /** Create and return a new managed proxy for <b>transport</b> using
@@ -1392,6 +1465,7 @@ managed_proxy_create(const smartlist_t *with_transport_list,
   mp->argv = proxy_argv;
   mp->transports = smartlist_new();
   mp->proxy_uri = get_pt_proxy_uri();
+  mp->process = process_new(proxy_argv[0]);
 
   mp->transports_to_launch = smartlist_new();
   SMARTLIST_FOREACH(with_transport_list, const char *, transport,
@@ -1736,3 +1810,93 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape)
 
   return new_string;
 }
+
+/** Callback function that is called when our PT process have data on its
+ * stdout. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stdout_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (mp == NULL)
+    return;
+
+  handle_proxy_line(line, mp);
+
+  if (proxy_configuration_finished(mp))
+    handle_finished_proxy(mp);
+}
+
+/** Callback function that is called when our PT process have data on its
+ * stderr. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stderr_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (BUG(mp == NULL))
+    return;
+
+  log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line);
+}
+
+/** Callback function that is called when our PT process terminates. The
+ * process exit code can be found in <b>exit_code</b> and our process can be
+ * found in <b>process</b>. Returns true iff we want the process subsystem to
+ * free our process_t handle for us. */
+STATIC bool
+managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_warn(LD_PT,
+          "Pluggable Transport process terminated with status code %" PRIu64,
+          exit_code);
+
+  /* Returning true here means that the process subsystem will take care of
+   * calling process_free() on our process_t. */
+  return true;
+}
+
+/** Returns a valid integer log severity level from <b>severity</b> that
+ * is compatible with Tor's logging functions. Returns <b>-1</b> on
+ * error. */
+STATIC int
+managed_proxy_severity_parse(const char *severity)
+{
+  tor_assert(severity);
+
+  /* Slightly different than log.c's parse_log_level :-( */
+  if (! strcmp(severity, "debug"))
+    return LOG_DEBUG;
+
+  if (! strcmp(severity, "info"))
+    return LOG_INFO;
+
+  if (! strcmp(severity, "notice"))
+    return LOG_NOTICE;
+
+  if (! strcmp(severity, "warning"))
+    return LOG_WARN;
+
+  if (! strcmp(severity, "error"))
+    return LOG_ERR;
+
+  return -1;
+}
diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h
index e2fa45828f..900dd9288e 100644
--- a/src/feature/client/transports.h
+++ b/src/feature/client/transports.h
@@ -11,6 +11,8 @@
 #ifndef TOR_TRANSPORTS_H
 #define TOR_TRANSPORTS_H
 
+#include "lib/process/process.h"
+
 /** Represents a pluggable transport used by a bridge. */
 typedef struct transport_t {
   /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */
@@ -81,7 +83,7 @@ enum pt_proto_state {
   PT_PROTO_FAILED_LAUNCH /* failed while launching */
 };
 
-struct process_handle_t;
+struct process_t;
 
 /** Structure containing information of a managed proxy. */
 typedef struct {
@@ -94,10 +96,8 @@ typedef struct {
 
   int is_server; /* is it a server proxy? */
 
-  /* A pointer to the process handle of this managed proxy. */
-  struct process_handle_t *process_handle;
-
-  int pid; /* The Process ID this managed proxy is using. */
+  /* A pointer to the process of this managed proxy. */
+  struct process_t *process;
 
   /** Boolean: We are re-parsing our config, and we are going to
    * remove this managed proxy if we don't find it any transport
@@ -128,6 +128,8 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp);
 STATIC void parse_env_error(const char *line);
 STATIC void parse_proxy_error(const char *line);
 STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_log_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_status_line(const char *line, managed_proxy_t *mp);
 STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp);
 
 STATIC void managed_proxy_destroy(managed_proxy_t *mp,
@@ -142,6 +144,12 @@ STATIC char* get_pt_proxy_uri(void);
 
 STATIC void free_execve_args(char **arg);
 
+STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t);
+STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t);
+STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t);
+
+STATIC int managed_proxy_severity_parse(const char *);
+
 #endif /* defined(PT_PRIVATE) */
 
 #endif /* !defined(TOR_TRANSPORTS_H) */
diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c
new file mode 100644
index 0000000000..3ce97dc855
--- /dev/null
+++ b/src/feature/control/btrack.c
@@ -0,0 +1,64 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack.c
+ * \brief Bootstrap trackers
+ *
+ * Initializes and shuts down the specific bootstrap trackers.  These
+ * trackers help the reporting of bootstrap progress by maintaining
+ * state information about various subsystems within tor.  When the
+ * correct state changes happen, these trackers emit controller
+ * events.
+ *
+ * These trackers avoid referring directly to the internals of state
+ * objects of other subsystems.
+ *
+ * btrack_circuit.c contains the tracker for origin circuits.
+ *
+ * btrack_orconn.c contains the tracker for OR connections.
+ *
+ * Eventually there will be a tracker for directory downloads as well.
+ **/
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_sys.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+static int
+btrack_init(void)
+{
+  if (btrack_orconn_init())
+    return -1;
+
+  return 0;
+}
+
+static void
+btrack_fini(void)
+{
+  btrack_orconn_fini();
+  btrack_circ_fini();
+}
+
+static int
+btrack_add_pubsub(pubsub_connector_t *connector)
+{
+  if (btrack_orconn_add_pubsub(connector))
+    return -1;
+  if (btrack_circ_add_pubsub(connector))
+    return -1;
+
+  return 0;
+}
+
+const subsys_fns_t sys_btrack = {
+  .name = "btrack",
+  .supported = true,
+  .level = -30,
+  .initialize = btrack_init,
+  .shutdown = btrack_fini,
+  .add_pubsub = btrack_add_pubsub,
+};
diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c
new file mode 100644
index 0000000000..2980c77ddc
--- /dev/null
+++ b/src/feature/control/btrack_circuit.c
@@ -0,0 +1,166 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.c
+ * \brief Bootstrap tracker for origin circuits
+ *
+ * Track state changes of origin circuits, as published by the circuit
+ * subsystem.
+ **/
+
+#include "core/or/or.h"
+
+#include "core/or/ocirc_event.h"
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/control.h"
+#include "lib/log/log.h"
+
+/** Pair of a best origin circuit GID with its state or status */
+typedef struct btc_best_t {
+  uint32_t gid;
+  int val;
+} btc_best_t;
+
+/** GID and state of the best origin circuit we've seen so far */
+static btc_best_t best_any_state = { 0, -1 };
+/** GID and state of the best application circuit we've seen so far */
+static btc_best_t best_ap_state = { 0, -1 };
+/** GID and status of the best origin circuit we've seen so far */
+static btc_best_t best_any_evtype = { 0, -1 };
+/** GID and status of the best application circuit we've seen so far */
+static btc_best_t best_ap_evtype = { 0, -1 };
+
+/** Reset cached "best" values */
+static void
+btc_reset_bests(void)
+{
+  best_any_state.gid = best_ap_state.gid = 0;
+  best_any_state.val = best_ap_state.val = -1;
+  best_any_evtype.gid = best_ap_state.gid = 0;
+  best_any_evtype.val = best_ap_evtype.val = -1;
+}
+
+/** True if @a state is a "better" origin circuit state than @a best->val */
+static bool
+btc_state_better(int state, const btc_best_t *best)
+{
+  return state > best->val;
+}
+
+/**
+ * Definine an ordering on circuit status events
+ *
+ * The CIRC_EVENT_ constants aren't sorted in a useful order, so this
+ * array helps to decode them.  This approach depends on the statuses
+ * being nonnegative and dense.
+ **/
+static int circ_event_order[] = {
+  [CIRC_EVENT_FAILED] = -1,
+  [CIRC_EVENT_CLOSED] = -1,
+  [CIRC_EVENT_LAUNCHED] = 1,
+  [CIRC_EVENT_EXTENDED] = 2,
+  [CIRC_EVENT_BUILT] = 3,
+};
+#define N_CIRC_EVENT_ORDER \
+  (sizeof(circ_event_order) / sizeof(circ_event_order[0]))
+
+/** True if @a state is a "better" origin circuit event status than @a
+    best->val */
+static bool
+btc_evtype_better(int state, const btc_best_t *best)
+{
+  if (state < 0)
+    return false;
+  if (best->val < 0)
+    return true;
+
+  tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER);
+  tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER);
+  return circ_event_order[state] > circ_event_order[best->val];
+}
+
+static bool
+btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best,
+                 const char *type)
+{
+  if (btc_state_better(msg->state, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type,
+             best->val, msg->state, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->state;
+    return true;
+  }
+  return false;
+}
+
+static bool
+btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best,
+                  const char *type)
+{
+  if (btc_evtype_better(msg->evtype, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type,
+             best->val, msg->evtype, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->evtype;
+    return true;
+  }
+  return false;
+}
+
+DECLARE_SUBSCRIBE(ocirc_state, btc_state_rcvr);
+DECLARE_SUBSCRIBE(ocirc_cevent, btc_cevent_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, btc_chan_rcvr);
+
+static void
+btc_state_rcvr(const msg_t *msg, const ocirc_state_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d",
+            arg->gid, arg->state, arg->onehop);
+
+  btc_update_state(arg, &best_any_state, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_state(arg, &best_ap_state, "AP");
+}
+
+static void
+btc_cevent_rcvr(const msg_t *msg, const ocirc_cevent_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d",
+            arg->gid, arg->evtype, arg->reason, arg->onehop);
+
+  btc_update_evtype(arg, &best_any_evtype, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_evtype(arg, &best_ap_evtype, "AP");
+}
+
+static void
+btc_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d",
+            arg->gid, arg->chan, arg->onehop);
+}
+
+int
+btrack_circ_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_state))
+    return -1;
+  return 0;
+}
+
+void
+btrack_circ_fini(void)
+{
+  btc_reset_bests();
+}
diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h
new file mode 100644
index 0000000000..b326c22ccf
--- /dev/null
+++ b/src/feature/control/btrack_circuit.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.h
+ * \brief Header file for btrack_circuit.c
+ **/
+
+#ifndef TOR_BTRACK_CIRCUIT_H
+#define TOR_BTRACK_CIRCUIT_H
+
+#include "lib/pubsub/pubsub.h"
+
+int btrack_circ_init(void);
+void btrack_circ_fini(void);
+int btrack_circ_add_pubsub(pubsub_connector_t *);
+
+#endif /* !defined(TOR_BTRACK_CIRCUIT_H) */
diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c
new file mode 100644
index 0000000000..922b542a0c
--- /dev/null
+++ b/src/feature/control/btrack_orconn.c
@@ -0,0 +1,206 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.c
+ * \brief Bootstrap tracker for OR connections
+ *
+ * Track state changes of OR connections, as published by the
+ * connection subsystem.  Also track circuit launch events, because
+ * they're one of the few ways to discover the association between a
+ * channel (and OR connection) and a circuit.
+ *
+ * We track all OR connections that we receive events for, whether or
+ * not they're carrying origin circuits.  (An OR connection might
+ * carry origin circuits only after we first find out about that
+ * connection.)
+ *
+ * All origin ORCONN events update the "any" state variables, while
+ * only application ORCONN events update the "ap" state variables (and
+ * also update the "any") variables.
+ *
+ * We do this because we want to report the first increments of
+ * connection progress as the earliest bootstrap phases.  This results
+ * in a better user experience because failures here translate into
+ * zero or very small amounts of displayed progress, instead of
+ * progress stuck near completion.  The first connection to a relay
+ * might be a one-hop circuit for directory lookups, or it might be a
+ * connection for an application circuit because we already have
+ * enough directory info to build an application circuit.
+ *
+ * We call functions in btrack_orconn_cevent.c to generate the actual
+ * controller events, because some of the state decoding we need to do
+ * is complicated.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+#include "lib/pubsub/pubsub.h"
+
+DECLARE_SUBSCRIBE(orconn_state, bto_state_rcvr);
+DECLARE_SUBSCRIBE(orconn_status, bto_status_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, bto_chan_rcvr);
+
+/** Pair of a best ORCONN GID and with its state */
+typedef struct bto_best_t {
+  uint64_t gid;
+  int state;
+} bto_best_t;
+
+/** GID and state of the best ORCONN we've seen so far */
+static bto_best_t best_any = { 0, -1 };
+/** GID and state of the best application circuit ORCONN we've seen so far */
+static bto_best_t best_ap = { 0, -1 };
+
+/**
+ * Update a cached state of a best ORCONN progress we've seen so far.
+ *
+ * Return true if the new state is better than the old.
+ **/
+static bool
+bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type)
+{
+  if (bto->state < best->state)
+    return false;
+  /* Update even if we won't change best->state, because it's more
+   * recent information that a particular connection transitioned to
+   * that state. */
+  best->gid = bto->gid;
+  if (bto->state > best->state) {
+    log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type,
+             best->state, bto->state, bto->gid);
+    best->state = bto->state;
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Update cached states of best ORCONN progress we've seen
+ *
+ * Only update the application ORCONN state if we know it's carrying
+ * an application circuit.
+ **/
+static void
+bto_update_bests(const bt_orconn_t *bto)
+{
+  tor_assert(bto->is_orig);
+
+  if (bto_update_best(bto, &best_any, "ANY"))
+    bto_cevent_anyconn(bto);
+  if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP"))
+    bto_cevent_apconn(bto);
+}
+
+/** Reset cached "best" values */
+static void
+bto_reset_bests(void)
+{
+  best_any.gid = best_ap.gid = 0;
+  best_any.state = best_ap.state = -1;
+}
+
+/**
+ * Update cached states of ORCONNs from the incoming message.  This
+ * message comes from code in connection_or.c.
+ **/
+static void
+bto_state_rcvr(const msg_t *msg, const orconn_state_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(arg->gid, arg->chan);
+  log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64
+            " proxy_type=%d state=%d",
+            arg->gid, arg->chan, arg->proxy_type, arg->state);
+  bto->proxy_type = arg->proxy_type;
+  bto->state = arg->state;
+  if (bto->is_orig)
+    bto_update_bests(bto);
+}
+
+/**
+ * Delete a cached ORCONN state if we get an incoming message saying
+ * the ORCONN is failed or closed.  This message comes from code in
+ * control.c.
+ **/
+static void
+bto_status_rcvr(const msg_t *msg, const orconn_status_msg_t *arg)
+{
+  (void)msg;
+  switch (arg->status) {
+  case OR_CONN_EVENT_FAILED:
+  case OR_CONN_EVENT_CLOSED:
+    log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d",
+             arg->gid, arg->status, arg->reason);
+    return bto_delete(arg->gid);
+  default:
+    break;
+  }
+}
+
+/**
+ * Create or update a cached ORCONN state for a newly launched
+ * connection, including whether it's launched by an origin circuit
+ * and whether it's a one-hop circuit.
+ **/
+static void
+bto_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(0, arg->chan);
+  if (!bto->is_orig || (bto->is_onehop && !arg->onehop)) {
+    log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d",
+              arg->chan, arg->onehop);
+  }
+  bto->is_orig = true;
+  if (!arg->onehop)
+    bto->is_onehop = false;
+  bto_update_bests(bto);
+}
+
+/**
+ * Initialize the hash maps and subscribe to ORCONN and origin
+ * circuit events.
+ **/
+int
+btrack_orconn_init(void)
+{
+  bto_init_maps();
+
+  return 0;
+}
+
+int
+btrack_orconn_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_state))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_status))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  return 0;
+}
+
+/** Clear the hash maps and reset the "best" states */
+void
+btrack_orconn_fini(void)
+{
+  bto_clear_maps();
+  bto_reset_bests();
+  bto_cevent_reset();
+}
diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h
new file mode 100644
index 0000000000..07b1b755f3
--- /dev/null
+++ b/src/feature/control/btrack_orconn.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.h
+ * \brief Header file for btrack_orconn.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_H
+#define TOR_BTRACK_ORCONN_H
+
+#include "lib/pubsub/pubsub.h"
+
+#ifdef BTRACK_ORCONN_PRIVATE
+
+#include "ht.h"
+
+/**
+ * Structure for tracking OR connection states
+ *
+ * This gets linked into two hash maps: one with connection IDs, and
+ * another with channel IDs.
+ **/
+typedef struct bt_orconn_t {
+  HT_ENTRY(bt_orconn_t) node;   /**< Hash map entry indexed by gid */
+  HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */
+  uint64_t gid;                    /**< Global ID of this ORCONN */
+  uint64_t chan;                   /**< Channel ID, if known */
+  int proxy_type;                  /**< Proxy type */
+  uint8_t state;                   /**< State of this ORCONN */
+  bool is_orig;             /**< Does this carry an origin circuit? */
+  bool is_onehop;           /**< Is this for a one-hop circuit? */
+} bt_orconn_t;
+
+#endif /* defined(BTRACK_ORCONN_PRIVATE) */
+
+int btrack_orconn_init(void);
+int btrack_orconn_add_pubsub(pubsub_connector_t *);
+void btrack_orconn_fini(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_H) */
diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c
new file mode 100644
index 0000000000..a00eb042d5
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.c
@@ -0,0 +1,160 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.c
+ * \brief Emit bootstrap status events for OR connections
+ *
+ * We do some decoding of the raw OR_CONN_STATE_* values.  For
+ * example, OR_CONN_STATE_CONNECTING means the first TCP connect()
+ * completing, regardless of whether it's directly to a relay instead
+ * of a proxy or a PT.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/control_events.h"
+
+/**
+ * Have we completed our first OR connection?
+ *
+ * Block display of application circuit progress until we do, to avoid
+ * some misleading behavior of jumping to high progress.
+ **/
+static bool bto_first_orconn = false;
+
+/** Is the ORCONN using a pluggable transport? */
+static bool
+using_pt(const bt_orconn_t *bto)
+{
+  return bto->proxy_type == PROXY_PLUGGABLE;
+}
+
+/** Is the ORCONN using a non-PT proxy? */
+static bool
+using_proxy(const bt_orconn_t *bto)
+{
+  switch (bto->proxy_type) {
+  case PROXY_CONNECT:
+  case PROXY_SOCKS4:
+  case PROXY_SOCKS5:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_anyconn(const bt_orconn_t *bto)
+{
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0);
+    /* Unblock directory progress display */
+    control_event_boot_first_orconn();
+    /* Unblock apconn progress display */
+    bto_first_orconn = true;
+    break;
+  default:
+    break;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any application circuit OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_apconn(const bt_orconn_t *bto)
+{
+  if (!bto_first_orconn)
+    return;
+
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0);
+    break;
+  default:
+    break;
+  }
+}
+
+/** Forget that we completed our first OR connection */
+void
+bto_cevent_reset(void)
+{
+  bto_first_orconn = false;
+}
diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h
new file mode 100644
index 0000000000..afec55581e
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.h
+ * \brief Header file for btrack_orconn_cevent.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_CEVENT_H
+#define TOR_BTRACK_ORCONN_CEVENT_H
+
+#include "feature/control/btrack_orconn.h"
+
+void bto_cevent_anyconn(const bt_orconn_t *);
+void bto_cevent_apconn(const bt_orconn_t *);
+void bto_cevent_reset(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_CEVENT_H) */
diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c
new file mode 100644
index 0000000000..e64bd3f0fe
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.c
@@ -0,0 +1,223 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.c
+ * \brief Hash map implementation for btrack_orconn.c
+ *
+ * These functions manipulate the hash maps that contain bt_orconn
+ * objects.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+
+static inline unsigned int
+bto_gid_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid));
+}
+
+static inline int
+bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->gid == b->gid;
+}
+
+static inline unsigned int
+bto_chan_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan));
+}
+
+static inline int
+bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->chan == b->chan;
+}
+
+HT_HEAD(bto_gid_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_)
+HT_GENERATE2(bto_gid_ht, bt_orconn_t, node,
+             bto_gid_hash_, bto_gid_eq_, 0.6,
+             tor_reallocarray_, tor_free_)
+static struct bto_gid_ht *bto_gid_map;
+
+HT_HEAD(bto_chan_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_, bto_chan_eq_)
+HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node,
+             bto_chan_hash_, bto_chan_eq_, 0.6,
+             tor_reallocarray_, tor_free_)
+static struct bto_chan_ht *bto_chan_map;
+
+/** Clear the GID hash map, freeing any bt_orconn_t objects that become
+ * unreferenced */
+static void
+bto_gid_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_gid_ht, bto_gid_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt);
+
+    c->gid = 0;
+    /* Don't delete if chan ID isn't zero: it's still in the chan hash map */
+    if (!c->chan)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_gid_ht, bto_gid_map);
+  tor_free(bto_gid_map);
+}
+
+/** Clear the chan ID hash map, freeing any bt_orconn_t objects that
+ * become unreferenced */
+static void
+bto_chan_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_chan_ht, bto_chan_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt);
+
+    c->chan = 0;
+    /* Don't delete if GID isn't zero, it's still in the GID hash map */
+    if (!c->gid)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_chan_ht, bto_chan_map);
+  tor_free(bto_chan_map);
+}
+
+/** Delete a bt_orconn from the hash maps by GID */
+void
+bto_delete(uint64_t gid)
+{
+  bt_orconn_t key, *bto;
+
+  key.gid = gid;
+  key.chan = 0;
+  bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto) {
+    /* The orconn might be unregistered because it's an EXT_OR_CONN? */
+    log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64,
+              gid);
+    return;
+  }
+  HT_REMOVE(bto_gid_ht, bto_gid_map, &key);
+  if (bto->chan) {
+    key.chan = bto->chan;
+    HT_REMOVE(bto_chan_ht, bto_chan_map, &key);
+  }
+  tor_free(bto);
+}
+
+/**
+ * Helper for bto_find_or_new().
+ *
+ * Update GID and chan ID of an existing bt_orconn object if needed,
+ * given a search key previously used within bto_find_or_new().
+ **/
+static bt_orconn_t *
+bto_update(bt_orconn_t *bto, const bt_orconn_t *key)
+{
+  /* ORCONN GIDs shouldn't change once assigned */
+  tor_assert(!bto->gid || !key->gid || bto->gid == key->gid);
+  if (!bto->gid && key->gid) {
+    /* Got a gid when we didn't already have one; insert into gid map */
+    log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan,
+              key->gid);
+    bto->gid = key->gid;
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  }
+  /* association of ORCONN with channel shouldn't change */
+  tor_assert(!bto->chan || !key->chan || bto->chan == key->chan);
+  if (!bto->chan && key->chan) {
+    /* Got a chan when we didn't already have one; insert into chan map */
+    log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64,
+              bto->gid, key->chan);
+    bto->chan = key->chan;
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+  }
+  return bto;
+}
+
+/** Helper for bto_find_or_new() */
+static bt_orconn_t *
+bto_new(const bt_orconn_t *key)
+{
+  struct bt_orconn_t *bto = tor_malloc(sizeof(*bto));
+
+  bto->gid = key->gid;
+  bto->chan = key->chan;
+  bto->state = 0;
+  bto->proxy_type = 0;
+  bto->is_orig = false;
+  bto->is_onehop = true;
+
+  if (bto->gid)
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  if (bto->chan)
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+
+  return bto;
+}
+
+/**
+ * Insert a new bt_orconn with the given GID and chan ID, or update
+ * the GID and chan ID if one already exists.
+ *
+ * Return the found or allocated bt_orconn.
+ **/
+bt_orconn_t *
+bto_find_or_new(uint64_t gid, uint64_t chan)
+{
+  bt_orconn_t key, *bto = NULL;
+
+  tor_assert(gid || chan);
+  key.gid = gid;
+  key.chan = chan;
+  if (key.gid)
+    bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto && key.chan) {
+    /* Not found by GID; look up by chan ID */
+    bto = HT_FIND(bto_chan_ht, bto_chan_map, &key);
+  }
+  if (bto)
+    return bto_update(bto, &key);
+  else
+    return bto_new(&key);
+}
+
+/** Initialize the hash maps  */
+void
+bto_init_maps(void)
+{
+  bto_gid_map = tor_malloc(sizeof(*bto_gid_map));
+  HT_INIT(bto_gid_ht, bto_gid_map);
+  bto_chan_map = tor_malloc(sizeof(*bto_chan_map));
+  HT_INIT(bto_chan_ht, bto_chan_map);
+}
+
+/** Clear the hash maps, freeing all associated storage */
+void
+bto_clear_maps(void)
+{
+  bto_gid_clear_map();
+  bto_chan_clear_map();
+}
diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h
new file mode 100644
index 0000000000..c2043fa153
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.h
+ * \brief Header file for btrack_orconn_maps.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_MAPS_H
+#define TOR_BTRACK_ORCONN_MAPS_H
+
+void bto_delete(uint64_t);
+bt_orconn_t *bto_find_or_new(uint64_t, uint64_t);
+
+void bto_init_maps(void);
+void bto_clear_maps(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_MAPS_H) */
diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h
new file mode 100644
index 0000000000..3f831d0640
--- /dev/null
+++ b/src/feature/control/btrack_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_sys.h
+ * \brief Declare subsystem object for the bootstrap tracker susbystem.
+ **/
+
+#ifndef TOR_BTRACK_SYS_H
+#define TOR_BTRACK_SYS_H
+
+extern const struct subsys_fns_t sys_btrack;
+
+#endif /* !defined(TOR_BTRACK_SYS_H) */
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
index 26ac12d307..ece5616907 100644
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@ -1,4 +1,3 @@
-
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
@@ -33,83 +32,27 @@
  * stack.
  **/
 
+#define CONTROL_MODULE_PRIVATE
 #define CONTROL_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
 #include "app/main/main.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
-#include "core/or/channel.h"
-#include "core/or/channeltls.h"
-#include "core/or/circuitbuild.h"
-#include "core/or/circuitlist.h"
-#include "core/or/circuitstats.h"
-#include "core/or/circuituse.h"
-#include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/policies.h"
-#include "core/or/reasons.h"
-#include "core/or/versions.h"
 #include "core/proto/proto_control0.h"
 #include "core/proto/proto_http.h"
-#include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/dnsserv.h"
-#include "feature/client/entrynodes.h"
 #include "feature/control/control.h"
-#include "feature/control/fmt_serverstatus.h"
-#include "feature/control/getinfo_geoip.h"
-#include "feature/dircache/dirserv.h"
-#include "feature/dirclient/dirclient.h"
-#include "feature/dirclient/dlstatus.h"
-#include "feature/dircommon/directory.h"
-#include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
-#include "feature/hs/hs_common.h"
-#include "feature/hs/hs_control.h"
-#include "feature/hs_common/shared_random_client.h"
-#include "feature/nodelist/authcert.h"
-#include "feature/nodelist/dirlist.h"
-#include "feature/nodelist/microdesc.h"
-#include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
-#include "feature/nodelist/routerlist.h"
-#include "feature/relay/router.h"
-#include "feature/relay/routermode.h"
-#include "feature/relay/selftest.h"
-#include "feature/rend/rendclient.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_proto.h"
 #include "feature/rend/rendcommon.h"
-#include "feature/rend/rendparse.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
-#include "feature/stats/predict_ports.h"
-#include "lib/container/buffers.h"
-#include "lib/crypt_ops/crypto_rand.h"
-#include "lib/crypt_ops/crypto_util.h"
-#include "lib/encoding/confline.h"
-#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/procmon.h"
 
-#include "feature/dircache/cached_dir_st.h"
 #include "feature/control/control_connection_st.h"
-#include "core/or/cpath_build_state_st.h"
-#include "core/or/entry_connection_st.h"
-#include "feature/nodelist/extrainfo_st.h"
-#include "feature/nodelist/networkstatus_st.h"
-#include "feature/nodelist/node_st.h"
-#include "core/or/or_connection_st.h"
-#include "core/or/or_circuit_st.h"
-#include "core/or/origin_circuit_st.h"
-#include "feature/nodelist/microdesc_st.h"
-#include "feature/rend/rend_authorized_client_st.h"
-#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
-#include "feature/rend/rend_service_descriptor_st.h"
-#include "feature/nodelist/routerinfo_st.h"
-#include "feature/nodelist/routerlist_st.h"
-#include "core/or/socks_request_st.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -118,152 +61,6 @@
 #include <sys/stat.h>
 #endif
 
-#ifndef _WIN32
-#include <pwd.h>
-#include <sys/resource.h>
-#endif
-
-#include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/evloop/procmon.h"
-#include "lib/evloop/compat_libevent.h"
-
-/** Yield true iff <b>s</b> is the state of a control_connection_t that has
- * finished authentication and is accepting commands. */
-#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
-
-/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
- * connection is interested in events of type <b>e</b>.  We use this
- * so that we can decide to skip generating event messages that nobody
- * has interest in without having to walk over the global connection
- * list to find out.
- **/
-typedef uint64_t event_mask_t;
-
-/** An event mask of all the events that any controller is interested in
- * receiving. */
-static event_mask_t global_event_mask = 0;
-
-/** True iff we have disabled log messages from being sent to the controller */
-static int disable_log_messages = 0;
-
-/** Macro: true if any control connection is interested in events of type
- * <b>e</b>. */
-#define EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & EVENT_MASK_(e)))
-
-/** Macro: true if any event from the bitfield 'e' is interesting. */
-#define ANY_EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & (e)))
-
-/** If we're using cookie-type authentication, how long should our cookies be?
- */
-#define AUTHENTICATION_COOKIE_LEN 32
-
-/** If true, we've set authentication_cookie to a secret code and
- * stored it to disk. */
-static int authentication_cookie_is_set = 0;
-/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
- * and which we're using to authenticate controllers.  (If the controller can
- * read it off disk, it has permission to connect.) */
-static uint8_t *authentication_cookie = NULL;
-
-#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
-  "Tor safe cookie authentication server-to-controller hash"
-#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
-  "Tor safe cookie authentication controller-to-server hash"
-#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
-
-/** The list of onion services that have been added via ADD_ONION that do not
- * belong to any particular control connection.
- */
-static smartlist_t *detached_onion_services = NULL;
-
-/** A sufficiently large size to record the last bootstrap phase string. */
-#define BOOTSTRAP_MSG_LEN 1024
-
-/** What was the last bootstrap phase message we sent? We keep track
- * of this so we can respond to getinfo status/bootstrap-phase queries. */
-static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
-
-static void connection_printf_to_buf(control_connection_t *conn,
-                                     const char *format, ...)
-  CHECK_PRINTF(2,3);
-static void send_control_event_impl(uint16_t event,
-                                    const char *format, va_list ap)
-  CHECK_PRINTF(2,0);
-static int control_event_status(int type, int severity, const char *format,
-                                va_list args)
-  CHECK_PRINTF(3,0);
-
-static void send_control_done(control_connection_t *conn);
-static void send_control_event(uint16_t event,
-                               const char *format, ...)
-  CHECK_PRINTF(2,3);
-static int handle_control_setconf(control_connection_t *conn, uint32_t len,
-                                  char *body);
-static int handle_control_resetconf(control_connection_t *conn, uint32_t len,
-                                    char *body);
-static int handle_control_getconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_setevents(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_authenticate(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_signal(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                                     const char *body);
-static char *list_getinfo_options(void);
-static int handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_extendcircuit(control_connection_t *conn,
-                                        uint32_t len,
-                                        const char *body);
-static int handle_control_setcircuitpurpose(control_connection_t *conn,
-                                            uint32_t len, const char *body);
-static int handle_control_attachstream(control_connection_t *conn,
-                                       uint32_t len,
-                                        const char *body);
-static int handle_control_postdescriptor(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_redirectstream(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_closestream(control_connection_t *conn, uint32_t len,
-                                      const char *body);
-static int handle_control_closecircuit(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_resolve(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_usefeature(control_connection_t *conn,
-                                     uint32_t len,
-                                     const char *body);
-static int handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_hspost(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_add_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_del_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
-                                      size_t len);
-static void orconn_target_get_name(char *buf, size_t len,
-                                   or_connection_t *conn);
-
-static int get_cached_network_liveness(void);
-static void set_cached_network_liveness(int liveness);
-
-static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
-
-static char * download_status_to_string(const download_status_t *dl);
-static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
-
 /** Convert a connection_t* to an control_connection_t*; assert if the cast is
  * invalid. */
 control_connection_t *
@@ -273,410 +70,6 @@ TO_CONTROL_CONN(connection_t *c)
   return DOWNCAST(control_connection_t, c);
 }
 
-/** Given a control event code for a message event, return the corresponding
- * log severity. */
-static inline int
-event_to_log_severity(int event)
-{
-  switch (event) {
-    case EVENT_DEBUG_MSG: return LOG_DEBUG;
-    case EVENT_INFO_MSG: return LOG_INFO;
-    case EVENT_NOTICE_MSG: return LOG_NOTICE;
-    case EVENT_WARN_MSG: return LOG_WARN;
-    case EVENT_ERR_MSG: return LOG_ERR;
-    default: return -1;
-  }
-}
-
-/** Given a log severity, return the corresponding control event code. */
-static inline int
-log_severity_to_event(int severity)
-{
-  switch (severity) {
-    case LOG_DEBUG: return EVENT_DEBUG_MSG;
-    case LOG_INFO: return EVENT_INFO_MSG;
-    case LOG_NOTICE: return EVENT_NOTICE_MSG;
-    case LOG_WARN: return EVENT_WARN_MSG;
-    case LOG_ERR: return EVENT_ERR_MSG;
-    default: return -1;
-  }
-}
-
-/** Helper: clear bandwidth counters of all origin circuits. */
-static void
-clear_circ_bw_fields(void)
-{
-  origin_circuit_t *ocirc;
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-    ocirc = TO_ORIGIN_CIRCUIT(circ);
-    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-  }
-  SMARTLIST_FOREACH_END(circ);
-}
-
-/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
- * connection's event_mask field. */
-void
-control_update_global_event_mask(void)
-{
-  smartlist_t *conns = get_connection_array();
-  event_mask_t old_mask, new_mask;
-  old_mask = global_event_mask;
-  int any_old_per_sec_events = control_any_per_second_event_enabled();
-
-  global_event_mask = 0;
-  SMARTLIST_FOREACH(conns, connection_t *, _conn,
-  {
-    if (_conn->type == CONN_TYPE_CONTROL &&
-        STATE_IS_OPEN(_conn->state)) {
-      control_connection_t *conn = TO_CONTROL_CONN(_conn);
-      global_event_mask |= conn->event_mask;
-    }
-  });
-
-  new_mask = global_event_mask;
-
-  /* Handle the aftermath.  Set up the log callback to tell us only what
-   * we want to hear...*/
-  control_adjust_event_log_severity();
-
-  /* Macro: true if ev was false before and is true now. */
-#define NEWLY_ENABLED(ev) \
-  (! (old_mask & (ev)) && (new_mask & (ev)))
-
-  /* ...then, if we've started logging stream or circ bw, clear the
-   * appropriate fields. */
-  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
-    SMARTLIST_FOREACH(conns, connection_t *, conn,
-    {
-      if (conn->type == CONN_TYPE_AP) {
-        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-        edge_conn->n_written = edge_conn->n_read = 0;
-      }
-    });
-  }
-  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
-    clear_circ_bw_fields();
-  }
-  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
-    uint64_t r, w;
-    control_get_bytes_rw_last_sec(&r, &w);
-  }
-  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
-    reschedule_per_second_timer();
-  }
-
-#undef NEWLY_ENABLED
-}
-
-/** Adjust the log severities that result in control_event_logmsg being called
- * to match the severity of log messages that any controllers are interested
- * in. */
-void
-control_adjust_event_log_severity(void)
-{
-  int i;
-  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
-
-  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      min_log_event = i;
-      break;
-    }
-  }
-  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      max_log_event = i;
-      break;
-    }
-  }
-  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
-    if (min_log_event > EVENT_NOTICE_MSG)
-      min_log_event = EVENT_NOTICE_MSG;
-    if (max_log_event < EVENT_ERR_MSG)
-      max_log_event = EVENT_ERR_MSG;
-  }
-  if (min_log_event <= max_log_event)
-    change_callback_log_severity(event_to_log_severity(min_log_event),
-                                 event_to_log_severity(max_log_event),
-                                 control_event_logmsg);
-  else
-    change_callback_log_severity(LOG_ERR, LOG_ERR,
-                                 control_event_logmsg);
-}
-
-/** Return true iff the event with code <b>c</b> is being sent to any current
- * control connection.  This is useful if the amount of work needed to prepare
- * to call the appropriate control_event_...() function is high.
- */
-int
-control_event_is_interesting(int event)
-{
-  return EVENT_IS_INTERESTING(event);
-}
-
-/** Return true if any event that needs to fire once a second is enabled. */
-int
-control_any_per_second_event_enabled(void)
-{
-  return ANY_EVENT_IS_INTERESTING(
-      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CELL_STATS) |
-      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CONN_BW) |
-      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
-  );
-}
-
-/* The value of 'get_bytes_read()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_read = 0;
-/* The value of 'get_bytes_written()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_written = 0;
-
-/**
- * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
- * and written by Tor since the last call to this function.
- *
- * Call this only from the main thread.
- */
-static void
-control_get_bytes_rw_last_sec(uint64_t *n_read,
-                              uint64_t *n_written)
-{
-  const uint64_t stats_n_bytes_read = get_bytes_read();
-  const uint64_t stats_n_bytes_written = get_bytes_written();
-
-  *n_read = stats_n_bytes_read - stats_prev_n_read;
-  *n_written = stats_n_bytes_written - stats_prev_n_written;
-  stats_prev_n_read = stats_n_bytes_read;
-  stats_prev_n_written = stats_n_bytes_written;
-}
-
-/**
- * Run all the controller events (if any) that are scheduled to trigger once
- * per second.
- */
-void
-control_per_second_events(void)
-{
-  if (!control_any_per_second_event_enabled())
-    return;
-
-  uint64_t bytes_read, bytes_written;
-  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
-  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
-
-  control_event_stream_bandwidth_used();
-  control_event_conn_bandwidth_used();
-  control_event_circ_bandwidth_used();
-  control_event_circuit_cell_stats();
-}
-
-/** Append a NUL-terminated string <b>s</b> to the end of
- * <b>conn</b>-\>outbuf.
- */
-static inline void
-connection_write_str_to_buf(const char *s, control_connection_t *conn)
-{
-  size_t len = strlen(s);
-  connection_buf_add(s, len, TO_CONN(conn));
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
- * contents of <b>data</b> into *<b>out</b>, adding a period before any period
- * that appears at the start of a line, and adding a period-CRLF line at
- * the end. Replace all LF characters sequences with CRLF.  Return the number
- * of bytes in *<b>out</b>.
- */
-STATIC size_t
-write_escaped_data(const char *data, size_t len, char **out)
-{
-  tor_assert(len < SIZE_MAX - 9);
-  size_t sz_out = len+8+1;
-  char *outp;
-  const char *start = data, *end;
-  size_t i;
-  int start_of_line;
-  for (i=0; i < len; ++i) {
-    if (data[i] == '\n') {
-      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
-      if (sz_out >= SIZE_T_CEILING) {
-        log_warn(LD_BUG, "Input to write_escaped_data was too long");
-        *out = tor_strdup(".\r\n");
-        return 3;
-      }
-    }
-  }
-  *out = outp = tor_malloc(sz_out);
-  end = data+len;
-  start_of_line = 1;
-  while (data < end) {
-    if (*data == '\n') {
-      if (data > start && data[-1] != '\r')
-        *outp++ = '\r';
-      start_of_line = 1;
-    } else if (*data == '.') {
-      if (start_of_line) {
-        start_of_line = 0;
-        *outp++ = '.';
-      }
-    } else {
-      start_of_line = 0;
-    }
-    *outp++ = *data++;
-  }
-  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
-    *outp++ = '\r';
-    *outp++ = '\n';
-  }
-  *outp++ = '.';
-  *outp++ = '\r';
-  *outp++ = '\n';
-  *outp = '\0'; /* NUL-terminate just in case. */
-  tor_assert(outp >= *out);
-  tor_assert((size_t)(outp - *out) <= sz_out);
-  return outp - *out;
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
- * the contents of <b>data</b> into *<b>out</b>, removing any period
- * that appears at the start of a line, and replacing all CRLF sequences
- * with LF.   Return the number of
- * bytes in *<b>out</b>. */
-STATIC size_t
-read_escaped_data(const char *data, size_t len, char **out)
-{
-  char *outp;
-  const char *next;
-  const char *end;
-
-  *out = outp = tor_malloc(len+1);
-
-  end = data+len;
-
-  while (data < end) {
-    /* we're at the start of a line. */
-    if (*data == '.')
-      ++data;
-    next = memchr(data, '\n', end-data);
-    if (next) {
-      size_t n_to_copy = next-data;
-      /* Don't copy a CR that precedes this LF. */
-      if (n_to_copy && *(next-1) == '\r')
-        --n_to_copy;
-      memcpy(outp, data, n_to_copy);
-      outp += n_to_copy;
-      data = next+1; /* This will point at the start of the next line,
-                      * or the end of the string, or a period. */
-    } else {
-      memcpy(outp, data, end-data);
-      outp += (end-data);
-      *outp = '\0';
-      return outp - *out;
-    }
-    *outp++ = '\n';
-  }
-
-  *outp = '\0';
-  return outp - *out;
-}
-
-/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
- * double-quoted string with escaped characters, return the length of that
- * string (as encoded, including quotes).  Otherwise return -1. */
-static inline int
-get_escaped_string_length(const char *start, size_t in_len_max,
-                          int *chars_out)
-{
-  const char *cp, *end;
-  int chars = 0;
-
-  if (*start != '\"')
-    return -1;
-
-  cp = start+1;
-  end = start+in_len_max;
-
-  /* Calculate length. */
-  while (1) {
-    if (cp >= end) {
-      return -1; /* Too long. */
-    } else if (*cp == '\\') {
-      if (++cp == end)
-        return -1; /* Can't escape EOS. */
-      ++cp;
-      ++chars;
-    } else if (*cp == '\"') {
-      break;
-    } else {
-      ++cp;
-      ++chars;
-    }
-  }
-  if (chars_out)
-    *chars_out = chars;
-  return (int)(cp - start+1);
-}
-
-/** As decode_escaped_string, but does not decode the string: copies the
- * entire thing, including quotation marks. */
-static const char *
-extract_escaped_string(const char *start, size_t in_len_max,
-                       char **out, size_t *out_len)
-{
-  int length = get_escaped_string_length(start, in_len_max, NULL);
-  if (length<0)
-    return NULL;
-  *out_len = length;
-  *out = tor_strndup(start, *out_len);
-  return start+length;
-}
-
-/** Given a pointer to a string starting at <b>start</b> containing
- * <b>in_len_max</b> characters, decode a string beginning with one double
- * quote, containing any number of non-quote characters or characters escaped
- * with a backslash, and ending with a final double quote.  Place the resulting
- * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
- * store its length in <b>out_len</b>.  On success, return a pointer to the
- * character immediately following the escaped string.  On failure, return
- * NULL. */
-static const char *
-decode_escaped_string(const char *start, size_t in_len_max,
-                   char **out, size_t *out_len)
-{
-  const char *cp, *end;
-  char *outp;
-  int len, n_chars = 0;
-
-  len = get_escaped_string_length(start, in_len_max, &n_chars);
-  if (len<0)
-    return NULL;
-
-  end = start+len-1; /* Index of last quote. */
-  tor_assert(*end == '\"');
-  outp = *out = tor_malloc(len+1);
-  *out_len = n_chars;
-
-  cp = start+1;
-  while (cp < end) {
-    if (*cp == '\\')
-      ++cp;
-    *outp++ = *cp++;
-  }
-  *outp = '\0';
-  tor_assert((outp - *out) == (int)*out_len);
-
-  return end+1;
-}
-
 /** Create and add a new controller connection on <b>sock</b>.  If
  * <b>CC_LOCAL_FD_IS_OWNER</b> is set in <b>flags</b>, this Tor process should
  * exit when the connection closes.  If <b>CC_LOCAL_FD_IS_AUTHENTICATED</b>
@@ -720,29 +113,6 @@ control_connection_add_local_fd(tor_socket_t sock, unsigned flags)
   return 0;
 }
 
-/** Acts like sprintf, but writes its formatted string to the end of
- * <b>conn</b>-\>outbuf. */
-static void
-connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
-{
-  va_list ap;
-  char *buf = NULL;
-  int len;
-
-  va_start(ap,format);
-  len = tor_vasprintf(&buf, format, ap);
-  va_end(ap);
-
-  if (len < 0) {
-    log_err(LD_BUG, "Unable to format string for controller.");
-    tor_assert(0);
-  }
-
-  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
-
-  tor_free(buf);
-}
-
 /** Write all of the open control ports to ControlPortWriteToFile */
 void
 control_ports_write_to_file(void)
@@ -787,886 +157,11 @@ control_ports_write_to_file(void)
   smartlist_free(lines);
 }
 
-/** Send a "DONE" message down the control connection <b>conn</b>. */
-static void
-send_control_done(control_connection_t *conn)
-{
-  connection_write_str_to_buf("250 OK\r\n", conn);
-}
-
-/** Represents an event that's queued to be sent to one or more
- * controllers. */
-typedef struct queued_event_s {
-  uint16_t event;
-  char *msg;
-} queued_event_t;
-
-/** Pointer to int. If this is greater than 0, we don't allow new events to be
- * queued. */
-static tor_threadlocal_t block_event_queue_flag;
-
-/** Holds a smartlist of queued_event_t objects that may need to be sent
- * to one or more controllers */
-static smartlist_t *queued_control_events = NULL;
-
-/** True if the flush_queued_events_event is pending. */
-static int flush_queued_event_pending = 0;
-
-/** Lock to protect the above fields. */
-static tor_mutex_t *queued_control_events_lock = NULL;
-
-/** An event that should fire in order to flush the contents of
- * queued_control_events. */
-static mainloop_event_t *flush_queued_events_event = NULL;
-
-void
-control_initialize_event_queue(void)
-{
-  if (queued_control_events == NULL) {
-    queued_control_events = smartlist_new();
-  }
-
-  if (flush_queued_events_event == NULL) {
-    struct event_base *b = tor_libevent_get_base();
-    if (b) {
-      flush_queued_events_event =
-        mainloop_event_new(flush_queued_events_cb, NULL);
-      tor_assert(flush_queued_events_event);
-    }
-  }
-
-  if (queued_control_events_lock == NULL) {
-    queued_control_events_lock = tor_mutex_new();
-    tor_threadlocal_init(&block_event_queue_flag);
-  }
-}
-
-static int *
-get_block_event_queue(void)
-{
-  int *val = tor_threadlocal_get(&block_event_queue_flag);
-  if (PREDICT_UNLIKELY(val == NULL)) {
-    val = tor_malloc_zero(sizeof(int));
-    tor_threadlocal_set(&block_event_queue_flag, val);
-  }
-  return val;
-}
-
-/** Helper: inserts an event on the list of events queued to be sent to
- * one or more controllers, and schedules the events to be flushed if needed.
- *
- * This function takes ownership of <b>msg</b>, and may free it.
- *
- * We queue these events rather than send them immediately in order to break
- * the dependency in our callgraph from code that generates events for the
- * controller, and the network layer at large.  Otherwise, nearly every
- * interesting part of Tor would potentially call every other interesting part
- * of Tor.
- */
-MOCK_IMPL(STATIC void,
-queue_control_event_string,(uint16_t event, char *msg))
-{
-  /* This is redundant with checks done elsewhere, but it's a last-ditch
-   * attempt to avoid queueing something we shouldn't have to queue. */
-  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
-    tor_free(msg);
-    return;
-  }
-
-  int *block_event_queue = get_block_event_queue();
-  if (*block_event_queue) {
-    tor_free(msg);
-    return;
-  }
-
-  queued_event_t *ev = tor_malloc(sizeof(*ev));
-  ev->event = event;
-  ev->msg = msg;
-
-  /* No queueing an event while queueing an event */
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  tor_assert(queued_control_events);
-  smartlist_add(queued_control_events, ev);
-
-  int activate_event = 0;
-  if (! flush_queued_event_pending && in_main_thread()) {
-    activate_event = 1;
-    flush_queued_event_pending = 1;
-  }
-
-  tor_mutex_release(queued_control_events_lock);
-
-  --*block_event_queue;
-
-  /* We just put an event on the queue; mark the queue to be
-   * flushed.  We only do this from the main thread for now; otherwise,
-   * we'd need to incur locking overhead in Libevent or use a socket.
-   */
-  if (activate_event) {
-    tor_assert(flush_queued_events_event);
-    mainloop_event_activate(flush_queued_events_event);
-  }
-}
-
-#define queued_event_free(ev) \
-  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
-
-/** Release all storage held by <b>ev</b>. */
-static void
-queued_event_free_(queued_event_t *ev)
-{
-  if (ev == NULL)
-    return;
-
-  tor_free(ev->msg);
-  tor_free(ev);
-}
-
-/** Send every queued event to every controller that's interested in it,
- * and remove the events from the queue.  If <b>force</b> is true,
- * then make all controllers send their data out immediately, since we
- * may be about to shut down. */
-static void
-queued_events_flush_all(int force)
-{
-  /* Make sure that we get all the pending log events, if there are any. */
-  flush_pending_log_callbacks();
-
-  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
-    return;
-  }
-  smartlist_t *all_conns = get_connection_array();
-  smartlist_t *controllers = smartlist_new();
-  smartlist_t *queued_events;
-
-  int *block_event_queue = get_block_event_queue();
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  /* No queueing an event while flushing events. */
-  flush_queued_event_pending = 0;
-  queued_events = queued_control_events;
-  queued_control_events = smartlist_new();
-  tor_mutex_release(queued_control_events_lock);
-
-  /* Gather all the controllers that will care... */
-  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
-    if (conn->type == CONN_TYPE_CONTROL &&
-        !conn->marked_for_close &&
-        conn->state == CONTROL_CONN_STATE_OPEN) {
-      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
-
-      smartlist_add(controllers, control_conn);
-    }
-  } SMARTLIST_FOREACH_END(conn);
-
-  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
-    const event_mask_t bit = ((event_mask_t)1) << ev->event;
-    const size_t msg_len = strlen(ev->msg);
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      if (control_conn->event_mask & bit) {
-        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
-      }
-    } SMARTLIST_FOREACH_END(control_conn);
-
-    queued_event_free(ev);
-  } SMARTLIST_FOREACH_END(ev);
-
-  if (force) {
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      connection_flush(TO_CONN(control_conn));
-    } SMARTLIST_FOREACH_END(control_conn);
-  }
-
-  smartlist_free(queued_events);
-  smartlist_free(controllers);
-
-  --*block_event_queue;
-}
-
-/** Libevent callback: Flushes pending events to controllers that are
- * interested in them. */
-static void
-flush_queued_events_cb(mainloop_event_t *event, void *arg)
-{
-  (void) event;
-  (void) arg;
-  queued_events_flush_all(0);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is given by <b>msg</b>.
- *
- * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
- * respect to the EXTENDED_EVENTS feature. */
-MOCK_IMPL(STATIC void,
-send_control_event_string,(uint16_t event,
-                           const char *msg))
-{
-  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
-  queue_control_event_string(event, tor_strdup(msg));
-}
-
-/** Helper for send_control_event and control_event_status:
- * Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event_impl(uint16_t event,
-                        const char *format, va_list ap)
-{
-  char *buf = NULL;
-  int len;
-
-  len = tor_vasprintf(&buf, format, ap);
-  if (len < 0) {
-    log_warn(LD_BUG, "Unable to format event for controller.");
-    return;
-  }
-
-  queue_control_event_string(event, buf);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event(uint16_t event,
-                   const char *format, ...)
-{
-  va_list ap;
-  va_start(ap, format);
-  send_control_event_impl(event, format, ap);
-  va_end(ap);
-}
-
-/** Given a text circuit <b>id</b>, return the corresponding circuit. */
-static origin_circuit_t *
-get_circ(const char *id)
-{
-  uint32_t n_id;
-  int ok;
-  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  return circuit_get_by_global_id(n_id);
-}
-
-/** Given a text stream <b>id</b>, return the corresponding AP connection. */
-static entry_connection_t *
-get_stream(const char *id)
-{
-  uint64_t n_id;
-  int ok;
-  connection_t *conn;
-  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  conn = connection_get_by_global_id(n_id);
-  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
-    return NULL;
-  return TO_ENTRY_CONN(conn);
-}
-
-/** Helper for setconf and resetconf. Acts like setconf, except
- * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
- * contents of body.
- */
-static int
-control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
-                       int use_defaults)
-{
-  setopt_err_t opt_err;
-  config_line_t *lines=NULL;
-  char *start = body;
-  char *errstring = NULL;
-  const unsigned flags =
-    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
-
-  char *config;
-  smartlist_t *entries = smartlist_new();
-
-  /* We have a string, "body", of the format '(key(=val|="val")?)' entries
-   * separated by space.  break it into a list of configuration entries. */
-  while (*body) {
-    char *eq = body;
-    char *key;
-    char *entry;
-    while (!TOR_ISSPACE(*eq) && *eq != '=')
-      ++eq;
-    key = tor_strndup(body, eq-body);
-    body = eq+1;
-    if (*eq == '=') {
-      char *val=NULL;
-      size_t val_len=0;
-      if (*body != '\"') {
-        char *val_start = body;
-        while (!TOR_ISSPACE(*body))
-          body++;
-        val = tor_strndup(val_start, body-val_start);
-        val_len = strlen(val);
-      } else {
-        body = (char*)extract_escaped_string(body, (len - (body-start)),
-                                             &val, &val_len);
-        if (!body) {
-          connection_write_str_to_buf("551 Couldn't parse string\r\n", conn);
-          SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-          smartlist_free(entries);
-          tor_free(key);
-          return 0;
-        }
-      }
-      tor_asprintf(&entry, "%s %s", key, val);
-      tor_free(key);
-      tor_free(val);
-    } else {
-      entry = key;
-    }
-    smartlist_add(entries, entry);
-    while (TOR_ISSPACE(*body))
-      ++body;
-  }
-
-  smartlist_add_strdup(entries, "");
-  config = smartlist_join_strings(entries, "\n", 0, NULL);
-  SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-  smartlist_free(entries);
-
-  if (config_get_lines(config, &lines, 0) < 0) {
-    log_warn(LD_CONTROL,"Controller gave us config lines we can't parse.");
-    connection_write_str_to_buf("551 Couldn't parse configuration\r\n",
-                                conn);
-    tor_free(config);
-    return 0;
-  }
-  tor_free(config);
-
-  opt_err = options_trial_assign(lines, flags, &errstring);
-  {
-    const char *msg;
-    switch (opt_err) {
-      case SETOPT_ERR_MISC:
-        msg = "552 Unrecognized option";
-        break;
-      case SETOPT_ERR_PARSE:
-        msg = "513 Unacceptable option value";
-        break;
-      case SETOPT_ERR_TRANSITION:
-        msg = "553 Transition not allowed";
-        break;
-      case SETOPT_ERR_SETTING:
-      default:
-        msg = "553 Unable to set option";
-        break;
-      case SETOPT_OK:
-        config_free_lines(lines);
-        send_control_done(conn);
-        return 0;
-    }
-    log_warn(LD_CONTROL,
-             "Controller gave us config lines that didn't validate: %s",
-             errstring);
-    connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    config_free_lines(lines);
-    tor_free(errstring);
-    return 0;
-  }
-}
-
-/** Called when we receive a SETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body.*/
-static int
-handle_control_setconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 0);
-}
-
-/** Called when we receive a RESETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body. */
-static int
-handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 1);
-}
-
-/** Called when we receive a GETCONF message.  Parse the request, and
- * reply with a CONFVALUE or an ERROR message */
-static int
-handle_control_getconf(control_connection_t *conn, uint32_t body_len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *msg = NULL;
-  size_t msg_len;
-  const or_options_t *options = get_options();
-  int i, len;
-
-  (void) body_len; /* body is NUL-terminated; so we can ignore len. */
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    if (!option_is_recognized(q)) {
-      smartlist_add(unrecognized, (char*) q);
-    } else {
-      config_line_t *answer = option_get_assignment(options,q);
-      if (!answer) {
-        const char *name = option_get_canonical_name(q);
-        smartlist_add_asprintf(answers, "250-%s\r\n", name);
-      }
-
-      while (answer) {
-        config_line_t *next;
-        smartlist_add_asprintf(answers, "250-%s=%s\r\n",
-                     answer->key, answer->value);
-
-        next = answer->next;
-        tor_free(answer->key);
-        tor_free(answer->value);
-        tor_free(answer);
-        answer = next;
-      }
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if ((len = smartlist_len(unrecognized))) {
-    for (i=0; i < len-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-Unrecognized configuration key \"%s\"\r\n",
-                               (char*)smartlist_get(unrecognized, i));
-    connection_printf_to_buf(conn,
-                             "552 Unrecognized configuration key \"%s\"\r\n",
-                             (char*)smartlist_get(unrecognized, len-1));
-  } else if ((len = smartlist_len(answers))) {
-    char *tmp = smartlist_get(answers, len-1);
-    tor_assert(strlen(tmp)>4);
-    tmp[3] = ' ';
-    msg = smartlist_join_strings(answers, "", 0, &msg_len);
-    connection_buf_add(msg, msg_len, TO_CONN(conn));
-  } else {
-    connection_write_str_to_buf("250 OK\r\n", conn);
-  }
-
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  smartlist_free(unrecognized);
-
-  tor_free(msg);
-
-  return 0;
-}
-
-/** Called when we get a +LOADCONF message. */
-static int
-handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  setopt_err_t retval;
-  char *errstring = NULL;
-  const char *msg = NULL;
-  (void) len;
-
-  retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring);
-
-  if (retval != SETOPT_OK)
-    log_warn(LD_CONTROL,
-             "Controller gave us config file that didn't validate: %s",
-             errstring);
-
-  switch (retval) {
-  case SETOPT_ERR_PARSE:
-    msg = "552 Invalid config file";
-    break;
-  case SETOPT_ERR_TRANSITION:
-    msg = "553 Transition not allowed";
-    break;
-  case SETOPT_ERR_SETTING:
-    msg = "553 Unable to set option";
-    break;
-  case SETOPT_ERR_MISC:
-  default:
-    msg = "550 Unable to load config";
-    break;
-  case SETOPT_OK:
-    break;
-  }
-  if (msg) {
-    if (errstring)
-      connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    else
-      connection_printf_to_buf(conn, "%s\r\n", msg);
-  } else {
-    send_control_done(conn);
-  }
-  tor_free(errstring);
-  return 0;
-}
-
-/** Helper structure: maps event values to their names. */
-struct control_event_t {
-  uint16_t event_code;
-  const char *event_name;
-};
-/** Table mapping event values to their names.  Used to implement SETEVENTS
- * and GETINFO events/names, and to keep they in sync. */
-static const struct control_event_t control_event_table[] = {
-  { EVENT_CIRCUIT_STATUS, "CIRC" },
-  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
-  { EVENT_STREAM_STATUS, "STREAM" },
-  { EVENT_OR_CONN_STATUS, "ORCONN" },
-  { EVENT_BANDWIDTH_USED, "BW" },
-  { EVENT_DEBUG_MSG, "DEBUG" },
-  { EVENT_INFO_MSG, "INFO" },
-  { EVENT_NOTICE_MSG, "NOTICE" },
-  { EVENT_WARN_MSG, "WARN" },
-  { EVENT_ERR_MSG, "ERR" },
-  { EVENT_NEW_DESC, "NEWDESC" },
-  { EVENT_ADDRMAP, "ADDRMAP" },
-  { EVENT_DESCCHANGED, "DESCCHANGED" },
-  { EVENT_NS, "NS" },
-  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
-  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
-  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
-  { EVENT_GUARD, "GUARD" },
-  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
-  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
-  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
-  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
-  { EVENT_GOT_SIGNAL, "SIGNAL" },
-  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
-  { EVENT_CONN_BW, "CONN_BW" },
-  { EVENT_CELL_STATS, "CELL_STATS" },
-  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
-  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
-  { EVENT_HS_DESC, "HS_DESC" },
-  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
-  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
-  { 0, NULL },
-};
-
-/** Called when we get a SETEVENTS message: update conn->event_mask,
- * and reply with DONE or ERROR. */
-static int
-handle_control_setevents(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  int event_code;
-  event_mask_t event_mask = 0;
-  smartlist_t *events = smartlist_new();
-
-  (void) len;
-
-  smartlist_split_string(events, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
-    {
-      if (!strcasecmp(ev, "EXTENDED") ||
-          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
-        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
-                 "supported.", ev);
-        continue;
-      } else {
-        int i;
-        event_code = -1;
-
-        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-          if (!strcasecmp(ev, control_event_table[i].event_name)) {
-            event_code = control_event_table[i].event_code;
-            break;
-          }
-        }
-
-        if (event_code == -1) {
-          connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n",
-                                   ev);
-          SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-          smartlist_free(events);
-          return 0;
-        }
-      }
-      event_mask |= (((event_mask_t)1) << event_code);
-    }
-  SMARTLIST_FOREACH_END(ev);
-  SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-  smartlist_free(events);
-
-  conn->event_mask = event_mask;
-
-  control_update_global_event_mask();
-  send_control_done(conn);
-  return 0;
-}
-
-/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
- * Return a smartlist of acceptable passwords (unterminated strings of
- * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
- * failure.
- */
-smartlist_t *
-decode_hashed_passwords(config_line_t *passwords)
-{
-  char decoded[64];
-  config_line_t *cl;
-  smartlist_t *sl = smartlist_new();
-
-  tor_assert(passwords);
-
-  for (cl = passwords; cl; cl = cl->next) {
-    const char *hashed = cl->value;
-
-    if (!strcmpstart(hashed, "16:")) {
-      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
-                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
-          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
-        goto err;
-      }
-    } else {
-        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
-            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
-          goto err;
-        }
-    }
-    smartlist_add(sl,
-                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
-  }
-
-  return sl;
-
- err:
-  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
-  smartlist_free(sl);
-  return NULL;
-}
-
-/** Called when we get an AUTHENTICATE message.  Check whether the
- * authentication is valid, and if so, update the connection's state to
- * OPEN.  Reply with DONE or ERROR.
- */
-static int
-handle_control_authenticate(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  int used_quoted_string = 0;
-  const or_options_t *options = get_options();
-  const char *errstr = "Unknown error";
-  char *password;
-  size_t password_len;
-  const char *cp;
-  int i;
-  int bad_cookie=0, bad_password=0;
-  smartlist_t *sl = NULL;
-
-  if (!len) {
-    password = tor_strdup("");
-    password_len = 0;
-  } else if (TOR_ISXDIGIT(body[0])) {
-    cp = body;
-    while (TOR_ISXDIGIT(*cp))
-      ++cp;
-    i = (int)(cp - body);
-    tor_assert(i>0);
-    password_len = i/2;
-    password = tor_malloc(password_len + 1);
-    if (base16_decode(password, password_len+1, body, i)
-                      != (int) password_len) {
-      connection_write_str_to_buf(
-            "551 Invalid hexadecimal encoding.  Maybe you tried a plain text "
-            "password?  If so, the standard requires that you put it in "
-            "double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(password);
-      return 0;
-    }
-  } else {
-    if (!decode_escaped_string(body, len, &password, &password_len)) {
-      connection_write_str_to_buf("551 Invalid quoted string.  You need "
-            "to put the password in double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return 0;
-    }
-    used_quoted_string = 1;
-  }
-
-  if (conn->safecookie_client_hash != NULL) {
-    /* The controller has chosen safe cookie authentication; the only
-     * acceptable authentication value is the controller-to-server
-     * response. */
-
-    tor_assert(authentication_cookie_is_set);
-
-    if (password_len != DIGEST256_LEN) {
-      log_warn(LD_CONTROL,
-               "Got safe cookie authentication response with wrong length "
-               "(%d)", (int)password_len);
-      errstr = "Wrong length for safe cookie response.";
-      goto err;
-    }
-
-    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
-      log_warn(LD_CONTROL,
-               "Got incorrect safe cookie authentication response");
-      errstr = "Safe cookie response did not match expected value.";
-      goto err;
-    }
-
-    tor_free(conn->safecookie_client_hash);
-    goto ok;
-  }
-
-  if (!options->CookieAuthentication && !options->HashedControlPassword &&
-      !options->HashedControlSessionPassword) {
-    /* if Tor doesn't demand any stronger authentication, then
-     * the controller can get in with anything. */
-    goto ok;
-  }
-
-  if (options->CookieAuthentication) {
-    int also_password = options->HashedControlPassword != NULL ||
-      options->HashedControlSessionPassword != NULL;
-    if (password_len != AUTHENTICATION_COOKIE_LEN) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
-                 "(%d)", (int)password_len);
-        errstr = "Wrong length on authentication cookie.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else if (tor_memneq(authentication_cookie, password, password_len)) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
-        errstr = "Authentication cookie did not match expected value.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else {
-      goto ok;
-    }
-  }
-
-  if (options->HashedControlPassword ||
-      options->HashedControlSessionPassword) {
-    int bad = 0;
-    smartlist_t *sl_tmp;
-    char received[DIGEST_LEN];
-    int also_cookie = options->CookieAuthentication;
-    sl = smartlist_new();
-    if (options->HashedControlPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (options->HashedControlSessionPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (bad) {
-      if (!also_cookie) {
-        log_warn(LD_BUG,
-                 "Couldn't decode HashedControlPassword: invalid base16");
-        errstr="Couldn't decode HashedControlPassword value in configuration.";
-        goto err;
-      }
-      bad_password = 1;
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-    } else {
-      SMARTLIST_FOREACH(sl, char *, expected,
-      {
-        secret_to_key_rfc2440(received,DIGEST_LEN,
-                              password,password_len,expected);
-        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
-                      received, DIGEST_LEN))
-          goto ok;
-      });
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-
-      if (used_quoted_string)
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration";
-      else
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration. Maybe you tried a plain text password? "
-          "If so, the standard requires that you put it in double quotes.";
-      bad_password = 1;
-      if (!also_cookie)
-        goto err;
-    }
-  }
-
-  /** We only get here if both kinds of authentication failed. */
-  tor_assert(bad_password && bad_cookie);
-  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
-  errstr = "Password did not match HashedControlPassword *or* authentication "
-    "cookie.";
-
- err:
-  tor_free(password);
-  connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr);
-  connection_mark_for_close(TO_CONN(conn));
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
- ok:
-  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
-           ")", conn->base_.s);
-  send_control_done(conn);
-  conn->base_.state = CONTROL_CONN_STATE_OPEN;
-  tor_free(password);
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
-}
-
-/** Called when we get a SAVECONF command. Try to flush the current options to
- * disk, and report success or failure. */
-static int
-handle_control_saveconf(control_connection_t *conn, uint32_t len,
-                        const char *body)
-{
-  (void) len;
-
-  int force = !strcmpstart(body, "FORCE");
-  const or_options_t *options = get_options();
-  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
-    connection_write_str_to_buf(
-      "551 Unable to write configuration to disk.\r\n", conn);
-  } else {
-    send_control_done(conn);
-  }
-  return 0;
-}
-
-struct signal_t {
-  int sig;
-  const char *signal_name;
-};
-
-static const struct signal_t signal_table[] = {
+const struct signal_name_t signal_table[] = {
+  /* NOTE: this table is used for handling SIGNAL commands and generating
+   * SIGNAL events.  Order is significant: if there are two entries for the
+   * same numeric signal, the first one is the canonical name generated
+   * for the events. */
   { SIGHUP, "RELOAD" },
   { SIGHUP, "HUP" },
   { SIGINT, "SHUTDOWN" },
@@ -1680,3576 +175,11 @@ static const struct signal_t signal_table[] = {
   { SIGNEWNYM, "NEWNYM" },
   { SIGCLEARDNSCACHE, "CLEARDNSCACHE"},
   { SIGHEARTBEAT, "HEARTBEAT"},
+  { SIGACTIVE, "ACTIVE" },
+  { SIGDORMANT, "DORMANT" },
   { 0, NULL },
 };
 
-/** Called when we get a SIGNAL command. React to the provided signal, and
- * report success or failure. (If the signal results in a shutdown, success
- * may not be reported.) */
-static int
-handle_control_signal(control_connection_t *conn, uint32_t len,
-                      const char *body)
-{
-  int sig = -1;
-  int i;
-  int n = 0;
-  char *s;
-
-  (void) len;
-
-  while (body[n] && ! TOR_ISSPACE(body[n]))
-    ++n;
-  s = tor_strndup(body, n);
-
-  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
-    if (!strcasecmp(s, signal_table[i].signal_name)) {
-      sig = signal_table[i].sig;
-      break;
-    }
-  }
-
-  if (sig < 0)
-    connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n",
-                             s);
-  tor_free(s);
-  if (sig < 0)
-    return 0;
-
-  send_control_done(conn);
-  /* Flush the "done" first if the signal might make us shut down. */
-  if (sig == SIGTERM || sig == SIGINT)
-    connection_flush(TO_CONN(conn));
-
-  activate_signal(sig);
-
-  return 0;
-}
-
-/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
- * as an owning connection, so that we will exit if the connection
- * closes. */
-static int
-handle_control_takeownership(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  (void)len;
-  (void)body;
-
-  conn->is_owning_control_connection = 1;
-
-  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
-           "Tor instance.",
-           (int)(conn->base_.s));
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
- * containing funny characters. */
-static int
-address_is_invalid_mapaddress_target(const char *addr)
-{
-  if (!strcmpstart(addr, "*."))
-    return address_is_invalid_destination(addr+2, 1);
-  else
-    return address_is_invalid_destination(addr, 1);
-}
-
-/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
- * and report success or failure. */
-static int
-handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                          const char *body)
-{
-  smartlist_t *elts;
-  smartlist_t *lines;
-  smartlist_t *reply;
-  char *r;
-  size_t sz;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  lines = smartlist_new();
-  elts = smartlist_new();
-  reply = smartlist_new();
-  smartlist_split_string(lines, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(lines, char *, line) {
-    tor_strlower(line);
-    smartlist_split_string(elts, line, "=", 0, 2);
-    if (smartlist_len(elts) == 2) {
-      const char *from = smartlist_get(elts,0);
-      const char *to = smartlist_get(elts,1);
-      if (address_is_invalid_mapaddress_target(to)) {
-        smartlist_add_asprintf(reply,
-                     "512-syntax error: invalid address '%s'", to);
-        log_warn(LD_CONTROL,
-                 "Skipping invalid argument '%s' in MapAddress msg", to);
-      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
-                 !strcmp(from, "::")) {
-        const char type =
-          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
-          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
-        const char *address = addressmap_register_virtual_address(
-                                                     type, tor_strdup(to));
-        if (!address) {
-          smartlist_add_asprintf(reply,
-                       "451-resource exhausted: skipping '%s'", line);
-          log_warn(LD_CONTROL,
-                   "Unable to allocate address for '%s' in MapAddress msg",
-                   safe_str_client(line));
-        } else {
-          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
-        }
-      } else {
-        const char *msg;
-        if (addressmap_register_auto(from, to, 1,
-                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
-          smartlist_add_asprintf(reply,
-                                 "512-syntax error: invalid address mapping "
-                                 " '%s': %s", line, msg);
-          log_warn(LD_CONTROL,
-                   "Skipping invalid argument '%s' in MapAddress msg: %s",
-                   line, msg);
-        } else {
-          smartlist_add_asprintf(reply, "250-%s", line);
-        }
-      }
-    } else {
-      smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is "
-                   "not of expected form 'foo=bar'.", line);
-      log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong "
-                           "number of items.",
-                           safe_str_client(line));
-    }
-    SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
-    smartlist_clear(elts);
-  } SMARTLIST_FOREACH_END(line);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  smartlist_free(elts);
-
-  if (smartlist_len(reply)) {
-    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
-    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
-    connection_buf_add(r, sz, TO_CONN(conn));
-    tor_free(r);
-  } else {
-    const char *response =
-      "512 syntax error: not enough arguments to mapaddress.\r\n";
-    connection_buf_add(response, strlen(response), TO_CONN(conn));
-  }
-
-  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
-  smartlist_free(reply);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for various
- * trivial-to-implement questions. */
-static int
-getinfo_helper_misc(control_connection_t *conn, const char *question,
-                    char **answer, const char **errmsg)
-{
-  (void) conn;
-  if (!strcmp(question, "version")) {
-    *answer = tor_strdup(get_version());
-  } else if (!strcmp(question, "bw-event-cache")) {
-    *answer = get_bw_samples();
-  } else if (!strcmp(question, "config-file")) {
-    const char *a = get_torrc_fname(0);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-defaults-file")) {
-    const char *a = get_torrc_fname(1);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-text")) {
-    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
-  } else if (!strcmp(question, "config-can-saveconf")) {
-    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
-  } else if (!strcmp(question, "info/names")) {
-    *answer = list_getinfo_options();
-  } else if (!strcmp(question, "dormant")) {
-    int dormant = rep_hist_circbuilding_dormant(time(NULL));
-    *answer = tor_strdup(dormant ? "1" : "0");
-  } else if (!strcmp(question, "events/names")) {
-    int i;
-    smartlist_t *event_names = smartlist_new();
-
-    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-      smartlist_add(event_names, (char *)control_event_table[i].event_name);
-    }
-
-    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
-
-    smartlist_free(event_names);
-  } else if (!strcmp(question, "signal/names")) {
-    smartlist_t *signal_names = smartlist_new();
-    int j;
-    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
-      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
-    }
-
-    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
-
-    smartlist_free(signal_names);
-  } else if (!strcmp(question, "features/names")) {
-    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
-  } else if (!strcmp(question, "address")) {
-    uint32_t addr;
-    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
-      *errmsg = "Address unknown";
-      return -1;
-    }
-    *answer = tor_dup_ip(addr);
-  } else if (!strcmp(question, "traffic/read")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
-  } else if (!strcmp(question, "traffic/written")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
-  } else if (!strcmp(question, "uptime")) {
-    long uptime_secs = get_uptime();
-    tor_asprintf(answer, "%ld", uptime_secs);
-  } else if (!strcmp(question, "process/pid")) {
-    int myPid = -1;
-
-#ifdef _WIN32
-      myPid = _getpid();
-#else
-      myPid = getpid();
-#endif
-
-    tor_asprintf(answer, "%d", myPid);
-  } else if (!strcmp(question, "process/uid")) {
-#ifdef _WIN32
-      *answer = tor_strdup("-1");
-#else
-      int myUid = geteuid();
-      tor_asprintf(answer, "%d", myUid);
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/user")) {
-#ifdef _WIN32
-      *answer = tor_strdup("");
-#else
-      int myUid = geteuid();
-      const struct passwd *myPwEntry = tor_getpwuid(myUid);
-
-      if (myPwEntry) {
-        *answer = tor_strdup(myPwEntry->pw_name);
-      } else {
-        *answer = tor_strdup("");
-      }
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/descriptor-limit")) {
-    int max_fds = get_max_sockets();
-    tor_asprintf(answer, "%d", max_fds);
-  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
-    tor_asprintf(answer, "%"PRIu64,
-                 (get_options()->MaxMemInQueues));
-  } else if (!strcmp(question, "fingerprint")) {
-    crypto_pk_t *server_key;
-    if (!server_mode(get_options())) {
-      *errmsg = "Not running in server mode";
-      return -1;
-    }
-    server_key = get_server_identity_key();
-    *answer = tor_malloc(HEX_DIGEST_LEN+1);
-    crypto_pk_get_fingerprint(server_key, *answer, 0);
-  }
-  return 0;
-}
-
-/** Awful hack: return a newly allocated string based on a routerinfo and
- * (possibly) an extrainfo, sticking the read-history and write-history from
- * <b>ei</b> into the resulting string.  The thing you get back won't
- * necessarily have a valid signature.
- *
- * New code should never use this; it's for backward compatibility.
- *
- * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
- * not be NUL-terminated. */
-static char *
-munge_extrainfo_into_routerinfo(const char *ri_body,
-                                const signed_descriptor_t *ri,
-                                const signed_descriptor_t *ei)
-{
-  char *out = NULL, *outp;
-  int i;
-  const char *router_sig;
-  const char *ei_body = signed_descriptor_get_body(ei);
-  size_t ri_len = ri->signed_descriptor_len;
-  size_t ei_len = ei->signed_descriptor_len;
-  if (!ei_body)
-    goto bail;
-
-  outp = out = tor_malloc(ri_len+ei_len+1);
-  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
-    goto bail;
-  ++router_sig;
-  memcpy(out, ri_body, router_sig-ri_body);
-  outp += router_sig-ri_body;
-
-  for (i=0; i < 2; ++i) {
-    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
-    const char *cp, *eol;
-    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
-      continue;
-    ++cp;
-    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
-      continue;
-    memcpy(outp, cp, eol-cp+1);
-    outp += eol-cp+1;
-  }
-  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
-  *outp++ = '\0';
-  tor_assert(outp-out < (int)(ri_len+ei_len+1));
-
-  return out;
- bail:
-  tor_free(out);
-  return tor_strndup(ri_body, ri->signed_descriptor_len);
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * which ports are bound. */
-static int
-getinfo_helper_listeners(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  int type;
-  smartlist_t *res;
-
-  (void)control_conn;
-  (void)errmsg;
-
-  if (!strcmp(question, "net/listeners/or"))
-    type = CONN_TYPE_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/extor"))
-    type = CONN_TYPE_EXT_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/dir"))
-    type = CONN_TYPE_DIR_LISTENER;
-  else if (!strcmp(question, "net/listeners/socks"))
-    type = CONN_TYPE_AP_LISTENER;
-  else if (!strcmp(question, "net/listeners/trans"))
-    type = CONN_TYPE_AP_TRANS_LISTENER;
-  else if (!strcmp(question, "net/listeners/natd"))
-    type = CONN_TYPE_AP_NATD_LISTENER;
-  else if (!strcmp(question, "net/listeners/httptunnel"))
-    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
-  else if (!strcmp(question, "net/listeners/dns"))
-    type = CONN_TYPE_AP_DNS_LISTENER;
-  else if (!strcmp(question, "net/listeners/control"))
-    type = CONN_TYPE_CONTROL_LISTENER;
-  else
-    return 0; /* unknown key */
-
-  res = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
-    struct sockaddr_storage ss;
-    socklen_t ss_len = sizeof(ss);
-
-    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
-      continue;
-
-    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
-      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
-    } else {
-      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
-      smartlist_add(res, esc_for_log(tmp));
-      tor_free(tmp);
-    }
-
-  } SMARTLIST_FOREACH_END(conn);
-
-  *answer = smartlist_join_strings(res, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
-  smartlist_free(res);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * the current time in both local and UTC forms. */
-STATIC int
-getinfo_helper_current_time(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-
-  struct timeval now;
-  tor_gettimeofday(&now);
-  char timebuf[ISO_TIME_LEN+1];
-
-  if (!strcmp(question, "current-time/local"))
-    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else if (!strcmp(question, "current-time/utc"))
-    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else
-    return 0;
-
-  *answer = tor_strdup(timebuf);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * directory information. */
-STATIC int
-getinfo_helper_dir(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  (void) control_conn;
-  if (!strcmpstart(question, "desc/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/id/* "
-                "instead of desc/id/*.";
-      return 0;
-    }
-  } else if (!strcmpstart(question, "desc/name/")) {
-    const routerinfo_t *ri = NULL;
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node =
-      node_get_by_nickname(question+strlen("desc/name/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/name/* "
-                "instead of desc/name/*.";
-      return 0;
-    }
-  } else if (!strcmp(question, "desc/download-enabled")) {
-    int r = we_fetch_router_descriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmp(question, "desc/all-recent")) {
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
-      {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        if (body)
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-      });
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
-    /* XXXX Remove this once Torstat asks for extrainfos. */
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
-                                     ri->cache_info.extra_info_digest);
-        if (ei && body) {
-          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
-                                                        &ri->cache_info, ei));
-        } else if (body) {
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-        }
-      } SMARTLIST_FOREACH_END(ri);
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/client/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      rend_cache_entry_t *e = NULL;
-      if (!rend_cache_lookup_entry(question, -1, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      const char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_cache_lookup_encoded_as_client(&service_pk);
-      if (desc) {
-        *answer = tor_strdup(desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/service/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-    rend_cache_entry_t *e = NULL;
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_service_lookup_current_desc(&service_pk);
-      if (desc) {
-        /* Newly allocated string, we have ownership. */
-        *answer = desc;
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "md/all")) {
-    const smartlist_t *nodes = nodelist_get_list();
-    tor_assert(nodes);
-
-    if (smartlist_len(nodes) == 0) {
-      *answer = tor_strdup("");
-      return 0;
-    }
-
-    smartlist_t *microdescs = smartlist_new();
-
-    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
-      if (n->md && n->md->body) {
-        char *copy = tor_strndup(n->md->body, n->md->bodylen);
-        smartlist_add(microdescs, copy);
-      }
-    } SMARTLIST_FOREACH_END(n);
-
-    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
-    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
-    smartlist_free(microdescs);
-  } else if (!strcmpstart(question, "md/id/")) {
-    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmpstart(question, "md/name/")) {
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
-    /* XXXX duplicated code */
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmp(question, "md/download-enabled")) {
-    int r = we_fetch_microdescriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmpstart(question, "desc-annotations/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node =
-      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *annotations =
-        signed_descriptor_get_annotations(&ri->cache_info);
-      if (annotations)
-        *answer = tor_strndup(annotations,
-                              ri->cache_info.annotations_len);
-    }
-  } else if (!strcmpstart(question, "dir/server/")) {
-    size_t answer_len = 0;
-    char *url = NULL;
-    smartlist_t *descs = smartlist_new();
-    const char *msg;
-    int res;
-    char *cp;
-    tor_asprintf(&url, "/tor/%s", question+4);
-    res = dirserv_get_routerdescs(descs, url, &msg);
-    if (res) {
-      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
-      smartlist_free(descs);
-      tor_free(url);
-      *errmsg = msg;
-      return -1;
-    }
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      answer_len += sd->signed_descriptor_len);
-    cp = *answer = tor_malloc(answer_len+1);
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      {
-                        memcpy(cp, signed_descriptor_get_body(sd),
-                               sd->signed_descriptor_len);
-                        cp += sd->signed_descriptor_len;
-                      });
-    *cp = '\0';
-    tor_free(url);
-    smartlist_free(descs);
-  } else if (!strcmpstart(question, "dir/status/")) {
-    *answer = tor_strdup("");
-  } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
-    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
-      const cached_dir_t *consensus = dirserv_get_consensus("ns");
-      if (consensus)
-        *answer = tor_strdup(consensus->dir);
-    }
-    if (!*answer) { /* try loading it from disk */
-      *answer = networkstatus_read_cached_consensus("ns");
-      if (!*answer) { /* generate an error */
-        *errmsg = "Could not open cached consensus. "
-          "Make sure FetchUselessDescriptors is set to 1.";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "network-status")) { /* v1 */
-    static int network_status_warned = 0;
-    if (!network_status_warned) {
-      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
-               "go away in a future version of Tor.");
-      network_status_warned = 1;
-    }
-    routerlist_t *routerlist = router_get_routerlist();
-    if (!routerlist || !routerlist->routers ||
-        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
-      return -1;
-    }
-  } else if (!strcmpstart(question, "extra-info/digest/")) {
-    question += strlen("extra-info/digest/");
-    if (strlen(question) == HEX_DIGEST_LEN) {
-      char d[DIGEST_LEN];
-      signed_descriptor_t *sd = NULL;
-      if (base16_decode(d, sizeof(d), question, strlen(question))
-                        == sizeof(d)) {
-        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
-         * but I don't want to risk affecting other parts of the code,
-         * especially since the rules for using our own extrainfo (including
-         * when it might be freed) are different from those for using one
-         * we have downloaded. */
-        if (router_extrainfo_digest_is_me(d))
-          sd = &(router_get_my_extrainfo()->cache_info);
-        else
-          sd = extrainfo_get_by_descriptor_digest(d);
-      }
-      if (sd) {
-        const char *body = signed_descriptor_get_body(sd);
-        if (body)
-          *answer = tor_strndup(body, sd->signed_descriptor_len);
-      }
-    }
-  }
-
-  return 0;
-}
-
-/** Given a smartlist of 20-byte digests, return a newly allocated string
- * containing each of those digests in order, formatted in HEX, and terminated
- * with a newline. */
-static char *
-digest_list_to_string(const smartlist_t *sl)
-{
-  int len;
-  char *result, *s;
-
-  /* Allow for newlines, and a \0 at the end */
-  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
-  result = tor_malloc_zero(len);
-
-  s = result;
-  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
-    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
-    s[HEX_DIGEST_LEN] = '\n';
-    s += HEX_DIGEST_LEN + 1;
-  } SMARTLIST_FOREACH_END(digest);
-  *s = '\0';
-
-  return result;
-}
-
-/** Turn a download_status_t into a human-readable description in a newly
- * allocated string.  The format is specified in control-spec.txt, under
- * the documentation for "GETINFO download/..." .  */
-static char *
-download_status_to_string(const download_status_t *dl)
-{
-  char *rv = NULL;
-  char tbuf[ISO_TIME_LEN+1];
-  const char *schedule_str, *want_authority_str;
-  const char *increment_on_str, *backoff_str;
-
-  if (dl) {
-    /* Get some substrings of the eventual output ready */
-    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
-
-    switch (dl->schedule) {
-      case DL_SCHED_GENERIC:
-        schedule_str = "DL_SCHED_GENERIC";
-        break;
-      case DL_SCHED_CONSENSUS:
-        schedule_str = "DL_SCHED_CONSENSUS";
-        break;
-      case DL_SCHED_BRIDGE:
-        schedule_str = "DL_SCHED_BRIDGE";
-        break;
-      default:
-        schedule_str = "unknown";
-        break;
-    }
-
-    switch (dl->want_authority) {
-      case DL_WANT_ANY_DIRSERVER:
-        want_authority_str = "DL_WANT_ANY_DIRSERVER";
-        break;
-      case DL_WANT_AUTHORITY:
-        want_authority_str = "DL_WANT_AUTHORITY";
-        break;
-      default:
-        want_authority_str = "unknown";
-        break;
-    }
-
-    switch (dl->increment_on) {
-      case DL_SCHED_INCREMENT_FAILURE:
-        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
-        break;
-      case DL_SCHED_INCREMENT_ATTEMPT:
-        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
-        break;
-      default:
-        increment_on_str = "unknown";
-        break;
-    }
-
-    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
-
-    /* Now assemble them */
-    tor_asprintf(&rv,
-                 "next-attempt-at %s\n"
-                 "n-download-failures %u\n"
-                 "n-download-attempts %u\n"
-                 "schedule %s\n"
-                 "want-authority %s\n"
-                 "increment-on %s\n"
-                 "backoff %s\n"
-                 "last-backoff-position %u\n"
-                 "last-delay-used %d\n",
-                 tbuf,
-                 dl->n_download_failures,
-                 dl->n_download_attempts,
-                 schedule_str,
-                 want_authority_str,
-                 increment_on_str,
-                 backoff_str,
-                 dl->last_backoff_position,
-                 dl->last_delay_used);
-  }
-
-  return rv;
-}
-
-/** Handle the consensus download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_networkstatus(const char *flavor,
-                                       download_status_t **dl_to_emit,
-                                       const char **errmsg)
-{
-  /*
-   * We get the one for the current bootstrapped status by default, or
-   * take an extra /bootstrap or /running suffix
-   */
-  if (strcmp(flavor, "ns") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
-  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
-  } else if (strcmp(flavor, "ns/running") == 0 ) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
-  } else if (strcmp(flavor, "microdesc") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/running") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
-  } else {
-    *errmsg = "Unknown flavor";
-  }
-}
-
-/** Handle the cert download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_cert(const char *fp_sk_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  const char *sk_req;
-  char id_digest[DIGEST_LEN];
-  char sk_digest[DIGEST_LEN];
-
-  /*
-   * We have to handle four cases; fp_sk_req is the request with
-   * a prefix of "downloads/cert/" snipped off.
-   *
-   * Case 1: fp_sk_req = "fps"
-   *  - We should emit a digest_list with a list of all the identity
-   *    fingerprints that can be queried for certificate download status;
-   *    get it by calling list_authority_ids_with_downloads().
-   *
-   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
-   *  - We want the default certificate for this identity fingerprint's
-   *    download status; this is the download we get from URLs starting
-   *    in /fp/ on the directory server.  We can get it with
-   *    id_only_download_status_for_authority_id().
-   *
-   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
-   *  - We want a list of all signing key digests for this identity
-   *    fingerprint which can be queried for certificate download status.
-   *    Get it with list_sk_digests_for_authority_id().
-   *
-   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
-   *         signing key digest sk
-   *   - We want the download status for the certificate for this specific
-   *     signing key and fingerprint.  These correspond to the ones we get
-   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
-   *     list_sk_digests_for_authority_id().
-   */
-
-  if (strcmp(fp_sk_req, "fps") == 0) {
-    *digest_list = list_authority_ids_with_downloads();
-    if (!(*digest_list)) {
-      *errmsg = "Failed to get list of authority identity digests (!)";
-    }
-  } else if (!strcmpstart(fp_sk_req, "fp/")) {
-    fp_sk_req += strlen("fp/");
-    /* Okay, look for another / to tell the fp from fp-sk cases */
-    sk_req = strchr(fp_sk_req, '/');
-    if (sk_req) {
-      /* okay, split it here and try to parse <fp> */
-      if (base16_decode(id_digest, DIGEST_LEN,
-                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
-        /* Skip past the '/' */
-        ++sk_req;
-        if (strcmp(sk_req, "sks") == 0) {
-          /* We're asking for the list of signing key fingerprints */
-          *digest_list = list_sk_digests_for_authority_id(id_digest);
-          if (!(*digest_list)) {
-            *errmsg = "Failed to get list of signing key digests for this "
-                      "authority identity digest";
-          }
-        } else {
-          /* We've got a signing key digest */
-          if (base16_decode(sk_digest, DIGEST_LEN,
-                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
-            *dl_to_emit =
-              download_status_for_authority_id_and_sk(id_digest, sk_digest);
-            if (!(*dl_to_emit)) {
-              *errmsg = "Failed to get download status for this identity/"
-                        "signing key digest pair";
-            }
-          } else {
-            *errmsg = "That didn't look like a signing key digest";
-          }
-        }
-      } else {
-        *errmsg = "That didn't look like an identity digest";
-      }
-    } else {
-      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
-      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
-        if (base16_decode(id_digest, DIGEST_LEN,
-                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
-          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
-          if (!(*dl_to_emit)) {
-            *errmsg = "Failed to get download status for this authority "
-                      "identity digest";
-          }
-        } else {
-          *errmsg = "That didn't look like a digest";
-        }
-      } else {
-        *errmsg = "That didn't look like a digest";
-      }
-    }
-  } else {
-    *errmsg = "Unknown certificate download status query";
-  }
-}
-
-/** Handle the routerdesc download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_desc(const char *desc_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  char desc_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: desc_req = "descs"
-   *   - Emit a list of all router descriptor digests, which we get by
-   *     calling router_get_descriptor_digests(); this can return NULL
-   *     if we have no current ns-flavor consensus.
-   *
-   * Case 2: desc_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using router_get_dl_status_by_descriptor_digest().
-   */
-
-  if (strcmp(desc_req, "descs") == 0) {
-    *digest_list = router_get_descriptor_digests();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
-    }
-    /*
-     * Microdescs don't use the download_status_t mechanism, so we don't
-     * answer queries about their downloads here; see microdesc.c.
-     */
-  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(desc_digest, DIGEST_LEN,
-                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such descriptor digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown router descriptor download status query";
-  }
-}
-
-/** Handle the bridge download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_bridge(const char *bridge_req,
-                                download_status_t **dl_to_emit,
-                                smartlist_t **digest_list,
-                                const char **errmsg)
-{
-  char bridge_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: bridge_req = "bridges"
-   *   - Emit a list of all bridge identity digests, which we get by
-   *     calling list_bridge_identities(); this can return NULL if we are
-   *     not using bridges.
-   *
-   * Case 2: bridge_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using get_bridge_dl_status_by_id().
-   */
-
-  if (strcmp(bridge_req, "bridges") == 0) {
-    *digest_list = list_bridge_identities();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to be using bridges";
-    }
-  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(bridge_digest, DIGEST_LEN,
-                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such bridge identity digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown bridge descriptor download status query";
-  }
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * download status information. */
-STATIC int
-getinfo_helper_downloads(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  download_status_t *dl_to_emit = NULL;
-  smartlist_t *digest_list = NULL;
-
-  /* Assert args are sane */
-  tor_assert(control_conn != NULL);
-  tor_assert(question != NULL);
-  tor_assert(answer != NULL);
-  tor_assert(errmsg != NULL);
-
-  /* We check for this later to see if we should supply a default */
-  *errmsg = NULL;
-
-  /* Are we after networkstatus downloads? */
-  if (!strcmpstart(question, "downloads/networkstatus/")) {
-    getinfo_helper_downloads_networkstatus(
-        question + strlen("downloads/networkstatus/"),
-        &dl_to_emit, errmsg);
-  /* Certificates? */
-  } else if (!strcmpstart(question, "downloads/cert/")) {
-    getinfo_helper_downloads_cert(
-        question + strlen("downloads/cert/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Router descriptors? */
-  } else if (!strcmpstart(question, "downloads/desc/")) {
-    getinfo_helper_downloads_desc(
-        question + strlen("downloads/desc/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Bridge descriptors? */
-  } else if (!strcmpstart(question, "downloads/bridge/")) {
-    getinfo_helper_downloads_bridge(
-        question + strlen("downloads/bridge/"),
-        &dl_to_emit, &digest_list, errmsg);
-  } else {
-    *errmsg = "Unknown download status query";
-  }
-
-  if (dl_to_emit) {
-    *answer = download_status_to_string(dl_to_emit);
-
-    return 0;
-  } else if (digest_list) {
-    *answer = digest_list_to_string(digest_list);
-    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
-    smartlist_free(digest_list);
-
-    return 0;
-  } else {
-    if (!(*errmsg)) {
-      *errmsg = "Unknown error";
-    }
-
-    return -1;
-  }
-}
-
-/** Allocate and return a description of <b>circ</b>'s current status,
- * including its path (if any). */
-static char *
-circuit_describe_status_for_controller(origin_circuit_t *circ)
-{
-  char *rv;
-  smartlist_t *descparts = smartlist_new();
-
-  {
-    char *vpath = circuit_list_path_for_controller(circ);
-    if (*vpath) {
-      smartlist_add(descparts, vpath);
-    } else {
-      tor_free(vpath); /* empty path; don't put an extra space in the result */
-    }
-  }
-
-  {
-    cpath_build_state_t *build_state = circ->build_state;
-    smartlist_t *flaglist = smartlist_new();
-    char *flaglist_joined;
-
-    if (build_state->onehop_tunnel)
-      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
-    if (build_state->is_internal)
-      smartlist_add(flaglist, (void *)"IS_INTERNAL");
-    if (build_state->need_capacity)
-      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
-    if (build_state->need_uptime)
-      smartlist_add(flaglist, (void *)"NEED_UPTIME");
-
-    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
-    if (smartlist_len(flaglist)) {
-      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
-
-      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
-
-      tor_free(flaglist_joined);
-    }
-
-    smartlist_free(flaglist);
-  }
-
-  smartlist_add_asprintf(descparts, "PURPOSE=%s",
-                    circuit_purpose_to_controller_string(circ->base_.purpose));
-
-  {
-    const char *hs_state =
-      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
-
-    if (hs_state != NULL) {
-      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
-    }
-  }
-
-  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
-    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
-    const char *onion_address;
-    if (circ->rend_data) {
-      onion_address = rend_data_get_address(circ->rend_data);
-    } else {
-      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
-      onion_address = addr;
-    }
-    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
-  }
-
-  {
-    char tbuf[ISO_TIME_USEC_LEN+1];
-    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
-
-    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
-  }
-
-  // Show username and/or password if available.
-  if (circ->socks_username_len > 0) {
-    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
-                                     (size_t) circ->socks_username_len);
-    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
-                           socks_username_escaped);
-    tor_free(socks_username_escaped);
-  }
-  if (circ->socks_password_len > 0) {
-    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
-                                     (size_t) circ->socks_password_len);
-    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
-                           socks_password_escaped);
-    tor_free(socks_password_escaped);
-  }
-
-  rv = smartlist_join_strings(descparts, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
-  smartlist_free(descparts);
-
-  return rv;
-}
-
-/** Implementation helper for GETINFO: knows how to generate summaries of the
- * current states of things we send events about. */
-static int
-getinfo_helper_events(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  const or_options_t *options = get_options();
-  (void) control_conn;
-  if (!strcmp(question, "circuit-status")) {
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
-      origin_circuit_t *circ;
-      char *circdesc;
-      const char *state;
-      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
-        continue;
-      circ = TO_ORIGIN_CIRCUIT(circ_);
-
-      if (circ->base_.state == CIRCUIT_STATE_OPEN)
-        state = "BUILT";
-      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
-        state = "GUARD_WAIT";
-      else if (circ->cpath)
-        state = "EXTENDED";
-      else
-        state = "LAUNCHED";
-
-      circdesc = circuit_describe_status_for_controller(circ);
-
-      smartlist_add_asprintf(status, "%lu %s%s%s",
-                   (unsigned long)circ->global_identifier,
-                   state, *circdesc ? " " : "", circdesc);
-      tor_free(circdesc);
-    }
-    SMARTLIST_FOREACH_END(circ_);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "stream-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    char buf[256];
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      entry_connection_t *conn;
-      circuit_t *circ;
-      origin_circuit_t *origin_circ = NULL;
-      if (base_conn->type != CONN_TYPE_AP ||
-          base_conn->marked_for_close ||
-          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
-          base_conn->state == AP_CONN_STATE_NATD_WAIT)
-        continue;
-      conn = TO_ENTRY_CONN(base_conn);
-      switch (base_conn->state)
-        {
-        case AP_CONN_STATE_CONTROLLER_WAIT:
-        case AP_CONN_STATE_CIRCUIT_WAIT:
-          if (conn->socks_request &&
-              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
-            state = "NEWRESOLVE";
-          else
-            state = "NEW";
-          break;
-        case AP_CONN_STATE_RENDDESC_WAIT:
-        case AP_CONN_STATE_CONNECT_WAIT:
-          state = "SENTCONNECT"; break;
-        case AP_CONN_STATE_RESOLVE_WAIT:
-          state = "SENTRESOLVE"; break;
-        case AP_CONN_STATE_OPEN:
-          state = "SUCCEEDED"; break;
-        default:
-          log_warn(LD_BUG, "Asked for stream in unknown state %d",
-                   base_conn->state);
-          continue;
-        }
-      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-      if (circ && CIRCUIT_IS_ORIGIN(circ))
-        origin_circ = TO_ORIGIN_CIRCUIT(circ);
-      write_stream_target_to_buf(conn, buf, sizeof(buf));
-      smartlist_add_asprintf(status, "%lu %s %lu %s",
-                   (unsigned long) base_conn->global_identifier,state,
-                   origin_circ?
-                         (unsigned long)origin_circ->global_identifier : 0ul,
-                   buf);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "orconn-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      char name[128];
-      or_connection_t *conn;
-      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
-        continue;
-      conn = TO_OR_CONN(base_conn);
-      if (conn->base_.state == OR_CONN_STATE_OPEN)
-        state = "CONNECTED";
-      else if (conn->nickname)
-        state = "LAUNCHED";
-      else
-        state = "NEW";
-      orconn_target_get_name(name, sizeof(name), conn);
-      smartlist_add_asprintf(status, "%s %s", name, state);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmpstart(question, "address-mappings/")) {
-    time_t min_e, max_e;
-    smartlist_t *mappings;
-    question += strlen("address-mappings/");
-    if (!strcmp(question, "all")) {
-      min_e = 0; max_e = TIME_MAX;
-    } else if (!strcmp(question, "cache")) {
-      min_e = 2; max_e = TIME_MAX;
-    } else if (!strcmp(question, "config")) {
-      min_e = 0; max_e = 0;
-    } else if (!strcmp(question, "control")) {
-      min_e = 1; max_e = 1;
-    } else {
-      return 0;
-    }
-    mappings = smartlist_new();
-    addressmap_get_mappings(mappings, min_e, max_e, 1);
-    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
-    smartlist_free(mappings);
-  } else if (!strcmpstart(question, "status/")) {
-    /* Note that status/ is not a catch-all for events; there's only supposed
-     * to be a status GETINFO if there's a corresponding STATUS event. */
-    if (!strcmp(question, "status/circuit-established")) {
-      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
-    } else if (!strcmp(question, "status/enough-dir-info")) {
-      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
-    } else if (!strcmp(question, "status/good-server-descriptor") ||
-               !strcmp(question, "status/accepted-server-descriptor")) {
-      /* They're equivalent for now, until we can figure out how to make
-       * good-server-descriptor be what we want. See comment in
-       * control-spec.txt. */
-      *answer = tor_strdup(directories_have_accepted_server_descriptor()
-                           ? "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
-      *answer = tor_strdup(check_whether_orport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
-      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded")) {
-      tor_asprintf(answer, "OR=%d DIR=%d",
-                   check_whether_orport_reachable(options) ? 1 : 0,
-                   check_whether_dirport_reachable(options) ? 1 : 0);
-    } else if (!strcmp(question, "status/bootstrap-phase")) {
-      *answer = tor_strdup(last_sent_bootstrap_message);
-    } else if (!strcmpstart(question, "status/version/")) {
-      int is_server = server_mode(options);
-      networkstatus_t *c = networkstatus_get_latest_consensus();
-      version_status_t status;
-      const char *recommended;
-      if (c) {
-        recommended = is_server ? c->server_versions : c->client_versions;
-        status = tor_version_is_obsolete(VERSION, recommended);
-      } else {
-        recommended = "?";
-        status = VS_UNKNOWN;
-      }
-
-      if (!strcmp(question, "status/version/recommended")) {
-        *answer = tor_strdup(recommended);
-        return 0;
-      }
-      if (!strcmp(question, "status/version/current")) {
-        switch (status)
-          {
-          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
-          case VS_OLD: *answer = tor_strdup("obsolete"); break;
-          case VS_NEW: *answer = tor_strdup("new"); break;
-          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
-          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
-          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
-          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
-          default: tor_fragile_assert();
-          }
-      } else if (!strcmp(question, "status/version/num-versioning") ||
-                 !strcmp(question, "status/version/num-concurring")) {
-        tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO));
-        log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful "
-                 "information", question);
-      }
-    } else if (!strcmp(question, "status/clients-seen")) {
-      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
-      if (!bridge_stats) {
-        *errmsg = "No bridge-client stats available";
-        return -1;
-      }
-      *answer = bridge_stats;
-    } else if (!strcmp(question, "status/fresh-relay-descs")) {
-      if (!server_mode(options)) {
-        *errmsg = "Only relays have descriptors";
-        return -1;
-      }
-      routerinfo_t *r;
-      extrainfo_t *e;
-      if (router_build_fresh_descriptor(&r, &e) < 0) {
-        *errmsg = "Error generating descriptor";
-        return -1;
-      }
-      size_t size = r->cache_info.signed_descriptor_len + 1;
-      if (e) {
-        size += e->cache_info.signed_descriptor_len + 1;
-      }
-      tor_assert(r->cache_info.signed_descriptor_len);
-      char *descs = tor_malloc(size);
-      char *cp = descs;
-      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
-             r->cache_info.signed_descriptor_len);
-      cp += r->cache_info.signed_descriptor_len - 1;
-      if (e) {
-        if (cp[0] == '\0') {
-          cp[0] = '\n';
-        } else if (cp[0] != '\n') {
-          cp[1] = '\n';
-          cp++;
-        }
-        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
-               e->cache_info.signed_descriptor_len);
-        cp += e->cache_info.signed_descriptor_len - 1;
-      }
-      if (cp[0] == '\n') {
-        cp[0] = '\0';
-      } else if (cp[0] != '\0') {
-        cp[1] = '\0';
-      }
-      *answer = descs;
-      routerinfo_free(r);
-      extrainfo_free(e);
-    } else {
-      return 0;
-    }
-  }
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows how to enumerate hidden services
- * created via the control port. */
-STATIC int
-getinfo_helper_onions(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  smartlist_t *onion_list = NULL;
-  (void) errmsg;  /* no errors from this method */
-
-  if (control_conn && !strcmp(question, "onions/current")) {
-    onion_list = control_conn->ephemeral_onion_services;
-  } else if (!strcmp(question, "onions/detached")) {
-    onion_list = detached_onion_services;
-  } else {
-    return 0;
-  }
-  if (!onion_list || smartlist_len(onion_list) == 0) {
-    if (answer) {
-      *answer = tor_strdup("");
-    }
-  } else {
-    if (answer) {
-      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about network
- * liveness. */
-static int
-getinfo_helper_liveness(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-  if (strcmp(question, "network-liveness") == 0) {
-    if (get_cached_network_liveness()) {
-      *answer = tor_strdup("up");
-    } else {
-      *answer = tor_strdup("down");
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about shared random
- * value. */
-static int
-getinfo_helper_sr(control_connection_t *control_conn,
-                  const char *question, char **answer,
-                  const char **errmsg)
-{
-  (void) control_conn;
-  (void) errmsg;
-
-  if (!strcmp(question, "sr/current")) {
-    *answer = sr_get_current_for_control();
-  } else if (!strcmp(question, "sr/previous")) {
-    *answer = sr_get_previous_for_control();
-  }
-  /* Else statement here is unrecognized key so do nothing. */
-
-  return 0;
-}
-
-/** Callback function for GETINFO: on a given control connection, try to
- * answer the question <b>q</b> and store the newly-allocated answer in
- * *<b>a</b>. If an internal error occurs, return -1 and optionally set
- * *<b>error_out</b> to point to an error message to be delivered to the
- * controller. On success, _or if the key is not recognized_, return 0. Do not
- * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
- * to improve the error message.
- */
-typedef int (*getinfo_helper_t)(control_connection_t *,
-                                const char *q, char **a,
-                                const char **error_out);
-
-/** A single item for the GETINFO question-to-answer-function table. */
-typedef struct getinfo_item_t {
-  const char *varname; /**< The value (or prefix) of the question. */
-  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
-                        * this entry is documentation-only. */
-  const char *desc; /**< Description of the variable. */
-  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
-} getinfo_item_t;
-
-#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
-#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
-#define DOC(name, desc) { name, NULL, desc, 0 }
-
-/** Table mapping questions accepted by GETINFO to the functions that know how
- * to answer them. */
-static const getinfo_item_t getinfo_items[] = {
-  ITEM("version", misc, "The current version of Tor."),
-  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
-  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
-  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
-  ITEM("config-text", misc,
-       "Return the string that would be written by a saveconf command."),
-  ITEM("config-can-saveconf", misc,
-       "Is it possible to save the configuration to the \"torrc\" file?"),
-  ITEM("accounting/bytes", accounting,
-       "Number of bytes read/written so far in the accounting interval."),
-  ITEM("accounting/bytes-left", accounting,
-      "Number of bytes left to write/read so far in the accounting interval."),
-  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
-  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
-  ITEM("accounting/interval-start", accounting,
-       "Time when the accounting period starts."),
-  ITEM("accounting/interval-end", accounting,
-       "Time when the accounting period ends."),
-  ITEM("accounting/interval-wake", accounting,
-       "Time to wake up in this accounting period."),
-  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
-  ITEM("entry-guards", entry_guards,
-       "Which nodes are we using as entry guards?"),
-  ITEM("fingerprint", misc, NULL),
-  PREFIX("config/", config, "Current configuration values."),
-  DOC("config/names",
-      "List of configuration options, types, and documentation."),
-  DOC("config/defaults",
-      "List of default values for configuration options. "
-      "See also config/names"),
-  PREFIX("current-time/", current_time, "Current time."),
-  DOC("current-time/local", "Current time on the local system."),
-  DOC("current-time/utc", "Current UTC time."),
-  PREFIX("downloads/networkstatus/", downloads,
-         "Download statuses for networkstatus objects"),
-  DOC("downloads/networkstatus/ns",
-      "Download status for current-mode networkstatus download"),
-  DOC("downloads/networkstatus/ns/bootstrap",
-      "Download status for bootstrap-time networkstatus download"),
-  DOC("downloads/networkstatus/ns/running",
-      "Download status for run-time networkstatus download"),
-  DOC("downloads/networkstatus/microdesc",
-      "Download status for current-mode microdesc download"),
-  DOC("downloads/networkstatus/microdesc/bootstrap",
-      "Download status for bootstrap-time microdesc download"),
-  DOC("downloads/networkstatus/microdesc/running",
-      "Download status for run-time microdesc download"),
-  PREFIX("downloads/cert/", downloads,
-         "Download statuses for certificates, by id fingerprint and "
-         "signing key"),
-  DOC("downloads/cert/fps",
-      "List of authority fingerprints for which any download statuses "
-      "exist"),
-  DOC("downloads/cert/fp/<fp>",
-      "Download status for <fp> with the default signing key; corresponds "
-      "to /fp/ URLs on directory server."),
-  DOC("downloads/cert/fp/<fp>/sks",
-      "List of signing keys for which specific download statuses are "
-      "available for this id fingerprint"),
-  DOC("downloads/cert/fp/<fp>/<sk>",
-      "Download status for <fp> with signing key <sk>; corresponds "
-      "to /fp-sk/ URLs on directory server."),
-  PREFIX("downloads/desc/", downloads,
-         "Download statuses for router descriptors, by descriptor digest"),
-  DOC("downloads/desc/descs",
-      "Return a list of known router descriptor digests"),
-  DOC("downloads/desc/<desc>",
-      "Return a download status for a given descriptor digest"),
-  PREFIX("downloads/bridge/", downloads,
-         "Download statuses for bridge descriptors, by bridge identity "
-         "digest"),
-  DOC("downloads/bridge/bridges",
-      "Return a list of configured bridge identity digests with download "
-      "statuses"),
-  DOC("downloads/bridge/<desc>",
-      "Return a download status for a given bridge identity digest"),
-  ITEM("info/names", misc,
-       "List of GETINFO options, types, and documentation."),
-  ITEM("events/names", misc,
-       "Events that the controller can ask for with SETEVENTS."),
-  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
-  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
-  PREFIX("desc/id/", dir, "Router descriptors by ID."),
-  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
-  ITEM("desc/all-recent", dir,
-       "All non-expired, non-superseded router descriptors."),
-  ITEM("desc/download-enabled", dir,
-       "Do we try to download router descriptors?"),
-  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
-  ITEM("md/all", dir, "All known microdescriptors."),
-  PREFIX("md/id/", dir, "Microdescriptors by ID"),
-  PREFIX("md/name/", dir, "Microdescriptors by name"),
-  ITEM("md/download-enabled", dir,
-       "Do we try to download microdescriptors?"),
-  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
-  PREFIX("hs/client/desc/id", dir,
-         "Hidden Service descriptor in client's cache by onion."),
-  PREFIX("hs/service/desc/id/", dir,
-         "Hidden Service descriptor in services's cache by onion."),
-  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
-  ITEM("ns/all", networkstatus,
-       "Brief summary of router status (v2 directory format)"),
-  PREFIX("ns/id/", networkstatus,
-         "Brief summary of router status by ID (v2 directory format)."),
-  PREFIX("ns/name/", networkstatus,
-         "Brief summary of router status by nickname (v2 directory format)."),
-  PREFIX("ns/purpose/", networkstatus,
-         "Brief summary of router status by purpose (v2 directory format)."),
-  PREFIX("consensus/", networkstatus,
-         "Information about and from the ns consensus."),
-  ITEM("network-status", dir,
-       "Brief summary of router status (v1 directory format)"),
-  ITEM("network-liveness", liveness,
-       "Current opinion on whether the network is live"),
-  ITEM("circuit-status", events, "List of current circuits originating here."),
-  ITEM("stream-status", events,"List of current streams."),
-  ITEM("orconn-status", events, "A list of current OR connections."),
-  ITEM("dormant", misc,
-       "Is Tor dormant (not building circuits because it's idle)?"),
-  PREFIX("address-mappings/", events, NULL),
-  DOC("address-mappings/all", "Current address mappings."),
-  DOC("address-mappings/cache", "Current cached DNS replies."),
-  DOC("address-mappings/config",
-      "Current address mappings from configuration."),
-  DOC("address-mappings/control", "Current address mappings from controller."),
-  PREFIX("status/", events, NULL),
-  DOC("status/circuit-established",
-      "Whether we think client functionality is working."),
-  DOC("status/enough-dir-info",
-      "Whether we have enough up-to-date directory information to build "
-      "circuits."),
-  DOC("status/bootstrap-phase",
-      "The last bootstrap phase status event that Tor sent."),
-  DOC("status/clients-seen",
-      "Breakdown of client countries seen by a bridge."),
-  DOC("status/fresh-relay-descs",
-      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
-  DOC("status/version/recommended", "List of currently recommended versions."),
-  DOC("status/version/current", "Status of the current version."),
-  DOC("status/version/num-versioning", "Number of versioning authorities."),
-  DOC("status/version/num-concurring",
-      "Number of versioning authorities agreeing on the status of the "
-      "current version"),
-  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
-  ITEM("traffic/read", misc,"Bytes read since the process was started."),
-  ITEM("traffic/written", misc,
-       "Bytes written since the process was started."),
-  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
-  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
-  ITEM("process/uid", misc, "User id running the tor process."),
-  ITEM("process/user", misc,
-       "Username under which the tor process is running."),
-  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
-  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
-  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
-  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
-  PREFIX("dir/status/", dir,
-         "v2 networkstatus docs as retrieved from a DirPort."),
-  ITEM("dir/status-vote/current/consensus", dir,
-       "v3 Networkstatus consensus as retrieved from a DirPort."),
-  ITEM("exit-policy/default", policies,
-       "The default value appended to the configured exit policy."),
-  ITEM("exit-policy/reject-private/default", policies,
-       "The default rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate."),
-  ITEM("exit-policy/reject-private/relay", policies,
-       "The relay-specific rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
-  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
-  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
-  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
-  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
-  ITEM("onions/current", onions,
-       "Onion services owned by the current control connection."),
-  ITEM("onions/detached", onions,
-       "Onion services detached from the control connection."),
-  ITEM("sr/current", sr, "Get current shared random value."),
-  ITEM("sr/previous", sr, "Get previous shared random value."),
-  { NULL, NULL, NULL, 0 }
-};
-
-/** Allocate and return a list of recognized GETINFO options. */
-static char *
-list_getinfo_options(void)
-{
-  int i;
-  smartlist_t *lines = smartlist_new();
-  char *ans;
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    if (!getinfo_items[i].desc)
-      continue;
-
-    smartlist_add_asprintf(lines, "%s%s -- %s\n",
-                 getinfo_items[i].varname,
-                 getinfo_items[i].is_prefix ? "*" : "",
-                 getinfo_items[i].desc);
-  }
-  smartlist_sort_strings(lines);
-
-  ans = smartlist_join_strings(lines, "", 0, NULL);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-
-  return ans;
-}
-
-/** Lookup the 'getinfo' entry <b>question</b>, and return
- * the answer in <b>*answer</b> (or NULL if key not recognized).
- * Return 0 if success or unrecognized, or -1 if recognized but
- * internal error. */
-static int
-handle_getinfo_helper(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **err_out)
-{
-  int i;
-  *answer = NULL; /* unrecognized key by default */
-
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    int match;
-    if (getinfo_items[i].is_prefix)
-      match = !strcmpstart(question, getinfo_items[i].varname);
-    else
-      match = !strcmp(question, getinfo_items[i].varname);
-    if (match) {
-      tor_assert(getinfo_items[i].fn);
-      return getinfo_items[i].fn(control_conn, question, answer, err_out);
-    }
-  }
-
-  return 0; /* unrecognized */
-}
-
-/** Called when we receive a GETINFO command.  Try to fetch all requested
- * information, and reply with information or error message. */
-static int
-handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *ans = NULL;
-  int i;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    const char *errmsg = NULL;
-
-    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
-      if (!errmsg)
-        errmsg = "Internal error";
-      connection_printf_to_buf(conn, "551 %s\r\n", errmsg);
-      goto done;
-    }
-    if (!ans) {
-      if (errmsg) /* use provided error message */
-        smartlist_add_strdup(unrecognized, errmsg);
-      else /* use default error message */
-        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
-    } else {
-      smartlist_add_strdup(answers, q);
-      smartlist_add(answers, ans);
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if (smartlist_len(unrecognized)) {
-    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
-    for (i=0; i < smartlist_len(unrecognized)-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-%s\r\n",
-                               (char *)smartlist_get(unrecognized, i));
-
-    connection_printf_to_buf(conn,
-                             "552 %s\r\n",
-                             (char *)smartlist_get(unrecognized, i));
-    goto done;
-  }
-
-  for (i = 0; i < smartlist_len(answers); i += 2) {
-    char *k = smartlist_get(answers, i);
-    char *v = smartlist_get(answers, i+1);
-    if (!strchr(v, '\n') && !strchr(v, '\r')) {
-      connection_printf_to_buf(conn, "250-%s=", k);
-      connection_write_str_to_buf(v, conn);
-      connection_write_str_to_buf("\r\n", conn);
-    } else {
-      char *esc = NULL;
-      size_t esc_len;
-      esc_len = write_escaped_data(v, strlen(v), &esc);
-      connection_printf_to_buf(conn, "250+%s=\r\n", k);
-      connection_buf_add(esc, esc_len, TO_CONN(conn));
-      tor_free(esc);
-    }
-  }
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
-  smartlist_free(unrecognized);
-
-  return 0;
-}
-
-/** Given a string, convert it to a circuit purpose. */
-static uint8_t
-circuit_purpose_from_string(const char *string)
-{
-  if (!strcasecmpstart(string, "purpose="))
-    string += strlen("purpose=");
-
-  if (!strcasecmp(string, "general"))
-    return CIRCUIT_PURPOSE_C_GENERAL;
-  else if (!strcasecmp(string, "controller"))
-    return CIRCUIT_PURPOSE_CONTROLLER;
-  else
-    return CIRCUIT_PURPOSE_UNKNOWN;
-}
-
-/** Return a newly allocated smartlist containing the arguments to the command
- * waiting in <b>body</b>. If there are fewer than <b>min_args</b> arguments,
- * or if <b>max_args</b> is nonnegative and there are more than
- * <b>max_args</b> arguments, send a 512 error to the controller, using
- * <b>command</b> as the command name in the error message. */
-static smartlist_t *
-getargs_helper(const char *command, control_connection_t *conn,
-               const char *body, int min_args, int max_args)
-{
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(args) < min_args) {
-    connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command);
-    goto err;
-  } else if (max_args >= 0 && smartlist_len(args) > max_args) {
-    connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command);
-    goto err;
-  }
-  return args;
- err:
-  SMARTLIST_FOREACH(args, char *, s, tor_free(s));
-  smartlist_free(args);
-  return NULL;
-}
-
-/** Helper.  Return the first element of <b>sl</b> at index <b>start_at</b> or
- * higher that starts with <b>prefix</b>, case-insensitive.  Return NULL if no
- * such element exists. */
-static const char *
-find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix)
-{
-  int i;
-  for (i = start_at; i < smartlist_len(sl); ++i) {
-    const char *elt = smartlist_get(sl, i);
-    if (!strcasecmpstart(elt, prefix))
-      return elt;
-  }
-  return NULL;
-}
-
-/** Helper.  Return true iff s is an argument that we should treat as a
- * key-value pair. */
-static int
-is_keyval_pair(const char *s)
-{
-  /* An argument is a key-value pair if it has an =, and it isn't of the form
-   * $fingeprint=name */
-  return strchr(s, '=') && s[0] != '$';
-}
-
-/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
- * circuit, and report success or failure. */
-static int
-handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  smartlist_t *router_nicknames=NULL, *nodes=NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
-  smartlist_t *args;
-  (void) len;
-
-  router_nicknames = smartlist_new();
-
-  args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1);
-  if (!args)
-    goto done;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,0));
-
-  if (zero_circ) {
-    const char *purp = find_element_starting_with(args, 1, "PURPOSE=");
-
-    if (purp) {
-      intended_purpose = circuit_purpose_from_string(purp);
-      if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-        SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-        smartlist_free(args);
-        goto done;
-      }
-    }
-
-    if ((smartlist_len(args) == 1) ||
-        (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) {
-      // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar"
-      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
-      if (!circ) {
-        connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      } else {
-        connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                  (unsigned long)circ->global_identifier);
-      }
-      SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-      smartlist_free(args);
-      goto done;
-    }
-    // "EXTENDCIRCUIT 0 router1,router2" ||
-    // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo"
-  }
-
-  if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  if (smartlist_len(args) < 2) {
-    connection_printf_to_buf(conn,
-                             "512 syntax error: not enough arguments.\r\n");
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0);
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-
-  nodes = smartlist_new();
-  int first_node = zero_circ;
-  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
-    const node_t *node = node_get_by_nickname(n, 0);
-    if (!node) {
-      connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n);
-      goto done;
-    }
-    if (!node_has_preferred_descriptor(node, first_node)) {
-      connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n);
-      goto done;
-    }
-    smartlist_add(nodes, (void*)node);
-    first_node = 0;
-  } SMARTLIST_FOREACH_END(n);
-  if (!smartlist_len(nodes)) {
-    connection_write_str_to_buf("512 No router names provided\r\n", conn);
-    goto done;
-  }
-
-  if (zero_circ) {
-    /* start a new circuit */
-    circ = origin_circuit_init(intended_purpose, 0);
-  }
-
-  /* now circ refers to something that is ready to be extended */
-  first_node = zero_circ;
-  SMARTLIST_FOREACH(nodes, const node_t *, node,
-  {
-    extend_info_t *info = extend_info_from_node(node, first_node);
-    if (!info) {
-      tor_assert_nonfatal(first_node);
-      log_warn(LD_CONTROL,
-               "controller tried to connect to a node that lacks a suitable "
-               "descriptor, or which doesn't have any "
-               "addresses that are allowed by the firewall configuration; "
-               "circuit marked for closing.");
-      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-    circuit_append_new_exit(circ, info);
-    if (circ->build_state->desired_path_len > 1) {
-      circ->build_state->onehop_tunnel = 0;
-    }
-    extend_info_free(info);
-    first_node = 0;
-  });
-
-  /* now that we've populated the cpath, start extending */
-  if (zero_circ) {
-    int err_reason = 0;
-    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
-      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-  } else {
-    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
-        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
-      int err_reason = 0;
-      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
-      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
-        log_info(LD_CONTROL,
-                 "send_next_onion_skin failed; circuit marked for closing.");
-        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-        connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn);
-        goto done;
-      }
-    }
-  }
-
-  connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                             (unsigned long)circ->global_identifier);
-  if (zero_circ) /* send a 'launched' event, for completeness */
-    control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
- done:
-  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
-  smartlist_free(router_nicknames);
-  smartlist_free(nodes);
-  return 0;
-}
-
-/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
- * circuit and it's a valid purpose, change it. */
-static int
-handle_control_setcircuitpurpose(control_connection_t *conn,
-                                 uint32_t len, const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  uint8_t new_purpose;
-  smartlist_t *args;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1);
-  if (!args)
-    goto done;
-
-  if (!(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    goto done;
-  }
-
-  {
-    const char *purp = find_element_starting_with(args,1,"PURPOSE=");
-    if (!purp) {
-      connection_write_str_to_buf("552 No purpose given\r\n", conn);
-      goto done;
-    }
-    new_purpose = circuit_purpose_from_string(purp);
-    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-      connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-      goto done;
-    }
-  }
-
-  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  if (args) {
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-  }
-  return 0;
-}
-
-/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
- * stream, and report success or failure. */
-static int
-handle_control_attachstream(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  smartlist_t *args;
-  crypt_path_t *cpath=NULL;
-  int hop=0, hop_line_ok=1;
-  (void) len;
-
-  args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,1));
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 1));
-  } else if (circ) {
-    const char *hopstring = find_element_starting_with(args,2,"HOP=");
-    if (hopstring) {
-      hopstring += strlen("HOP=");
-      hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX,
-                                  &hop_line_ok, NULL);
-      if (!hop_line_ok) { /* broken hop line */
-        connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring);
-      }
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok)
-    return 0;
-
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
-    connection_write_str_to_buf(
-                    "555 Connection is not managed by controller.\r\n",
-                    conn);
-    return 0;
-  }
-
-  /* Do we need to detach it first? */
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
-    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
-    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
-    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
-    /* Un-mark it as ending, since we're going to reuse it. */
-    edge_conn->edge_has_sent_end = 0;
-    edge_conn->end_reason = 0;
-    if (tmpcirc)
-      circuit_detach_stream(tmpcirc, edge_conn);
-    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
-    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
-  }
-
-  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
-    connection_write_str_to_buf(
-                    "551 Can't attach stream to non-open origin circuit\r\n",
-                    conn);
-    return 0;
-  }
-  /* Is this a single hop circuit? */
-  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
-    connection_write_str_to_buf(
-               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
-    return 0;
-  }
-
-  if (circ && hop>0) {
-    /* find this hop in the circuit, and set cpath */
-    cpath = circuit_get_cpath_hop(circ, hop);
-    if (!cpath) {
-      connection_printf_to_buf(conn,
-                               "551 Circuit doesn't have %d hops.\r\n", hop);
-      return 0;
-    }
-  }
-  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
-    connection_write_str_to_buf("551 Unable to attach stream\r\n", conn);
-    return 0;
-  }
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
- * descriptor, and report success or failure. */
-static int
-handle_control_postdescriptor(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  char *desc;
-  const char *msg=NULL;
-  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
-  int cache = 0; /* eventually, we may switch this to 1 */
-
-  const char *cp = memchr(body, '\n', len);
-
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  ++cp;
-
-  char *cmdline = tor_memdup_nulterm(body, cp-body);
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, cmdline, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, char *, option) {
-    if (!strcasecmpstart(option, "purpose=")) {
-      option += strlen("purpose=");
-      purpose = router_purpose_from_string(option);
-      if (purpose == ROUTER_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else if (!strcasecmpstart(option, "cache=")) {
-      option += strlen("cache=");
-      if (!strcasecmp(option, "no"))
-        cache = 0;
-      else if (!strcasecmp(option, "yes"))
-        cache = 1;
-      else {
-        connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else { /* unrecognized argument? */
-      connection_printf_to_buf(conn,
-        "512 Unexpected argument \"%s\" to postdescriptor\r\n", option);
-      goto done;
-    }
-  } SMARTLIST_FOREACH_END(option);
-
-  read_escaped_data(cp, len-(cp-body), &desc);
-
-  switch (router_load_single_router(desc, purpose, cache, &msg)) {
-  case -1:
-    if (!msg) msg = "Could not parse descriptor";
-    connection_printf_to_buf(conn, "554 %s\r\n", msg);
-    break;
-  case 0:
-    if (!msg) msg = "Descriptor not added";
-    connection_printf_to_buf(conn, "251 %s\r\n",msg);
-    break;
-  case 1:
-    send_control_done(conn);
-    break;
-  }
-
-  tor_free(desc);
- done:
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  tor_free(cmdline);
-  return 0;
-}
-
-/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
- * address of the named AP stream, and report success or failure. */
-static int
-handle_control_redirectstream(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  char *new_addr = NULL;
-  uint16_t new_port = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
-           || !ap_conn->socks_request) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else {
-    int ok = 1;
-    if (smartlist_len(args) > 2) { /* they included a port too */
-      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
-                                            10, 1, 65535, &ok, NULL);
-    }
-    if (!ok) {
-      connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n",
-                               (char*)smartlist_get(args, 2));
-    } else {
-      new_addr = tor_strdup(smartlist_get(args, 1));
-    }
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!new_addr)
-    return 0;
-
-  strlcpy(ap_conn->socks_request->address, new_addr,
-          sizeof(ap_conn->socks_request->address));
-  if (new_port)
-    ap_conn->socks_request->port = new_port;
-  tor_free(new_addr);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSESTREAM command; try to close the named stream
- * and report success or failure. */
-static int
-handle_control_closestream(control_connection_t *conn, uint32_t len,
-                           const char *body)
-{
-  entry_connection_t *ap_conn=NULL;
-  uint8_t reason=0;
-  smartlist_t *args;
-  int ok;
-  (void) len;
-
-  args = getargs_helper("CLOSESTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  else if (!(ap_conn = get_stream(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
-                                       &ok, NULL);
-    if (!ok) {
-      connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n",
-                               (char*)smartlist_get(args, 1));
-      ap_conn = NULL;
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn)
-    return 0;
-
-  connection_mark_unattached_ap(ap_conn, reason);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
- * and report success or failure. */
-static int
-handle_control_closecircuit(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  int safe = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1);
-  if (!args)
-    return 0;
-
-  if (!(circ=get_circ(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    int i;
-    for (i=1; i < smartlist_len(args); ++i) {
-      if (!strcasecmp(smartlist_get(args, i), "IfUnused"))
-        safe = 1;
-      else
-        log_info(LD_CONTROL, "Skipping unknown option %s",
-                 (char*)smartlist_get(args,i));
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!circ)
-    return 0;
-
-  if (!safe || !circ->p_streams) {
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
-  }
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a RESOLVE command: start trying to resolve
- * the listed addresses. */
-static int
-handle_control_resolve(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *args, *failed;
-  int is_reverse = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-
-  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
-    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
-             "isn't listening for ADDRMAP events.  It probably won't see "
-             "the answer.");
-  }
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  {
-    const char *modearg = find_element_starting_with(args, 0, "mode=");
-    if (modearg && !strcasecmp(modearg, "mode=reverse"))
-      is_reverse = 1;
-  }
-  failed = smartlist_new();
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      if (!is_keyval_pair(arg)) {
-          if (dnsserv_launch_request(arg, is_reverse, conn)<0)
-            smartlist_add(failed, (char*)arg);
-      }
-  });
-
-  send_control_done(conn);
-  SMARTLIST_FOREACH(failed, const char *, arg, {
-      control_event_address_mapped(arg, arg, time(NULL),
-                                   "internal", 0);
-  });
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  smartlist_free(failed);
-  return 0;
-}
-
-/** Called when we get a PROTOCOLINFO command: send back a reply. */
-static int
-handle_control_protocolinfo(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  const char *bad_arg = NULL;
-  smartlist_t *args;
-  (void)len;
-
-  conn->have_sent_protocolinfo = 1;
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      int ok;
-      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
-      if (!ok) {
-        bad_arg = arg;
-        break;
-      }
-    });
-  if (bad_arg) {
-    connection_printf_to_buf(conn, "513 No such version %s\r\n",
-                             escaped(bad_arg));
-    /* Don't tolerate bad arguments when not authenticated. */
-    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
-      connection_mark_for_close(TO_CONN(conn));
-    goto done;
-  } else {
-    const or_options_t *options = get_options();
-    int cookies = options->CookieAuthentication;
-    char *cfile = get_controller_cookie_file_name();
-    char *abs_cfile;
-    char *esc_cfile;
-    char *methods;
-    abs_cfile = make_path_absolute(cfile);
-    esc_cfile = esc_for_log(abs_cfile);
-    {
-      int passwd = (options->HashedControlPassword != NULL ||
-                    options->HashedControlSessionPassword != NULL);
-      smartlist_t *mlist = smartlist_new();
-      if (cookies) {
-        smartlist_add(mlist, (char*)"COOKIE");
-        smartlist_add(mlist, (char*)"SAFECOOKIE");
-      }
-      if (passwd)
-        smartlist_add(mlist, (char*)"HASHEDPASSWORD");
-      if (!cookies && !passwd)
-        smartlist_add(mlist, (char*)"NULL");
-      methods = smartlist_join_strings(mlist, ",", 0, NULL);
-      smartlist_free(mlist);
-    }
-
-    connection_printf_to_buf(conn,
-                             "250-PROTOCOLINFO 1\r\n"
-                             "250-AUTH METHODS=%s%s%s\r\n"
-                             "250-VERSION Tor=%s\r\n"
-                             "250 OK\r\n",
-                             methods,
-                             cookies?" COOKIEFILE=":"",
-                             cookies?esc_cfile:"",
-                             escaped(VERSION));
-    tor_free(methods);
-    tor_free(cfile);
-    tor_free(abs_cfile);
-    tor_free(esc_cfile);
-  }
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Called when we get an AUTHCHALLENGE command. */
-static int
-handle_control_authchallenge(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  const char *cp = body;
-  char *client_nonce;
-  size_t client_nonce_len;
-  char server_hash[DIGEST256_LEN];
-  char server_hash_encoded[HEX_DIGEST256_LEN+1];
-  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
-  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
-
-  cp += strspn(cp, " \t\n\r");
-  if (!strcasecmpstart(cp, "SAFECOOKIE")) {
-    cp += strlen("SAFECOOKIE");
-  } else {
-    connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE "
-                                "authentication\r\n", conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  if (!authentication_cookie_is_set) {
-    connection_write_str_to_buf("515 Cookie authentication is disabled\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp == '"') {
-    const char *newcp =
-      decode_escaped_string(cp, len - (cp - body),
-                            &client_nonce, &client_nonce_len);
-    if (newcp == NULL) {
-      connection_write_str_to_buf("513 Invalid quoted client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return -1;
-    }
-    cp = newcp;
-  } else {
-    size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef");
-
-    client_nonce_len = client_nonce_encoded_len / 2;
-    client_nonce = tor_malloc_zero(client_nonce_len);
-
-    if (base16_decode(client_nonce, client_nonce_len,
-                      cp, client_nonce_encoded_len)
-                      != (int) client_nonce_len) {
-      connection_write_str_to_buf("513 Invalid base16 client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(client_nonce);
-      return -1;
-    }
-
-    cp += client_nonce_encoded_len;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp != '\0' ||
-      cp != body + len) {
-    connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    tor_free(client_nonce);
-    return -1;
-  }
-  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-  /* Now compute and send the server-to-controller response, and the
-   * server's nonce. */
-  tor_assert(authentication_cookie != NULL);
-
-  {
-    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
-                      client_nonce_len +
-                      SAFECOOKIE_SERVER_NONCE_LEN);
-    char *tmp = tor_malloc_zero(tmp_len);
-    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
-    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
-           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-    crypto_hmac_sha256(server_hash,
-                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
-                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    crypto_hmac_sha256(client_hash,
-                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
-                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    conn->safecookie_client_hash = client_hash;
-
-    tor_free(tmp);
-  }
-
-  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
-                server_hash, sizeof(server_hash));
-  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
-                server_nonce, sizeof(server_nonce));
-
-  connection_printf_to_buf(conn,
-                           "250 AUTHCHALLENGE SERVERHASH=%s "
-                           "SERVERNONCE=%s\r\n",
-                           server_hash_encoded,
-                           server_nonce_encoded);
-
-  tor_free(client_nonce);
-  return 0;
-}
-
-/** Called when we get a USEFEATURE command: parse the feature list, and
- * set up the control_connection's options properly. */
-static int
-handle_control_usefeature(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  int bad = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmp(arg, "VERBOSE_NAMES"))
-        ;
-      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
-        ;
-      else {
-        connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n",
-                                 arg);
-        bad = 1;
-        break;
-      }
-  } SMARTLIST_FOREACH_END(arg);
-
-  if (!bad) {
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the DROPGUARDS command. */
-static int
-handle_control_dropguards(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-
-  static int have_warned = 0;
-  if (! have_warned) {
-    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
-             "the risks before using it. It may be removed in a future "
-             "version of Tor.");
-    have_warned = 1;
-  }
-
-  if (smartlist_len(args)) {
-    connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
-  } else {
-    remove_all_entry_guards();
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the HSFETCH command. */
-static int
-handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  int i;
-  char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL;
-  smartlist_t *args = NULL, *hsdirs = NULL;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  static const char *hsfetch_command = "HSFETCH";
-  static const char *v2_str = "v2-";
-  const size_t v2_str_len = strlen(v2_str);
-  rend_data_t *rend_query = NULL;
-
-  /* Make sure we have at least one argument, the HSAddress. */
-  args = getargs_helper(hsfetch_command, conn, body, 1, -1);
-  if (!args) {
-    goto exit;
-  }
-
-  /* Extract the first argument (either HSAddress or DescID). */
-  arg1 = smartlist_get(args, 0);
-  /* Test if it's an HS address without the .onion part. */
-  if (rend_valid_v2_service_id(arg1)) {
-    hsaddress = arg1;
-  } else if (strcmpstart(arg1, v2_str) == 0 &&
-             rend_valid_descriptor_id(arg1 + v2_str_len) &&
-             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
-                           REND_DESC_ID_V2_LEN_BASE32) == 0) {
-    /* We have a well formed version 2 descriptor ID. Keep the decoded value
-     * of the id. */
-    desc_id = digest;
-  } else {
-    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
-                             arg1);
-    goto done;
-  }
-
-  static const char *opt_server = "SERVER=";
-
-  /* Skip first argument because it's the HSAddress or DescID. */
-  for (i = 1; i < smartlist_len(args); ++i) {
-    const char *arg = smartlist_get(args, i);
-    const node_t *node;
-
-    if (!strcasecmpstart(arg, opt_server)) {
-      const char *server;
-
-      server = arg + strlen(opt_server);
-      node = node_get_by_hex_id(server, 0);
-      if (!node) {
-        connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                 server);
-        goto done;
-      }
-      if (!hsdirs) {
-        /* Stores routerstatus_t object for each specified server. */
-        hsdirs = smartlist_new();
-      }
-      /* Valid server, add it to our local list. */
-      smartlist_add(hsdirs, node->rs);
-    } else {
-      connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n",
-                               arg);
-      goto done;
-    }
-  }
-
-  rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
-                                       REND_NO_AUTH);
-  if (rend_query == NULL) {
-    connection_printf_to_buf(conn, "551 Error creating the HS query\r\n");
-    goto done;
-  }
-
-  /* Using a descriptor ID, we force the user to provide at least one
-   * hsdir server using the SERVER= option. */
-  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
-      connection_printf_to_buf(conn, "512 %s option is required\r\n",
-                               opt_server);
-      goto done;
-  }
-
-  /* We are about to trigger HSDir fetch so send the OK now because after
-   * that 650 event(s) are possible so better to have the 250 OK before them
-   * to avoid out of order replies. */
-  send_control_done(conn);
-
-  /* Trigger the fetch using the built rend query and possibly a list of HS
-   * directory to use. This function ignores the client cache thus this will
-   * always send a fetch command. */
-  rend_client_fetch_v2_desc(rend_query, hsdirs);
-
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  /* Contains data pointer that we don't own thus no cleanup. */
-  smartlist_free(hsdirs);
-  rend_data_free(rend_query);
- exit:
-  return 0;
-}
-
-/** Implementation for the HSPOST command. */
-static int
-handle_control_hspost(control_connection_t *conn,
-                      uint32_t len,
-                      const char *body)
-{
-  static const char *opt_server = "SERVER=";
-  static const char *opt_hsaddress = "HSADDRESS=";
-  smartlist_t *hs_dirs = NULL;
-  const char *encoded_desc = body;
-  size_t encoded_desc_len = len;
-  const char *onion_address = NULL;
-
-  char *cp = memchr(body, '\n', len);
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  char *argline = tor_strndup(body, cp-body);
-
-  smartlist_t *args = smartlist_new();
-
-  /* If any SERVER= or HSADDRESS= options were specified, try to parse
-   * the options line. */
-  if (!strcasecmpstart(argline, opt_server) ||
-      !strcasecmpstart(argline, opt_hsaddress)) {
-    /* encoded_desc begins after a newline character */
-    cp = cp + 1;
-    encoded_desc = cp;
-    encoded_desc_len = len-(cp-body);
-
-    smartlist_split_string(args, argline, " ",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmpstart(arg, opt_server)) {
-        const char *server = arg + strlen(opt_server);
-        const node_t *node = node_get_by_hex_id(server, 0);
-
-        if (!node || !node->rs) {
-          connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                   server);
-          goto done;
-        }
-        /* Valid server, add it to our local list. */
-        if (!hs_dirs)
-          hs_dirs = smartlist_new();
-        smartlist_add(hs_dirs, node->rs);
-      } else if (!strcasecmpstart(arg, opt_hsaddress)) {
-        const char *address = arg + strlen(opt_hsaddress);
-        if (!hs_address_is_valid(address)) {
-          connection_printf_to_buf(conn, "512 Malformed onion address\r\n");
-          goto done;
-        }
-        onion_address = address;
-      } else {
-        connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n",
-                                 arg);
-        goto done;
-      }
-    } SMARTLIST_FOREACH_END(arg);
-  }
-
-  /* Handle the v3 case. */
-  if (onion_address) {
-    char *desc_str = NULL;
-    read_escaped_data(encoded_desc, encoded_desc_len, &desc_str);
-    if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) {
-      connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-    } else {
-      send_control_done(conn);
-    }
-    tor_free(desc_str);
-    goto done;
-  }
-
-  /* From this point on, it is only v2. */
-
-  /* Read the dot encoded descriptor, and parse it. */
-  rend_encoded_v2_service_descriptor_t *desc =
-      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
-  read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str);
-
-  rend_service_descriptor_t *parsed = NULL;
-  char *intro_content = NULL;
-  size_t intro_size;
-  size_t encoded_size;
-  const char *next_desc;
-  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
-                                        &intro_size, &encoded_size,
-                                        &next_desc, desc->desc_str, 1)) {
-    /* Post the descriptor. */
-    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
-    if (!rend_get_service_id(parsed->pk, serviceid)) {
-      smartlist_t *descs = smartlist_new();
-      smartlist_add(descs, desc);
-
-      /* We are about to trigger HS descriptor upload so send the OK now
-       * because after that 650 event(s) are possible so better to have the
-       * 250 OK before them to avoid out of order replies. */
-      send_control_done(conn);
-
-      /* Trigger the descriptor upload */
-      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
-      smartlist_free(descs);
-    }
-
-    rend_service_descriptor_free(parsed);
-  } else {
-    connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-  }
-
-  tor_free(intro_content);
-  rend_encoded_v2_service_descriptor_free(desc);
- done:
-  tor_free(argline);
-  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  return 0;
-}
-
-/* Helper function for ADD_ONION that adds an ephemeral service depending on
- * the given hs_version.
- *
- * The secret key in pk depends on the hs_version. The ownership of the key
- * used in pk is given to the HS subsystem so the caller must stop accessing
- * it after.
- *
- * The port_cfgs is a list of service port. Ownership transferred to service.
- * The max_streams refers to the MaxStreams= key.
- * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
- * The auth_type is the authentication type of the clients in auth_clients.
- * The ownership of that list is transferred to the service.
- *
- * On success (RSAE_OKAY), the address_out points to a newly allocated string
- * containing the onion address without the .onion part. On error, address_out
- * is untouched. */
-static hs_service_add_ephemeral_status_t
-add_onion_helper_add_service(int hs_version,
-                             add_onion_secret_key_t *pk,
-                             smartlist_t *port_cfgs, int max_streams,
-                             int max_streams_close_circuit, int auth_type,
-                             smartlist_t *auth_clients, char **address_out)
-{
-  hs_service_add_ephemeral_status_t ret;
-
-  tor_assert(pk);
-  tor_assert(port_cfgs);
-  tor_assert(address_out);
-
-  switch (hs_version) {
-  case HS_VERSION_TWO:
-    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
-                                     max_streams_close_circuit, auth_type,
-                                     auth_clients, address_out);
-    break;
-  case HS_VERSION_THREE:
-    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
-                                   max_streams_close_circuit, address_out);
-    break;
-  default:
-    tor_assert_unreached();
-  }
-
-  return ret;
-}
-
-/** Called when we get a ADD_ONION command; parse the body, and set up
- * the new ephemeral Onion Service. */
-static int
-handle_control_add_onion(control_connection_t *conn,
-                         uint32_t len,
-                         const char *body)
-{
-  smartlist_t *args;
-  int arg_len;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("ADD_ONION", conn, body, 2, -1);
-  if (!args)
-    return 0;
-  arg_len = smartlist_len(args);
-
-  /* Parse all of the arguments that do not involve handling cryptographic
-   * material first, since there's no reason to touch that at all if any of
-   * the other arguments are malformed.
-   */
-  smartlist_t *port_cfgs = smartlist_new();
-  smartlist_t *auth_clients = NULL;
-  smartlist_t *auth_created_clients = NULL;
-  int discard_pk = 0;
-  int detach = 0;
-  int max_streams = 0;
-  int max_streams_close_circuit = 0;
-  rend_auth_type_t auth_type = REND_NO_AUTH;
-  /* Default to adding an anonymous hidden service if no flag is given */
-  int non_anonymous = 0;
-  for (int i = 1; i < arg_len; i++) {
-    static const char *port_prefix = "Port=";
-    static const char *flags_prefix = "Flags=";
-    static const char *max_s_prefix = "MaxStreams=";
-    static const char *auth_prefix = "ClientAuth=";
-
-    const char *arg = smartlist_get(args, (int)i);
-    if (!strcasecmpstart(arg, port_prefix)) {
-      /* "Port=VIRTPORT[,TARGET]". */
-      const char *port_str = arg + strlen(port_prefix);
-
-      rend_service_port_config_t *cfg =
-          rend_service_parse_port_config(port_str, ",", NULL);
-      if (!cfg) {
-        connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-        goto out;
-      }
-      smartlist_add(port_cfgs, cfg);
-    } else if (!strcasecmpstart(arg, max_s_prefix)) {
-      /* "MaxStreams=[0..65535]". */
-      const char *max_s_str = arg + strlen(max_s_prefix);
-      int ok = 0;
-      max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL);
-      if (!ok) {
-        connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n");
-        goto out;
-      }
-    } else if (!strcasecmpstart(arg, flags_prefix)) {
-      /* "Flags=Flag[,Flag]", where Flag can be:
-       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
-       *                   the response.
-       *   * 'Detach' - Do not tie this onion service to any particular control
-       *                connection.
-       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
-       *                                exceeded.
-       *   * 'BasicAuth' - Client authorization using the 'basic' method.
-       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
-       *                      flag is present, tor must be in non-anonymous
-       *                      hidden service mode. If this flag is absent,
-       *                      tor must be in anonymous hidden service mode.
-       */
-      static const char *discard_flag = "DiscardPK";
-      static const char *detach_flag = "Detach";
-      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
-      static const char *basicauth_flag = "BasicAuth";
-      static const char *non_anonymous_flag = "NonAnonymous";
-
-      smartlist_t *flags = smartlist_new();
-      int bad = 0;
-
-      smartlist_split_string(flags, arg + strlen(flags_prefix), ",",
-                             SPLIT_IGNORE_BLANK, 0);
-      if (smartlist_len(flags) < 1) {
-        connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n");
-        bad = 1;
-      }
-      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
-      {
-        if (!strcasecmp(flag, discard_flag)) {
-          discard_pk = 1;
-        } else if (!strcasecmp(flag, detach_flag)) {
-          detach = 1;
-        } else if (!strcasecmp(flag, max_s_close_flag)) {
-          max_streams_close_circuit = 1;
-        } else if (!strcasecmp(flag, basicauth_flag)) {
-          auth_type = REND_BASIC_AUTH;
-        } else if (!strcasecmp(flag, non_anonymous_flag)) {
-          non_anonymous = 1;
-        } else {
-          connection_printf_to_buf(conn,
-                                   "512 Invalid 'Flags' argument: %s\r\n",
-                                   escaped(flag));
-          bad = 1;
-          break;
-        }
-      } SMARTLIST_FOREACH_END(flag);
-      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
-      smartlist_free(flags);
-      if (bad)
-        goto out;
-    } else if (!strcasecmpstart(arg, auth_prefix)) {
-      char *err_msg = NULL;
-      int created = 0;
-      rend_authorized_client_t *client =
-        add_onion_helper_clientauth(arg + strlen(auth_prefix),
-                                    &created, &err_msg);
-      if (!client) {
-        if (err_msg) {
-          connection_write_str_to_buf(err_msg, conn);
-          tor_free(err_msg);
-        }
-        goto out;
-      }
-
-      if (auth_clients != NULL) {
-        int bad = 0;
-        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
-          if (strcmp(ac->client_name, client->client_name) == 0) {
-            bad = 1;
-            break;
-          }
-        } SMARTLIST_FOREACH_END(ac);
-        if (bad) {
-          connection_printf_to_buf(conn,
-                                   "512 Duplicate name in ClientAuth\r\n");
-          rend_authorized_client_free(client);
-          goto out;
-        }
-      } else {
-        auth_clients = smartlist_new();
-        auth_created_clients = smartlist_new();
-      }
-      smartlist_add(auth_clients, client);
-      if (created) {
-        smartlist_add(auth_created_clients, client);
-      }
-    } else {
-      connection_printf_to_buf(conn, "513 Invalid argument\r\n");
-      goto out;
-    }
-  }
-  if (smartlist_len(port_cfgs) == 0) {
-    connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n");
-    goto out;
-  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
-    connection_printf_to_buf(conn, "512 No auth type specified\r\n");
-    goto out;
-  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
-    connection_printf_to_buf(conn, "512 No auth clients specified\r\n");
-    goto out;
-  } else if ((auth_type == REND_BASIC_AUTH &&
-              smartlist_len(auth_clients) > 512) ||
-             (auth_type == REND_STEALTH_AUTH &&
-              smartlist_len(auth_clients) > 16)) {
-    connection_printf_to_buf(conn, "512 Too many auth clients\r\n");
-    goto out;
-  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
-                                                              get_options())) {
-    /* If we failed, and the non-anonymous flag is set, Tor must be in
-     * anonymous hidden service mode.
-     * The error message changes based on the current Tor config:
-     * 512 Tor is in anonymous hidden service mode
-     * 512 Tor is in non-anonymous hidden service mode
-     * (I've deliberately written them out in full here to aid searchability.)
-     */
-    connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service "
-                             "mode\r\n",
-                             non_anonymous ? "" : "non-");
-    goto out;
-  }
-
-  /* Parse the "keytype:keyblob" argument. */
-  int hs_version = 0;
-  add_onion_secret_key_t pk = { NULL };
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-
-  if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
-                              &key_new_alg, &key_new_blob, &pk, &hs_version,
-                              &err_msg) < 0) {
-    if (err_msg) {
-      connection_write_str_to_buf(err_msg, conn);
-      tor_free(err_msg);
-    }
-    goto out;
-  }
-  tor_assert(!err_msg);
-
-  /* Hidden service version 3 don't have client authentication support so if
-   * ClientAuth was given, send back an error. */
-  if (hs_version == HS_VERSION_THREE && auth_clients) {
-    connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n");
-    goto out;
-  }
-
-  /* Create the HS, using private key pk, client authentication auth_type,
-   * the list of auth_clients, and port config port_cfg.
-   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
-   * regardless of success/failure.
-   */
-  char *service_id = NULL;
-  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
-                                         max_streams,
-                                         max_streams_close_circuit, auth_type,
-                                         auth_clients, &service_id);
-  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
-  auth_clients = NULL; /* so is auth_clients */
-  switch (ret) {
-  case RSAE_OKAY:
-  {
-    if (detach) {
-      if (!detached_onion_services)
-        detached_onion_services = smartlist_new();
-      smartlist_add(detached_onion_services, service_id);
-    } else {
-      if (!conn->ephemeral_onion_services)
-        conn->ephemeral_onion_services = smartlist_new();
-      smartlist_add(conn->ephemeral_onion_services, service_id);
-    }
-
-    tor_assert(service_id);
-    connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id);
-    if (key_new_alg) {
-      tor_assert(key_new_blob);
-      connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n",
-                               key_new_alg, key_new_blob);
-    }
-    if (auth_created_clients) {
-      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
-        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
-                                                auth_type);
-        tor_assert(encoded);
-        connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n",
-                                 ac->client_name, encoded);
-        memwipe(encoded, 0, strlen(encoded));
-        tor_free(encoded);
-      });
-    }
-
-    connection_printf_to_buf(conn, "250 OK\r\n");
-    break;
-  }
-  case RSAE_BADPRIVKEY:
-    connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n");
-    break;
-  case RSAE_ADDREXISTS:
-    connection_printf_to_buf(conn, "550 Onion address collision\r\n");
-    break;
-  case RSAE_BADVIRTPORT:
-    connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-    break;
-  case RSAE_BADAUTH:
-    connection_printf_to_buf(conn, "512 Invalid client authorization\r\n");
-    break;
-  case RSAE_INTERNAL: FALLTHROUGH;
-  default:
-    connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n");
-  }
-  if (key_new_blob) {
-    memwipe(key_new_blob, 0, strlen(key_new_blob));
-    tor_free(key_new_blob);
-  }
-
- out:
-  if (port_cfgs) {
-    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
-                      rend_service_port_config_free(p));
-    smartlist_free(port_cfgs);
-  }
-
-  if (auth_clients) {
-    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
-                      rend_authorized_client_free(ac));
-    smartlist_free(auth_clients);
-  }
-  if (auth_created_clients) {
-    // Do not free entries; they are the same as auth_clients
-    smartlist_free(auth_created_clients);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
-/** Helper function to handle parsing the KeyType:KeyBlob argument to the
- * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
- * and the private key not discarded, the algorithm and serialized private key,
- * or NULL and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned key_new_blob and err_msg.
- *
- * Note: The error messages returned are deliberately vague to avoid echoing
- * key material.
- */
-STATIC int
-add_onion_helper_keyarg(const char *arg, int discard_pk,
-                        const char **key_new_alg_out, char **key_new_blob_out,
-                        add_onion_secret_key_t *decoded_key, int *hs_version,
-                        char **err_msg_out)
-{
-  smartlist_t *key_args = smartlist_new();
-  crypto_pk_t *pk = NULL;
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-  int ret = -1;
-
-  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(key_args) != 2) {
-    err_msg = tor_strdup("512 Invalid key type/blob\r\n");
-    goto err;
-  }
-
-  /* The format is "KeyType:KeyBlob". */
-  static const char *key_type_new = "NEW";
-  static const char *key_type_best = "BEST";
-  static const char *key_type_rsa1024 = "RSA1024";
-  static const char *key_type_ed25519_v3 = "ED25519-V3";
-
-  const char *key_type = smartlist_get(key_args, 0);
-  const char *key_blob = smartlist_get(key_args, 1);
-
-  if (!strcasecmp(key_type_rsa1024, key_type)) {
-    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
-    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
-    if (!pk) {
-      err_msg = tor_strdup("512 Failed to decode RSA key\r\n");
-      goto err;
-    }
-    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
-      crypto_pk_free(pk);
-      err_msg = tor_strdup("512 Invalid RSA key size\r\n");
-      goto err;
-    }
-    decoded_key->v2 = pk;
-    *hs_version = HS_VERSION_TWO;
-  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
-    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
-    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
-                      strlen(key_blob)) != sizeof(sk->seckey)) {
-      tor_free(sk);
-      err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n");
-      goto err;
-    }
-    decoded_key->v3 = sk;
-    *hs_version = HS_VERSION_THREE;
-  } else if (!strcasecmp(key_type_new, key_type)) {
-    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
-    if (!strcasecmp(key_type_rsa1024, key_blob) ||
-        !strcasecmp(key_type_best, key_blob)) {
-      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
-      pk = crypto_pk_new();
-      if (crypto_pk_generate_key(pk)) {
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_rsa1024);
-        goto err;
-      }
-      if (!discard_pk) {
-        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
-          crypto_pk_free(pk);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_rsa1024);
-          goto err;
-        }
-        key_new_alg = key_type_rsa1024;
-      }
-      decoded_key->v2 = pk;
-      *hs_version = HS_VERSION_TWO;
-    } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) {
-      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-      if (ed25519_secret_key_generate(sk, 1) < 0) {
-        tor_free(sk);
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_ed25519_v3);
-        goto err;
-      }
-      if (!discard_pk) {
-        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
-        key_new_blob = tor_malloc_zero(len);
-        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
-                          sizeof(sk->seckey), 0) != (len - 1)) {
-          tor_free(sk);
-          tor_free(key_new_blob);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_ed25519_v3);
-          goto err;
-        }
-        key_new_alg = key_type_ed25519_v3;
-      }
-      decoded_key->v3 = sk;
-      *hs_version = HS_VERSION_THREE;
-    } else {
-      err_msg = tor_strdup("513 Invalid key type\r\n");
-      goto err;
-    }
-  } else {
-    err_msg = tor_strdup("513 Invalid key type\r\n");
-    goto err;
-  }
-
-  /* Succeeded in loading or generating a private key. */
-  ret = 0;
-
- err:
-  SMARTLIST_FOREACH(key_args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(key_args);
-
-  if (err_msg_out) {
-    *err_msg_out = err_msg;
-  } else {
-    tor_free(err_msg);
-  }
-  *key_new_alg_out = key_new_alg;
-  *key_new_blob_out = key_new_blob;
-
-  return ret;
-}
-
-/** Helper function to handle parsing a ClientAuth argument to the
- * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
- * and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned auth_client and err_msg.
- *
- * If 'created' is specified, it will be set to 1 when a new cookie has
- * been generated.
- */
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg)
-{
-  int ok = 0;
-
-  tor_assert(arg);
-  tor_assert(created);
-  tor_assert(err_msg);
-  *err_msg = NULL;
-
-  smartlist_t *auth_args = smartlist_new();
-  rend_authorized_client_t *client =
-    tor_malloc_zero(sizeof(rend_authorized_client_t));
-  smartlist_split_string(auth_args, arg, ":", 0, 0);
-  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
-    *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n");
-    goto err;
-  }
-  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
-  if (smartlist_len(auth_args) == 2) {
-    char *decode_err_msg = NULL;
-    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
-                                client->descriptor_cookie,
-                                NULL, &decode_err_msg) < 0) {
-      tor_assert(decode_err_msg);
-      tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg);
-      tor_free(decode_err_msg);
-      goto err;
-    }
-    *created = 0;
-  } else {
-    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
-    *created = 1;
-  }
-
-  if (!rend_valid_client_name(client->client_name)) {
-    *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n");
-    goto err;
-  }
-
-  ok = 1;
- err:
-  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
-  smartlist_free(auth_args);
-  if (!ok) {
-    rend_authorized_client_free(client);
-    client = NULL;
-  }
-  return client;
-}
-
-/** Called when we get a DEL_ONION command; parse the body, and remove
- * the existing ephemeral Onion Service. */
-static int
-handle_control_del_onion(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  int hs_version = 0;
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("DEL_ONION", conn, body, 1, 1);
-  if (!args)
-    return 0;
-
-  const char *service_id = smartlist_get(args, 0);
-  if (rend_valid_v2_service_id(service_id)) {
-    hs_version = HS_VERSION_TWO;
-  } else if (hs_address_is_valid(service_id)) {
-    hs_version = HS_VERSION_THREE;
-  } else {
-    connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n");
-    goto out;
-  }
-
-  /* Determine if the onion service belongs to this particular control
-   * connection, or if it is in the global list of detached services.  If it
-   * is in neither, either the service ID is invalid in some way, or it
-   * explicitly belongs to a different control connection, and an error
-   * should be returned.
-   */
-  smartlist_t *services[2] = {
-    conn->ephemeral_onion_services,
-    detached_onion_services
-  };
-  smartlist_t *onion_services = NULL;
-  int idx = -1;
-  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
-    idx = smartlist_string_pos(services[i], service_id);
-    if (idx != -1) {
-      onion_services = services[i];
-      break;
-    }
-  }
-  if (onion_services == NULL) {
-    connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n");
-  } else {
-    int ret = -1;
-    switch (hs_version) {
-    case HS_VERSION_TWO:
-      ret = rend_service_del_ephemeral(service_id);
-      break;
-    case HS_VERSION_THREE:
-      ret = hs_service_del_ephemeral(service_id);
-      break;
-    default:
-      /* The ret value will be -1 thus hitting the warning below. This should
-       * never happen because of the check at the start of the function. */
-      break;
-    }
-    if (ret < 0) {
-      /* This should *NEVER* fail, since the service is on either the
-       * per-control connection list, or the global one.
-       */
-      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
-               escaped(service_id));
-      tor_fragile_assert();
-    }
-
-    /* Remove/scrub the service_id from the appropriate list. */
-    char *cp = smartlist_get(onion_services, idx);
-    smartlist_del(onion_services, idx);
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-
-    send_control_done(conn);
-  }
-
- out:
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
 /** Called when <b>conn</b> has no more bytes left on its outbuf. */
 int
 connection_control_finished_flushing(control_connection_t *conn)
@@ -5349,6 +279,44 @@ peek_connection_has_http_command(connection_t *conn)
   return peek_buf_has_http_command(conn->inbuf);
 }
 
+/**
+ * Helper: take a nul-terminated command of given length, and find where the
+ * command starts and the arguments begin.  Separate them, allocate a new
+ * string in <b>current_cmd_out</b> for the command, and return a pointer
+ * to the arguments.
+ **/
+STATIC char *
+control_split_incoming_command(char *incoming_cmd,
+                               size_t *data_len,
+                               char **current_cmd_out)
+{
+  const bool is_multiline = *data_len && incoming_cmd[0] == '+';
+  size_t cmd_len = 0;
+  while (cmd_len < *data_len
+         && !TOR_ISSPACE(incoming_cmd[cmd_len]))
+    ++cmd_len;
+
+  *current_cmd_out = tor_memdup_nulterm(incoming_cmd, cmd_len);
+  char *args = incoming_cmd+cmd_len;
+  tor_assert(*data_len>=cmd_len);
+  *data_len -= cmd_len;
+  if (is_multiline) {
+    // Only match horizontal space: any line after the first is data,
+    // not arguments.
+    while ((*args == '\t' || *args == ' ') && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  } else {
+    while (TOR_ISSPACE(*args) && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  }
+
+  return args;
+}
+
 static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy"
   "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n"
@@ -5375,6 +343,60 @@ static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "</body>\n"
   "</html>\n";
 
+/** Return an error on a control connection that tried to use the v0 protocol.
+ */
+static void
+control_send_v0_reject(control_connection_t *conn)
+{
+  size_t body_len;
+  char buf[128];
+  set_uint16(buf+2, htons(0x0000)); /* type == error */
+  set_uint16(buf+4, htons(0x0001)); /* code == internal error */
+  strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
+          "and later; upgrade your controller.",
+          sizeof(buf)-6);
+  body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
+  set_uint16(buf+0, htons(body_len));
+  connection_buf_add(buf, 4+body_len, TO_CONN(conn));
+
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Return an error on a control connection that tried to use HTTP.
+ */
+static void
+control_send_http_reject(control_connection_t *conn)
+{
+  connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
+  log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Check if a control connection has tried to use a known invalid protocol.
+ * If it has, then:
+ *  - send a reject response,
+ *  - log a notice-level message, and
+ *  - return false. */
+static bool
+control_protocol_is_valid(control_connection_t *conn)
+{
+  /* Detect v0 commands and send a "no more v0" message. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_control0_command(TO_CONN(conn))) {
+    control_send_v0_reject(conn);
+    return 0;
+  }
+
+  /* If the user has the HTTP proxy port and the control port confused. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_http_command(TO_CONN(conn))) {
+    control_send_http_reject(conn);
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Called when data has arrived on a v1 control connection: Try to fetch
  * commands from conn->inbuf, and execute them.
  */
@@ -5383,7 +405,6 @@ connection_control_process_inbuf(control_connection_t *conn)
 {
   size_t data_len;
   uint32_t cmd_data_len;
-  int cmd_len;
   char *args;
 
   tor_assert(conn);
@@ -5396,30 +417,7 @@ connection_control_process_inbuf(control_connection_t *conn)
     conn->incoming_cmd_cur_len = 0;
   }
 
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_control0_command(TO_CONN(conn))) {
-    /* Detect v0 commands and send a "no more v0" message. */
-    size_t body_len;
-    char buf[128];
-    set_uint16(buf+2, htons(0x0000)); /* type == error */
-    set_uint16(buf+4, htons(0x0001)); /* code == internal error */
-    strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
-            "and later; upgrade your controller.",
-            sizeof(buf)-6);
-    body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
-    set_uint16(buf+0, htons(body_len));
-    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
-
-    connection_mark_and_flush(TO_CONN(conn));
-    return 0;
-  }
-
-  /* If the user has the HTTP proxy port and the control port confused. */
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_http_command(TO_CONN(conn))) {
-    connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
-    log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
-    connection_mark_and_flush(TO_CONN(conn));
+  if (!control_protocol_is_valid(conn)) {
     return 0;
   }
 
@@ -5438,7 +436,7 @@ connection_control_process_inbuf(control_connection_t *conn)
         return 0;
       else if (r == -1) {
         if (data_len + conn->incoming_cmd_cur_len > MAX_COMMAND_LINE_LENGTH) {
-          connection_write_str_to_buf("500 Line too long.\r\n", conn);
+          control_write_endreply(conn, 500, "Line too long.");
           connection_stop_reading(TO_CONN(conn));
           connection_mark_and_flush(TO_CONN(conn));
         }
@@ -5475,22 +473,15 @@ connection_control_process_inbuf(control_connection_t *conn)
     /* Otherwise, read another line. */
   }
   data_len = conn->incoming_cmd_cur_len;
+
   /* Okay, we now have a command sitting on conn->incoming_cmd. See if we
    * recognize it.
    */
-  cmd_len = 0;
-  while ((size_t)cmd_len < data_len
-         && !TOR_ISSPACE(conn->incoming_cmd[cmd_len]))
-    ++cmd_len;
-
-  conn->incoming_cmd[cmd_len]='\0';
-  args = conn->incoming_cmd+cmd_len+1;
-  tor_assert(data_len>(size_t)cmd_len);
-  data_len -= (cmd_len+1); /* skip the command and NUL we added after it */
-  while (TOR_ISSPACE(*args)) {
-    ++args;
-    --data_len;
-  }
+  tor_free(conn->current_cmd);
+  args = control_split_incoming_command(conn->incoming_cmd, &data_len,
+                                        &conn->current_cmd);
+  if (BUG(!conn->current_cmd))
+    return -1;
 
   /* If the connection is already closing, ignore further commands */
   if (TO_CONN(conn)->marked_for_close) {
@@ -5498,1448 +489,50 @@ connection_control_process_inbuf(control_connection_t *conn)
   }
 
   /* Otherwise, Quit is always valid. */
-  if (!strcasecmp(conn->incoming_cmd, "QUIT")) {
-    connection_write_str_to_buf("250 closing connection\r\n", conn);
+  if (!strcasecmp(conn->current_cmd, "QUIT")) {
+    control_write_endreply(conn, 250, "closing connection");
     connection_mark_and_flush(TO_CONN(conn));
     return 0;
   }
 
   if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      !is_valid_initial_command(conn, conn->incoming_cmd)) {
-    connection_write_str_to_buf("514 Authentication required.\r\n", conn);
+      !is_valid_initial_command(conn, conn->current_cmd)) {
+    control_write_endreply(conn, 514, "Authentication required.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
   if (data_len >= UINT32_MAX) {
-    connection_write_str_to_buf("500 A 4GB command? Nice try.\r\n", conn);
+    control_write_endreply(conn, 500, "A 4GB command? Nice try.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
-  /* XXXX Why is this not implemented as a table like the GETINFO
-   * items are?  Even handling the plus signs at the beginnings of
-   * commands wouldn't be very hard with proper macros. */
   cmd_data_len = (uint32_t)data_len;
-  if (!strcasecmp(conn->incoming_cmd, "SETCONF")) {
-    if (handle_control_setconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) {
-    if (handle_control_resetconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) {
-    if (handle_control_getconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) {
-    if (handle_control_loadconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) {
-    if (handle_control_setevents(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) {
-    if (handle_control_authenticate(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) {
-    if (handle_control_saveconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) {
-    if (handle_control_signal(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) {
-    if (handle_control_takeownership(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) {
-    if (handle_control_mapaddress(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) {
-    if (handle_control_getinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) {
-    if (handle_control_extendcircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) {
-    if (handle_control_setcircuitpurpose(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) {
-    connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn);
-  } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) {
-    if (handle_control_attachstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) {
-    if (handle_control_postdescriptor(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) {
-    if (handle_control_redirectstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) {
-    if (handle_control_closestream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) {
-    if (handle_control_closecircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) {
-    if (handle_control_usefeature(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) {
-    if (handle_control_resolve(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) {
-    if (handle_control_protocolinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) {
-    if (handle_control_authchallenge(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) {
-    if (handle_control_dropguards(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) {
-    if (handle_control_hsfetch(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) {
-    if (handle_control_hspost(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) {
-    int ret = handle_control_add_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the private key. */
-    if (ret)
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) {
-    int ret = handle_control_del_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */
-    if (ret)
-      return -1;
-  } else {
-    connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n",
-                             conn->incoming_cmd);
-  }
+  if (handle_control_command(conn, cmd_data_len, args) < 0)
+    return -1;
 
   conn->incoming_cmd_cur_len = 0;
   goto again;
 }
 
-/** Something major has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-int
-control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
-                             int reason_code)
-{
-  const char *status;
-  char reasons[64] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
-    return 0;
-  tor_assert(circ);
-
-  switch (tp)
-    {
-    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case CIRC_EVENT_BUILT: status = "BUILT"; break;
-    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
-    case CIRC_EVENT_FAILED: status = "FAILED"; break;
-    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
-    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
-    char unk_reason_buf[16];
-    if (!reason_str) {
-      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
-      reason_str = unk_reason_buf;
-    }
-    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
-    } else {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=%s", reason_str);
-    }
-  }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS,
-                                "650 CIRC %lu %s%s%s%s\r\n",
-                                (unsigned long)circ->global_identifier,
-                                status, sp,
-                                circdesc,
-                                reasons);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/** Something minor has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-static int
-control_event_circuit_status_minor(origin_circuit_t *circ,
-                                   circuit_status_minor_event_t e,
-                                   int purpose, const struct timeval *tv)
-{
-  const char *event_desc;
-  char event_tail[160] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
-    return 0;
-  tor_assert(circ);
-
-  switch (e)
-    {
-    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
-      event_desc = "PURPOSE_CHANGED";
-
-      {
-        /* event_tail can currently be up to 68 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "");
-      }
-
-      break;
-    case CIRC_MINOR_EVENT_CANNIBALIZED:
-      event_desc = "CANNIBALIZED";
-
-      {
-        /* event_tail can currently be up to 130 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        const struct timeval *old_timestamp_began = tv;
-        char tbuf[ISO_TIME_USEC_LEN+1];
-        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
-
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "",
-                     tbuf);
-      }
-
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
-                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
-                       (unsigned long)circ->global_identifier,
-                       event_desc, sp,
-                       circdesc,
-                       event_tail);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
- * interested controllers.
- */
-int
-control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                      int old_purpose)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-                                            old_purpose,
-                                            NULL);
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
- * created-time from <b>old_tv_created</b>: tell any interested controllers.
- */
-int
-control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                   int old_purpose,
-                                   const struct timeval *old_tv_created)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_CANNIBALIZED,
-                                            old_purpose,
-                                            old_tv_created);
-}
-
-/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
- * <b>buf</b>, determine the address:port combination requested on
- * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
- * failure. */
-static int
-write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
-{
-  char buf2[256];
-  if (conn->chosen_exit_name)
-    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
-      return -1;
-  if (!conn->socks_request)
-    return -1;
-  if (tor_snprintf(buf, len, "%s%s%s:%d",
-               conn->socks_request->address,
-               conn->chosen_exit_name ? buf2 : "",
-               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
-                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
-               conn->socks_request->port)<0)
-    return -1;
-  return 0;
-}
-
-/** Something has happened to the stream associated with AP connection
- * <b>conn</b>: tell any interested control connections. */
-int
-control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
-                            int reason_code)
-{
-  char reason_buf[64];
-  char addrport_buf[64];
-  const char *status;
-  circuit_t *circ;
-  origin_circuit_t *origin_circ = NULL;
-  char buf[256];
-  const char *purpose = "";
-  tor_assert(conn->socks_request);
-
-  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
-    return 0;
-
-  if (tp == STREAM_EVENT_CLOSED &&
-      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
-    return 0;
-
-  write_stream_target_to_buf(conn, buf, sizeof(buf));
-
-  reason_buf[0] = '\0';
-  switch (tp)
-    {
-    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
-    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
-    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
-    case STREAM_EVENT_FAILED: status = "FAILED"; break;
-    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
-    case STREAM_EVENT_NEW: status = "NEW"; break;
-    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
-    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
-    case STREAM_EVENT_REMAP: status = "REMAP"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (reason_code && (tp == STREAM_EVENT_FAILED ||
-                      tp == STREAM_EVENT_CLOSED ||
-                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
-    const char *reason_str = stream_end_reason_to_control_string(reason_code);
-    char *r = NULL;
-    if (!reason_str) {
-      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
-      reason_str = r;
-    }
-    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=END REMOTE_REASON=%s", reason_str);
-    else
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=%s", reason_str);
-    tor_free(r);
-  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
-    switch (reason_code) {
-    case REMAP_STREAM_SOURCE_CACHE:
-      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
-      break;
-    case REMAP_STREAM_SOURCE_EXIT:
-      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
-      break;
-    default:
-      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
-                   reason_code);
-      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
-      break;
-    }
-  }
-
-  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
-    /*
-     * When the control conn is an AF_UNIX socket and we have no address,
-     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
-     * dnsserv.c.
-     */
-    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
-      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
-                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
-    } else {
-      /*
-       * else leave it blank so control on AF_UNIX doesn't need to make
-       * something up.
-       */
-      addrport_buf[0] = '\0';
-    }
-  } else {
-    addrport_buf[0] = '\0';
-  }
-
-  if (tp == STREAM_EVENT_NEW_RESOLVE) {
-    purpose = " PURPOSE=DNS_REQUEST";
-  } else if (tp == STREAM_EVENT_NEW) {
-    if (conn->use_begindir) {
-      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
-      int linked_dir_purpose = -1;
-      if (linked && linked->type == CONN_TYPE_DIR)
-        linked_dir_purpose = linked->purpose;
-      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
-        purpose = " PURPOSE=DIR_UPLOAD";
-      else
-        purpose = " PURPOSE=DIR_FETCH";
-    } else
-      purpose = " PURPOSE=USER";
-  }
-
-  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-  if (circ && CIRCUIT_IS_ORIGIN(circ))
-    origin_circ = TO_ORIGIN_CIRCUIT(circ);
-  send_control_event(EVENT_STREAM_STATUS,
-                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
-                     (ENTRY_TO_CONN(conn)->global_identifier),
-                     status,
-                        origin_circ?
-                           (unsigned long)origin_circ->global_identifier : 0ul,
-                        buf, reason_buf, addrport_buf, purpose);
-
-  /* XXX need to specify its intended exit, etc? */
-
-  return 0;
-}
-
-/** Figure out the best name for the target router of an OR connection
- * <b>conn</b>, and write it into the <b>len</b>-character buffer
- * <b>name</b>. */
-static void
-orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
-{
-  const node_t *node = node_get_by_id(conn->identity_digest);
-  if (node) {
-    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
-    node_get_verbose_nickname(node, name);
-  } else if (! tor_digest_is_zero(conn->identity_digest)) {
-    name[0] = '$';
-    base16_encode(name+1, len-1, conn->identity_digest,
-                  DIGEST_LEN);
-  } else {
-    tor_snprintf(name, len, "%s:%d",
-                 conn->base_.address, conn->base_.port);
-  }
-}
-
-/** Called when the status of an OR connection <b>conn</b> changes: tell any
- * interested control connections. <b>tp</b> is the new status for the
- * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
- * may be the reason why.
- */
-int
-control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
-                             int reason)
-{
-  int ncircs = 0;
-  const char *status;
-  char name[128];
-  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
-
-  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
-    return 0;
-
-  switch (tp)
-    {
-    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
-    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
-    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
-    case OR_CONN_EVENT_NEW: status = "NEW"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (conn->chan) {
-    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
-  } else {
-    ncircs = 0;
-  }
-  ncircs += connection_or_get_num_circuits(conn);
-  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
-    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
-  }
-
-  orconn_target_get_name(name, sizeof(name), conn);
-  send_control_event(EVENT_OR_CONN_STATUS,
-                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
-                              name, status,
-                              reason ? " REASON=" : "",
-                              orconn_end_reason_to_control_string(reason),
-                              ncircs_buf,
-                              (conn->base_.global_identifier));
-
-  return 0;
-}
-
-/**
- * Print out STREAM_BW event for a single conn
- */
-int
-control_event_stream_bandwidth(edge_connection_t *edge_conn)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    if (!edge_conn->n_read && !edge_conn->n_written)
-      return 0;
-
-    tor_gettimeofday(&now);
-    format_iso_time_nospace_usec(tbuf, &now);
-    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                       (edge_conn->base_.global_identifier),
-                       (unsigned long)edge_conn->n_read,
-                       (unsigned long)edge_conn->n_written,
-                       tbuf);
-
-    edge_conn->n_written = edge_conn->n_read = 0;
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth streams have used. */
-int
-control_event_stream_bandwidth_used(void)
-{
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    smartlist_t *conns = get_connection_array();
-    edge_connection_t *edge_conn;
-    struct timeval now;
-    char tbuf[ISO_TIME_USEC_LEN+1];
-
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
-    {
-        if (conn->type != CONN_TYPE_AP)
-          continue;
-        edge_conn = TO_EDGE_CONN(conn);
-        if (!edge_conn->n_read && !edge_conn->n_written)
-          continue;
-
-        tor_gettimeofday(&now);
-        format_iso_time_nospace_usec(tbuf, &now);
-        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                           (edge_conn->base_.global_identifier),
-                           (unsigned long)edge_conn->n_read,
-                           (unsigned long)edge_conn->n_written,
-                           tbuf);
-
-        edge_conn->n_written = edge_conn->n_read = 0;
-    }
-    SMARTLIST_FOREACH_END(conn);
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control connections
- * how much bandwidth origin circuits have used. */
-int
-control_event_circ_bandwidth_used(void)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-
-    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
-  }
-  SMARTLIST_FOREACH_END(circ);
-
-  return 0;
-}
-
-/**
- * Emit a CIRC_BW event line for a specific circuit.
- *
- * This function sets the values it emits to 0, and does not emit
- * an event if there is no new data to report since the last call.
- *
- * Therefore, it may be called at any frequency.
- */
-int
-control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-
-  tor_assert(ocirc);
-
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  /* n_read_circ_bw and n_written_circ_bw are always updated
-   * when there is any new cell on a circuit, and set to 0 after
-   * the event, below.
-   *
-   * Therefore, checking them is sufficient to determine if there
-   * is new data to report. */
-  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
-    return 0;
-
-  tor_gettimeofday(&now);
-  format_iso_time_nospace_usec(tbuf, &now);
-  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
-                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
-                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
-                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
-                     ocirc->global_identifier,
-                     (unsigned long)ocirc->n_read_circ_bw,
-                     (unsigned long)ocirc->n_written_circ_bw,
-                     tbuf,
-                     (unsigned long)ocirc->n_delivered_read_circ_bw,
-                     (unsigned long)ocirc->n_overhead_read_circ_bw,
-                     (unsigned long)ocirc->n_delivered_written_circ_bw,
-                     (unsigned long)ocirc->n_overhead_written_circ_bw);
-  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-
-  return 0;
-}
-
-/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
-  * bandwidth counters. */
-int
-control_event_conn_bandwidth(connection_t *conn)
-{
-  const char *conn_type_str;
-  if (!get_options()->TestingEnableConnBwEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
-    return 0;
-  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
-    return 0;
-  switch (conn->type) {
-    case CONN_TYPE_OR:
-      conn_type_str = "OR";
-      break;
-    case CONN_TYPE_DIR:
-      conn_type_str = "DIR";
-      break;
-    case CONN_TYPE_EXIT:
-      conn_type_str = "EXIT";
-      break;
-    default:
-      return 0;
-  }
-  send_control_event(EVENT_CONN_BW,
-                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
-                     "READ=%lu WRITTEN=%lu\r\n",
-                     (conn->global_identifier),
-                     conn_type_str,
-                     (unsigned long)conn->n_read_conn_bw,
-                     (unsigned long)conn->n_written_conn_bw);
-  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth connections have used. */
-int
-control_event_conn_bandwidth_used(void)
-{
-  if (get_options()->TestingEnableConnBwEvent &&
-      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
-    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
-                      control_event_conn_bandwidth(conn));
-  }
-  return 0;
-}
-
-/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
- * cells, removed cells, and waiting times by cell command and direction.
- * Store results in <b>cell_stats</b>.  Free cell statistics of the
- * circuit afterwards. */
-void
-sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
-{
-  memset(cell_stats, 0, sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
-                          const testing_cell_stats_entry_t *, ent) {
-    tor_assert(ent->command <= CELL_COMMAND_MAX_);
-    if (!ent->removed && !ent->exitward) {
-      cell_stats->added_cells_appward[ent->command] += 1;
-    } else if (!ent->removed && ent->exitward) {
-      cell_stats->added_cells_exitward[ent->command] += 1;
-    } else if (!ent->exitward) {
-      cell_stats->removed_cells_appward[ent->command] += 1;
-      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
-    } else {
-      cell_stats->removed_cells_exitward[ent->command] += 1;
-      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
-    }
-  } SMARTLIST_FOREACH_END(ent);
-  circuit_clear_testing_cell_stats(circ);
-}
-
-/** Helper: append a cell statistics string to <code>event_parts</code>,
- * prefixed with <code>key</code>=.  Statistics consist of comma-separated
- * key:value pairs with lower-case command strings as keys and cell
- * numbers or total waiting times as values.  A key:value pair is included
- * if the entry in <code>include_if_non_zero</code> is not zero, but with
- * the (possibly zero) entry from <code>number_to_include</code>.  Both
- * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
- * entry in <code>include_if_non_zero</code> is positive, no string will
- * be added to <code>event_parts</code>. */
-void
-append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
-                             const uint64_t *include_if_non_zero,
-                             const uint64_t *number_to_include)
-{
-  smartlist_t *key_value_strings = smartlist_new();
-  int i;
-  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
-    if (include_if_non_zero[i] > 0) {
-      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
-                             cell_command_to_string(i),
-                             (number_to_include[i]));
-    }
-  }
-  if (smartlist_len(key_value_strings) > 0) {
-    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
-    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
-    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
-    tor_free(joined);
-  }
-  smartlist_free(key_value_strings);
-}
-
-/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
- * CELL_STATS event and write result string to <b>event_string</b>. */
-void
-format_cell_stats(char **event_string, circuit_t *circ,
-                  cell_stats_t *cell_stats)
-{
-  smartlist_t *event_parts = smartlist_new();
-  if (CIRCUIT_IS_ORIGIN(circ)) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "ID=%lu",
-                 (unsigned long)ocirc->global_identifier);
-  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
-    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
-                 (unsigned long)or_circ->p_circ_id);
-    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
-                 (or_circ->p_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "InboundAdded",
-                                 cell_stats->added_cells_appward,
-                                 cell_stats->added_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundRemoved",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->removed_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundTime",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->total_time_appward);
-  }
-  if (circ->n_chan) {
-    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
-                     (unsigned long)circ->n_circ_id);
-    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
-                 (circ->n_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "OutboundAdded",
-                                 cell_stats->added_cells_exitward,
-                                 cell_stats->added_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundRemoved",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->removed_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundTime",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->total_time_exitward);
-  }
-  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
-  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
-  smartlist_free(event_parts);
-}
-
-/** A second or more has elapsed: tell any interested control connection
- * how many cells have been processed for a given circuit. */
-int
-control_event_circuit_cell_stats(void)
-{
-  cell_stats_t *cell_stats;
-  char *event_string;
-  if (!get_options()->TestingEnableCellStatsEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
-    return 0;
-  cell_stats = tor_malloc(sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!circ->testing_cell_stats)
-      continue;
-    sum_up_cell_stats_by_command(circ, cell_stats);
-    format_cell_stats(&event_string, circ, cell_stats);
-    send_control_event(EVENT_CELL_STATS,
-                       "650 CELL_STATS %s\r\n", event_string);
-    tor_free(event_string);
-  }
-  SMARTLIST_FOREACH_END(circ);
-  tor_free(cell_stats);
-  return 0;
-}
-
-/* about 5 minutes worth. */
-#define N_BW_EVENTS_TO_CACHE 300
-/* Index into cached_bw_events to next write. */
-static int next_measurement_idx = 0;
-/* number of entries set in n_measurements */
-static int n_measurements = 0;
-static struct cached_bw_event_s {
-  uint32_t n_read;
-  uint32_t n_written;
-} cached_bw_events[N_BW_EVENTS_TO_CACHE];
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth we used. */
-int
-control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
-{
-  cached_bw_events[next_measurement_idx].n_read = n_read;
-  cached_bw_events[next_measurement_idx].n_written = n_written;
-  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
-    next_measurement_idx = 0;
-  if (n_measurements < N_BW_EVENTS_TO_CACHE)
-    ++n_measurements;
-
-  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
-    send_control_event(EVENT_BANDWIDTH_USED,
-                       "650 BW %lu %lu\r\n",
-                       (unsigned long)n_read,
-                       (unsigned long)n_written);
-  }
-
-  return 0;
-}
-
-STATIC char *
-get_bw_samples(void)
-{
-  int i;
-  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
-    % N_BW_EVENTS_TO_CACHE;
-  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-
-  smartlist_t *elements = smartlist_new();
-
-  for (i = 0; i < n_measurements; ++i) {
-    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-    const struct cached_bw_event_s *bwe = &cached_bw_events[idx];
-
-    smartlist_add_asprintf(elements, "%u,%u",
-                           (unsigned)bwe->n_read,
-                           (unsigned)bwe->n_written);
-
-    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
-  }
-
-  char *result = smartlist_join_strings(elements, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-
-  return result;
-}
-
-/** Called when we are sending a log message to the controllers: suspend
- * sending further log messages to the controllers until we're done.  Used by
- * CONN_LOG_PROTECT. */
-void
-disable_control_logging(void)
-{
-  ++disable_log_messages;
-}
-
-/** We're done sending a log message to the controllers: re-enable controller
- * logging.  Used by CONN_LOG_PROTECT. */
-void
-enable_control_logging(void)
-{
-  if (--disable_log_messages < 0)
-    tor_assert(0);
-}
-
-/** We got a log message: tell any interested control connections. */
-void
-control_event_logmsg(int severity, uint32_t domain, const char *msg)
-{
-  int event;
-
-  /* Don't even think of trying to add stuff to a buffer from a cpuworker
-   * thread. (See #25987 for plan to fix.) */
-  if (! in_main_thread())
-    return;
-
-  if (disable_log_messages)
-    return;
-
-  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
-      severity <= LOG_NOTICE) {
-    char *esc = esc_for_log(msg);
-    ++disable_log_messages;
-    control_event_general_status(severity, "BUG REASON=%s", esc);
-    --disable_log_messages;
-    tor_free(esc);
-  }
-
-  event = log_severity_to_event(severity);
-  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
-    char *b = NULL;
-    const char *s;
-    if (strchr(msg, '\n')) {
-      char *cp;
-      b = tor_strdup(msg);
-      for (cp = b; *cp; ++cp)
-        if (*cp == '\r' || *cp == '\n')
-          *cp = ' ';
-    }
-    switch (severity) {
-      case LOG_DEBUG: s = "DEBUG"; break;
-      case LOG_INFO: s = "INFO"; break;
-      case LOG_NOTICE: s = "NOTICE"; break;
-      case LOG_WARN: s = "WARN"; break;
-      case LOG_ERR: s = "ERR"; break;
-      default: s = "UnknownLogSeverity"; break;
-    }
-    ++disable_log_messages;
-    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
-    if (severity == LOG_ERR) {
-      /* Force a flush, since we may be about to die horribly */
-      queued_events_flush_all(1);
-    }
-    --disable_log_messages;
-    tor_free(b);
-  }
-}
-
-/**
- * Logging callback: called when there is a queued pending log callback.
- */
-void
-control_event_logmsg_pending(void)
-{
-  if (! in_main_thread()) {
-    /* We can't handle this case yet, since we're using a
-     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
-     * use a different mechanism instead: see #25987.
-     **/
-    return;
-  }
-  tor_assert(flush_queued_events_event);
-  mainloop_event_activate(flush_queued_events_event);
-}
-
-/** Called whenever we receive new router descriptors: tell any
- * interested control connections.  <b>routers</b> is a list of
- * routerinfo_t's.
- */
-int
-control_event_descriptors_changed(smartlist_t *routers)
-{
-  char *msg;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
-    return 0;
-
-  {
-    smartlist_t *names = smartlist_new();
-    char *ids;
-    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
-        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
-        router_get_verbose_nickname(b, ri);
-        smartlist_add(names, b);
-      });
-    ids = smartlist_join_strings(names, " ", 0, NULL);
-    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
-    send_control_event_string(EVENT_NEW_DESC,  msg);
-    tor_free(ids);
-    tor_free(msg);
-    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
-    smartlist_free(names);
-  }
-  return 0;
-}
-
-/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
- * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
- * <b>error</b> is non-NULL, it is an error code describing the failure
- * mode of the mapping.
- */
-int
-control_event_address_mapped(const char *from, const char *to, time_t expires,
-                             const char *error, const int cached)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
-    return 0;
-
-  if (expires < 3 || expires == TIME_MAX)
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s NEVER %s%s"
-                                "CACHED=\"%s\"\r\n",
-                                  from, to, error?error:"", error?" ":"",
-                                cached?"YES":"NO");
-  else {
-    char buf[ISO_TIME_LEN+1];
-    char buf2[ISO_TIME_LEN+1];
-    format_local_iso_time(buf,expires);
-    format_iso_time(buf2,expires);
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s \"%s\""
-                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
-                                from, to, buf,
-                                error?error:"", error?" ":"",
-                                buf2, cached?"YES":"NO");
-  }
-
-  return 0;
-}
-
 /** Cached liveness for network liveness events and GETINFO
  */
 
 static int network_is_live = 0;
 
-static int
+int
 get_cached_network_liveness(void)
 {
   return network_is_live;
 }
 
-static void
+void
 set_cached_network_liveness(int liveness)
 {
   network_is_live = liveness;
 }
 
-/** The network liveness has changed; this is called from circuitstats.c
- * whenever we receive a cell, or when timeout expires and we assume the
- * network is down. */
-int
-control_event_network_liveness_update(int liveness)
-{
-  if (liveness > 0) {
-    if (get_cached_network_liveness() <= 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(1);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS UP\r\n");
-    }
-    /* else was already live, no-op */
-  } else {
-    if (get_cached_network_liveness() > 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(0);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS DOWN\r\n");
-    }
-    /* else was already dead, no-op */
-  }
-
-  return 0;
-}
-
-/** Helper function for NS-style events. Constructs and sends an event
- * of type <b>event</b> with string <b>event_string</b> out of the set of
- * networkstatuses <b>statuses</b>. Currently it is used for NS events
- * and NEWCONSENSUS events. */
-static int
-control_event_networkstatus_changed_helper(smartlist_t *statuses,
-                                           uint16_t event,
-                                           const char *event_string)
-{
-  smartlist_t *strs;
-  char *s, *esc = NULL;
-  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
-    return 0;
-
-  strs = smartlist_new();
-  smartlist_add_strdup(strs, "650+");
-  smartlist_add_strdup(strs, event_string);
-  smartlist_add_strdup(strs, "\r\n");
-  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
-    {
-      s = networkstatus_getinfo_helper_single(rs);
-      if (!s) continue;
-      smartlist_add(strs, s);
-    });
-
-  s = smartlist_join_strings(strs, "", 0, NULL);
-  write_escaped_data(s, strlen(s), &esc);
-  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
-  smartlist_free(strs);
-  tor_free(s);
-  send_control_event_string(event,  esc);
-  send_control_event_string(event,
-                            "650 OK\r\n");
-
-  tor_free(esc);
-  return 0;
-}
-
-/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
- * an NS event to any controller that cares. */
-int
-control_event_networkstatus_changed(smartlist_t *statuses)
-{
-  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
-}
-
-/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
- * event consisting of an NS-style line for each relay in the consensus. */
-int
-control_event_newconsensus(const networkstatus_t *consensus)
-{
-  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
-    return 0;
-  return control_event_networkstatus_changed_helper(
-           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
-}
-
-/** Called when we compute a new circuitbuildtimeout */
-int
-control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                               const char *args)
-{
-  const char *type_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
-    return 0;
-
-  switch (type) {
-    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
-      type_string = "COMPUTED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESET:
-      type_string = "RESET";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
-      type_string = "SUSPENDED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_DISCARD:
-      type_string = "DISCARD";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESUME:
-      type_string = "RESUME";
-      break;
-    default:
-      type_string = "UNKNOWN";
-      break;
-  }
-
-  send_control_event(EVENT_BUILDTIMEOUT_SET,
-                     "650 BUILDTIMEOUT_SET %s %s\r\n",
-                     type_string, args);
-
-  return 0;
-}
-
-/** Called when a signal has been processed from signal_callback */
-int
-control_event_signal(uintptr_t signal_num)
-{
-  const char *signal_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
-    return 0;
-
-  switch (signal_num) {
-    case SIGHUP:
-      signal_string = "RELOAD";
-      break;
-    case SIGUSR1:
-      signal_string = "DUMP";
-      break;
-    case SIGUSR2:
-      signal_string = "DEBUG";
-      break;
-    case SIGNEWNYM:
-      signal_string = "NEWNYM";
-      break;
-    case SIGCLEARDNSCACHE:
-      signal_string = "CLEARDNSCACHE";
-      break;
-    case SIGHEARTBEAT:
-      signal_string = "HEARTBEAT";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
-               (unsigned long)signal_num);
-      return -1;
-  }
-
-  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
-                     signal_string);
-  return 0;
-}
-
-/** Called when a single local_routerstatus_t has changed: Sends an NS event
- * to any controller that cares. */
-int
-control_event_networkstatus_changed_single(const routerstatus_t *rs)
-{
-  smartlist_t *statuses;
-  int r;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NS))
-    return 0;
-
-  statuses = smartlist_new();
-  smartlist_add(statuses, (void*)rs);
-  r = control_event_networkstatus_changed(statuses);
-  smartlist_free(statuses);
-  return r;
-}
-
-/** Our own router descriptor has changed; tell any controllers that care.
- */
-int
-control_event_my_descriptor_changed(void)
-{
-  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
-  return 0;
-}
-
-/** Helper: sends a status event where <b>type</b> is one of
- * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
- * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
- * string corresponding to <b>args</b>. */
-static int
-control_event_status(int type, int severity, const char *format, va_list args)
-{
-  char *user_buf = NULL;
-  char format_buf[160];
-  const char *status, *sev;
-
-  switch (type) {
-    case EVENT_STATUS_GENERAL:
-      status = "STATUS_GENERAL";
-      break;
-    case EVENT_STATUS_CLIENT:
-      status = "STATUS_CLIENT";
-      break;
-    case EVENT_STATUS_SERVER:
-      status = "STATUS_SERVER";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status type %d", type);
-      return -1;
-  }
-  switch (severity) {
-    case LOG_NOTICE:
-      sev = "NOTICE";
-      break;
-    case LOG_WARN:
-      sev = "WARN";
-      break;
-    case LOG_ERR:
-      sev = "ERR";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
-      return -1;
-  }
-  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
-                   status, sev)<0) {
-    log_warn(LD_BUG, "Format string too long.");
-    return -1;
-  }
-  tor_vasprintf(&user_buf, format, args);
-
-  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
-  tor_free(user_buf);
-  return 0;
-}
-
-#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
-  int r;                                                        \
-  do {                                                          \
-    va_list ap;                                                 \
-    if (!EVENT_IS_INTERESTING(event))                           \
-      return 0;                                                 \
-                                                                \
-    va_start(ap, format);                                       \
-    r = control_event_status((event), (sev), format, ap);       \
-    va_end(ap);                                                 \
-  } while (0)
-
-/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_general_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_general_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_client_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_client_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_server_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_server_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Called when the status of an entry guard with the given <b>nickname</b>
- * and identity <b>digest</b> has changed to <b>status</b>: tells any
- * controllers that care. */
-int
-control_event_guard(const char *nickname, const char *digest,
-                    const char *status)
-{
-  char hbuf[HEX_DIGEST_LEN+1];
-  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
-  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
-    return 0;
-
-  {
-    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
-    const node_t *node = node_get_by_id(digest);
-    if (node) {
-      node_get_verbose_nickname(node, buf);
-    } else {
-      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
-    }
-    send_control_event(EVENT_GUARD,
-                       "650 GUARD ENTRY %s %s\r\n", buf, status);
-  }
-  return 0;
-}
-
-/** Called when a configuration option changes. This is generally triggered
- * by SETCONF requests and RELOAD/SIGHUP signals. The <b>elements</b> is
- * a smartlist_t containing (key, value, ...) pairs in sequence.
- * <b>value</b> can be NULL. */
-int
-control_event_conf_changed(const smartlist_t *elements)
-{
-  int i;
-  char *result;
-  smartlist_t *lines;
-  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) ||
-      smartlist_len(elements) == 0) {
-    return 0;
-  }
-  lines = smartlist_new();
-  for (i = 0; i < smartlist_len(elements); i += 2) {
-    char *k = smartlist_get(elements, i);
-    char *v = smartlist_get(elements, i+1);
-    if (v == NULL) {
-      smartlist_add_asprintf(lines, "650-%s", k);
-    } else {
-      smartlist_add_asprintf(lines, "650-%s=%s", k, v);
-    }
-  }
-  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
-  send_control_event(EVENT_CONF_CHANGED,
-    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
-  tor_free(result);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  return 0;
-}
-
-/** Helper: Return a newly allocated string containing a path to the
- * file where we store our authentication cookie. */
-char *
-get_controller_cookie_file_name(void)
-{
-  const or_options_t *options = get_options();
-  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
-    return tor_strdup(options->CookieAuthFile);
-  } else {
-    return get_datadir_fname("control_auth_cookie");
-  }
-}
-
-/* Initialize the cookie-based authentication system of the
- * ControlPort. If <b>enabled</b> is 0, then disable the cookie
- * authentication system.  */
-int
-init_control_cookie_authentication(int enabled)
-{
-  char *fname = NULL;
-  int retval;
-
-  if (!enabled) {
-    authentication_cookie_is_set = 0;
-    return 0;
-  }
-
-  fname = get_controller_cookie_file_name();
-  retval = init_cookie_authentication(fname, "", /* no header */
-                                      AUTHENTICATION_COOKIE_LEN,
-                                   get_options()->CookieAuthFileGroupReadable,
-                                      &authentication_cookie,
-                                      &authentication_cookie_is_set);
-  tor_free(fname);
-  return retval;
-}
-
 /** A copy of the process specifier of Tor's owning controller, or
  * NULL if this Tor instance is not currently owned by a process. */
 static char *owning_controller_process_spec = NULL;
@@ -7008,895 +601,12 @@ monitor_owning_controller_process(const char *process_spec)
   }
 }
 
-/** Convert the name of a bootstrapping phase <b>s</b> into strings
- * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
-static int
-bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
-                           const char **summary)
-{
-  switch (s) {
-    case BOOTSTRAP_STATUS_UNDEF:
-      *tag = "undef";
-      *summary = "Undefined";
-      break;
-    case BOOTSTRAP_STATUS_STARTING:
-      *tag = "starting";
-      *summary = "Starting";
-      break;
-    case BOOTSTRAP_STATUS_CONN_DIR:
-      *tag = "conn_dir";
-      *summary = "Connecting to directory server";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE:
-      *tag = "status_handshake";
-      *summary = "Finishing handshake";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_DIR:
-      *tag = "handshake_dir";
-      *summary = "Finishing handshake with directory server";
-      break;
-    case BOOTSTRAP_STATUS_ONEHOP_CREATE:
-      *tag = "onehop_create";
-      *summary = "Establishing an encrypted directory connection";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_STATUS:
-      *tag = "requesting_status";
-      *summary = "Asking for networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_STATUS:
-      *tag = "loading_status";
-      *summary = "Loading networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_KEYS:
-      *tag = "loading_keys";
-      *summary = "Loading authority key certs";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS:
-      *tag = "requesting_descriptors";
-      /* XXXX this appears to incorrectly report internal on most loads */
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Asking for relay descriptors for internal paths" :
-        "Asking for relay descriptors";
-      break;
-    /* If we're sure there are no exits in the consensus,
-     * inform the controller by adding "internal"
-     * to the status summaries.
-     * (We only check this while loading descriptors,
-     * so we may not know in the earlier stages.)
-     * But if there are exits, we can't be sure whether
-     * we're creating internal or exit paths/circuits.
-     * XXXX Or should be use different tags or statuses
-     * for internal and exit/all? */
-    case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS:
-      *tag = "loading_descriptors";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Loading relay descriptors for internal paths" :
-        "Loading relay descriptors";
-      break;
-    case BOOTSTRAP_STATUS_CONN_OR:
-      *tag = "conn_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Connecting to the Tor network internally" :
-        "Connecting to the Tor network";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_OR:
-      *tag = "handshake_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Finishing handshake with first hop of internal circuit" :
-        "Finishing handshake with first hop";
-      break;
-    case BOOTSTRAP_STATUS_CIRCUIT_CREATE:
-      *tag = "circuit_create";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Establishing an internal Tor circuit" :
-        "Establishing a Tor circuit";
-      break;
-    case BOOTSTRAP_STATUS_DONE:
-      *tag = "done";
-      *summary = "Done";
-      break;
-    default:
-//      log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s);
-      *tag = *summary = "unknown";
-      return -1;
-  }
-  return 0;
-}
-
-/** What percentage through the bootstrap process are we? We remember
- * this so we can avoid sending redundant bootstrap status events, and
- * so we can guess context for the bootstrap messages which are
- * ambiguous. It starts at 'undef', but gets set to 'starting' while
- * Tor initializes. */
-static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_percent, but only takes on the enumerated values in
- * bootstrap_status_t.
- */
-static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** As bootstrap_percent, but holds the bootstrapping level at which we last
- * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
- * to avoid flooding the log with a new message every time we get a few more
- * microdescriptors */
-static int notice_bootstrap_percent = 0;
-
-/** How many problems have we had getting to the next bootstrapping phase?
- * These include failure to establish a connection to a Tor relay,
- * failures to finish the TLS handshake, failures to validate the
- * consensus document, etc. */
-static int bootstrap_problems = 0;
-
-/** We only tell the controller once we've hit a threshold of problems
- * for the current phase. */
-#define BOOTSTRAP_PROBLEM_THRESHOLD 10
-
-/** When our bootstrapping progress level changes, but our bootstrapping
- * status has not advanced, we only log at NOTICE when we have made at least
- * this much progress.
- */
-#define BOOTSTRAP_PCT_INCREMENT 5
-
-/** Do the actual logging and notifications for
- * control_event_bootstrap().  Doesn't change any state beyond that.
- */
-static void
-control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
-                             int progress)
-{
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *tag, *summary;
-
-  bootstrap_status_to_string(status, &tag, &summary);
-  /* Locally reset status if there's incremental progress */
-  if (progress)
-    status = progress;
-
-  tor_log(loglevel, LD_CONTROL,
-          "Bootstrapped %d%%: %s", status, summary);
-  tor_snprintf(buf, sizeof(buf),
-               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
-               status, tag, summary);
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "NOTICE %s", buf);
-  control_event_client_status(LOG_NOTICE, "%s", buf);
-}
-
-/** Called when Tor has made progress at bootstrapping its directory
- * information and initial circuits.
- *
- * <b>status</b> is the new status, that is, what task we will be doing
- * next. <b>progress</b> is zero if we just started this task, else it
- * represents progress on the task.
- */
-void
-control_event_bootstrap(bootstrap_status_t status, int progress)
-{
-  int loglevel = LOG_NOTICE;
-
-  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  /* special case for handshaking status, since our TLS handshaking code
-   * can't distinguish what the connection is going to be for. */
-  if (status == BOOTSTRAP_STATUS_HANDSHAKE) {
-    if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_DIR;
-    } else {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_OR;
-    }
-  }
-
-  if (status <= bootstrap_percent) {
-    /* If there's no new progress, return early. */
-    if (!progress || progress <= bootstrap_percent)
-      return;
-    /* Log at INFO if not enough progress happened. */
-    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
-      loglevel = LOG_INFO;
-  }
-
-  control_event_bootstrap_core(loglevel, status, progress);
-
-  if (status > bootstrap_percent) {
-    bootstrap_phase = status; /* new milestone reached */
-    bootstrap_percent = status;
-  }
-  if (progress > bootstrap_percent) {
-    /* incremental progress within a milestone */
-    bootstrap_percent = progress;
-    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
-  }
-  if (loglevel == LOG_NOTICE &&
-      bootstrap_percent > notice_bootstrap_percent) {
-    /* Remember that we gave a notice at this level. */
-    notice_bootstrap_percent = bootstrap_percent;
-  }
-}
-
-/** Flag whether we've opened an OR_CONN yet  */
-static int bootstrap_first_orconn = 0;
-
-/** Like bootstrap_phase, but for (possibly deferred) directory progress */
-static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
-static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-
-/** Defer directory info bootstrap events until we have successfully
- * completed our first connection to a router.  */
-void
-control_event_boot_dir(bootstrap_status_t status, int progress)
-{
-  if (status > bootstrap_dir_progress) {
-    bootstrap_dir_progress = status;
-    bootstrap_dir_phase = status;
-  }
-  if (progress && progress >= bootstrap_dir_progress) {
-    bootstrap_dir_progress = progress;
-  }
-
-  /* Don't report unless we have successfully opened at least one OR_CONN */
-  if (!bootstrap_first_orconn)
-    return;
-
-  control_event_bootstrap(status, progress);
-}
-
-/** Set a flag to allow reporting of directory bootstrap progress.
- * (Code that reports completion of an OR_CONN calls this.)  Also,
- * report directory progress so far. */
-void
-control_event_boot_first_orconn(void)
-{
-  bootstrap_first_orconn = 1;
-  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a human-readable hint
- * as to why, and <b>reason</b> provides a controller-facing short
- * tag.  <b>conn</b> is the connection that caused this problem and
- * can be NULL if a connection cannot be easily identified.
- */
-void
-control_event_bootstrap_problem(const char *warn, const char *reason,
-                                const connection_t *conn, int dowarn)
-{
-  int status = bootstrap_percent;
-  const char *tag = "", *summary = "";
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *recommendation = "ignore";
-  int severity;
-  char *or_id = NULL, *hostaddr = NULL;
-  or_connection_t *or_conn = NULL;
-
-  /* bootstrap_percent must not be in "undefined" state here. */
-  tor_assert(status >= 0);
-
-  if (bootstrap_percent == 100)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  bootstrap_problems++;
-
-  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
-    dowarn = 1;
-
-  /* Don't warn about our bootstrapping status if we are hibernating or
-   * shutting down. */
-  if (we_are_hibernating())
-    dowarn = 0;
-
-  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
-
-  severity = dowarn ? LOG_WARN : LOG_INFO;
-
-  if (dowarn)
-    recommendation = "warn";
-
-  if (conn && conn->type == CONN_TYPE_OR) {
-    /* XXX TO_OR_CONN can't deal with const */
-    or_conn = TO_OR_CONN((connection_t *)conn);
-    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
-  } else {
-    or_id = tor_strdup("?");
-  }
-
-  if (conn)
-    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
-  else
-    hostaddr = tor_strdup("?");
-
-  log_fn(severity,
-         LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; "
-         "count %d; recommendation %s; host %s at %s)",
-         status, summary, warn, reason,
-         bootstrap_problems, recommendation,
-         or_id, hostaddr);
-
-  connection_or_report_broken_states(severity, LD_HANDSHAKE);
-
-  tor_snprintf(buf, sizeof(buf),
-      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
-      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
-      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
-      recommendation,
-      or_id, hostaddr);
-
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "WARN %s", buf);
-  control_event_client_status(LOG_WARN, "%s", buf);
-
-  tor_free(hostaddr);
-  tor_free(or_id);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a hint as to why, and
- * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
- * is the connection that caused this problem.
- */
-MOCK_IMPL(void,
-control_event_bootstrap_prob_or, (const char *warn, int reason,
-                                  or_connection_t *or_conn))
-{
-  int dowarn = 0;
-
-  if (or_conn->have_noted_bootstrap_problem)
-    return;
-
-  or_conn->have_noted_bootstrap_problem = 1;
-
-  if (reason == END_OR_CONN_REASON_NO_ROUTE)
-    dowarn = 1;
-
-  /* If we are using bridges and all our OR connections are now
-     closed, it means that we totally failed to connect to our
-     bridges. Throw a warning. */
-  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
-    dowarn = 1;
-
-  control_event_bootstrap_problem(warn,
-                                  orconn_end_reason_to_control_string(reason),
-                                  TO_CONN(or_conn), dowarn);
-}
-
-/** We just generated a new summary of which countries we've seen clients
- * from recently. Send a copy to the controller in case it wants to
- * display it for the user. */
-void
-control_event_clients_seen(const char *controller_str)
-{
-  send_control_event(EVENT_CLIENTS_SEEN,
-    "650 CLIENTS_SEEN %s\r\n", controller_str);
-}
-
-/** A new pluggable transport called <b>transport_name</b> was
- *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
- *  "server" or "client" depending on the mode of the pluggable
- *  transport.
- *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
- */
-void
-control_event_transport_launched(const char *mode, const char *transport_name,
-                                 tor_addr_t *addr, uint16_t port)
-{
-  send_control_event(EVENT_TRANSPORT_LAUNCHED,
-                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
-                     mode, transport_name, fmt_addr(addr), port);
-}
-
-/** Convert rendezvous auth type to string for HS_DESC control events
- */
-const char *
-rend_auth_type_to_string(rend_auth_type_t auth_type)
-{
-  const char *str;
-
-  switch (auth_type) {
-    case REND_NO_AUTH:
-      str = "NO_AUTH";
-      break;
-    case REND_BASIC_AUTH:
-      str = "BASIC_AUTH";
-      break;
-    case REND_STEALTH_AUTH:
-      str = "STEALTH_AUTH";
-      break;
-    default:
-      str = "UNKNOWN";
-  }
-
-  return str;
-}
-
-/** Return a longname the node whose identity is <b>id_digest</b>. If
- * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
- * returned instead.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-MOCK_IMPL(const char *,
-node_describe_longname_by_id,(const char *id_digest))
-{
-  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
-  node_get_verbose_nickname_by_id(id_digest, longname);
-  return longname;
-}
-
-/** Return either the onion address if the given pointer is a non empty
- * string else the unknown string. */
-static const char *
-rend_hsaddress_str_or_unknown(const char *onion_address)
-{
-  static const char *str_unknown = "UNKNOWN";
-  const char *str_ret = str_unknown;
-
-  /* No valid pointer, unknown it is. */
-  if (!onion_address) {
-    goto end;
-  }
-  /* Empty onion address thus we don't know, unknown it is. */
-  if (onion_address[0] == '\0') {
-    goto end;
-  }
-  /* All checks are good so return the given onion address. */
-  str_ret = onion_address;
-
- end:
-  return str_ret;
-}
-
-/** send HS_DESC requested event.
- *
- * <b>rend_query</b> is used to fetch requested onion address and auth type.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id_base32</b> is the ID of requested hs descriptor.
- * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
- */
-void
-control_event_hs_descriptor_requested(const char *onion_address,
-                                      rend_auth_type_t auth_type,
-                                      const char *id_digest,
-                                      const char *desc_id,
-                                      const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** For an HS descriptor query <b>rend_data</b>, using the
- * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
- * which descriptor ID in the query is the right one.
- *
- * Return a pointer of the binary descriptor ID found in the query's object
- * or NULL if not found. */
-static const char *
-get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
-{
-  int replica;
-  const char *desc_id = NULL;
-  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
-
-  /* Possible if the fetch was done using a descriptor ID. This means that
-   * the HSFETCH command was used. */
-  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
-    desc_id = rend_data_v2->desc_id_fetch;
-    goto end;
-  }
-
-  /* Without a directory fingerprint at this stage, we can't do much. */
-  if (hsdir_fp == NULL) {
-     goto end;
-  }
-
-  /* OK, we have an onion address so now let's find which descriptor ID
-   * is the one associated with the HSDir fingerprint. */
-  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
-       replica++) {
-    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
-
-    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
-      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
-        /* Found it! This descriptor ID is the right one. */
-        desc_id = digest;
-        goto end;
-      }
-    } SMARTLIST_FOREACH_END(fingerprint);
-  }
-
- end:
-  return desc_id;
-}
-
-/** send HS_DESC CREATED event when a local service generates a descriptor.
- *
- * <b>onion_address</b> is service address.
- * <b>desc_id</b> is the descriptor ID.
- * <b>replica</b> is the the descriptor replica number. If it is negative, it
- * is ignored.
- */
-void
-control_event_hs_descriptor_created(const char *onion_address,
-                                    const char *desc_id,
-                                    int replica)
-{
-  char *replica_field = NULL;
-
-  if (BUG(!onion_address || !desc_id)) {
-    return;
-  }
-
-  if (replica >= 0) {
-    tor_asprintf(&replica_field, " REPLICA=%d", replica);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
-                     onion_address, desc_id,
-                     replica_field ? replica_field : "");
-  tor_free(replica_field);
-}
-
-/** send HS_DESC upload event.
- *
- * <b>onion_address</b> is service address.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id</b> is the ID of requested hs descriptor.
- */
-void
-control_event_hs_descriptor_upload(const char *onion_address,
-                                   const char *id_digest,
-                                   const char *desc_id,
-                                   const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!onion_address || !id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
-                     onion_address,
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hsv2_descriptor_received
- * control_event_hsv2_descriptor_failed
- * control_event_hsv3_descriptor_failed
- *
- * So do not call this function directly.
- */
-static void
-event_hs_descriptor_receive_end(const char *action,
-                                const char *onion_address,
-                                const char *desc_id,
-                                rend_auth_type_t auth_type,
-                                const char *hsdir_id_digest,
-                                const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !onion_address)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s %s %s%s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     desc_id ? desc_id : "",
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hs_descriptor_uploaded
- * control_event_hs_descriptor_upload_failed
- *
- * So do not call this function directly.
- */
-void
-control_event_hs_descriptor_upload_end(const char *action,
-                                       const char *onion_address,
-                                       const char *id_digest,
-                                       const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !id_digest)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     node_describe_longname_by_id(id_digest),
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC RECEIVED event
- *
- * called when we successfully received a hidden service descriptor.
- */
-void
-control_event_hsv2_descriptor_received(const char *onion_address,
-                                       const rend_data_t *rend_data,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/* Send HS_DESC RECEIVED event
- *
- * Called when we successfully received a hidden service descriptor. */
-void
-control_event_hsv3_descriptor_received(const char *onion_address,
-                                       const char *desc_id,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/** send HS_DESC UPLOADED event
- *
- * called when we successfully uploaded a hidden service descriptor.
- */
-void
-control_event_hs_descriptor_uploaded(const char *id_digest,
-                                     const char *onion_address)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-
-  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
-                                         id_digest, NULL);
-}
-
-/** Send HS_DESC event to inform controller that query <b>rend_data</b>
- * failed to retrieve hidden service descriptor from directory identified by
- * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
- * add it to REASON= field.
- */
-void
-control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
-                                  desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC event to inform controller that the query to
- * <b>onion_address</b> failed to retrieve hidden service descriptor
- * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
- * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
- * field. */
-void
-control_event_hsv3_descriptor_failed(const char *onion_address,
-                                     const char *desc_id,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !reason)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC_CONTENT event after completion of a successful fetch
- * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
- * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
- * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
- * not trigger the control event. */
-void
-control_event_hs_descriptor_content(const char *onion_address,
-                                    const char *desc_id,
-                                    const char *hsdir_id_digest,
-                                    const char *content)
-{
-  static const char *event_name = "HS_DESC_CONTENT";
-  char *esc_content = NULL;
-
-  if (!onion_address || !desc_id) {
-    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
-             onion_address, desc_id);
-    return;
-  }
-
-  if (content == NULL) {
-    /* Point it to empty content so it can still be escaped. */
-    content = "";
-  }
-  write_escaped_data(content, strlen(content), &esc_content);
-
-  send_control_event(EVENT_HS_DESC_CONTENT,
-                     "650+%s %s %s %s\r\n%s650 OK\r\n",
-                     event_name,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     desc_id,
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     esc_content);
-  tor_free(esc_content);
-}
-
-/** Send HS_DESC event to inform controller upload of hidden service
- * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
- * is not NULL, add it to REASON= field.
- */
-void
-control_event_hs_descriptor_upload_failed(const char *id_digest,
-                                          const char *onion_address,
-                                          const char *reason)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-  control_event_hs_descriptor_upload_end("FAILED", onion_address,
-                                         id_digest, reason);
-}
-
 /** Free any leftover allocated memory of the control.c subsystem. */
 void
 control_free_all(void)
 {
-  smartlist_t *queued_events = NULL;
-
-  stats_prev_n_read = stats_prev_n_written = 0;
-
-  if (authentication_cookie) /* Free the auth cookie */
-    tor_free(authentication_cookie);
-  if (detached_onion_services) { /* Free the detached onion services */
-    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
-    smartlist_free(detached_onion_services);
-  }
-
-  if (queued_control_events_lock) {
-    tor_mutex_acquire(queued_control_events_lock);
-    flush_queued_event_pending = 0;
-    queued_events = queued_control_events;
-    queued_control_events = NULL;
-    tor_mutex_release(queued_control_events_lock);
-  }
-  if (queued_events) {
-    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
-                      queued_event_free(ev));
-    smartlist_free(queued_events);
-  }
-  if (flush_queued_events_event) {
-    mainloop_event_free(flush_queued_events_event);
-    flush_queued_events_event = NULL;
-  }
-  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-  notice_bootstrap_percent = 0;
-  bootstrap_problems = 0;
-  bootstrap_first_orconn = 0;
-  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-  authentication_cookie_is_set = 0;
-  global_event_mask = 0;
-  disable_log_messages = 0;
-  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
-}
-
-#ifdef TOR_UNIT_TESTS
-/* For testing: change the value of global_event_mask */
-void
-control_testing_set_global_event_mask(uint64_t mask)
-{
-  global_event_mask = mask;
+  control_auth_free_all();
+  control_events_free_all();
+  control_cmd_free_all();
+  control_event_bootstrap_reset();
 }
-#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control.h b/src/feature/control/control.h
index a09c1cd11b..8d3595d2ed 100644
--- a/src/feature/control/control.h
+++ b/src/feature/control/control.h
@@ -12,76 +12,6 @@
 #ifndef TOR_CONTROL_H
 #define TOR_CONTROL_H
 
-/** Used to indicate the type of a circuit event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum circuit_status_event_t {
-  CIRC_EVENT_LAUNCHED = 0,
-  CIRC_EVENT_BUILT    = 1,
-  CIRC_EVENT_EXTENDED = 2,
-  CIRC_EVENT_FAILED   = 3,
-  CIRC_EVENT_CLOSED   = 4,
-} circuit_status_event_t;
-
-/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
- * The various types are defined in control-spec.txt . */
-typedef enum circuit_status_minor_event_t {
-  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-  CIRC_MINOR_EVENT_CANNIBALIZED,
-} circuit_status_minor_event_t;
-
-/** Used to indicate the type of a stream event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum stream_status_event_t {
-  STREAM_EVENT_SENT_CONNECT = 0,
-  STREAM_EVENT_SENT_RESOLVE = 1,
-  STREAM_EVENT_SUCCEEDED    = 2,
-  STREAM_EVENT_FAILED       = 3,
-  STREAM_EVENT_CLOSED       = 4,
-  STREAM_EVENT_NEW          = 5,
-  STREAM_EVENT_NEW_RESOLVE  = 6,
-  STREAM_EVENT_FAILED_RETRIABLE = 7,
-  STREAM_EVENT_REMAP        = 8
-} stream_status_event_t;
-
-/** Used to indicate the type of an OR connection event passed to the
- * controller.  The various types are defined in control-spec.txt */
-typedef enum or_conn_status_event_t {
-  OR_CONN_EVENT_LAUNCHED     = 0,
-  OR_CONN_EVENT_CONNECTED    = 1,
-  OR_CONN_EVENT_FAILED       = 2,
-  OR_CONN_EVENT_CLOSED       = 3,
-  OR_CONN_EVENT_NEW          = 4,
-} or_conn_status_event_t;
-
-/** Used to indicate the type of a buildtime event */
-typedef enum buildtimeout_set_event_t {
-  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
-  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
-  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
-  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
-  BUILDTIMEOUT_SET_EVENT_RESUME = 4
-} buildtimeout_set_event_t;
-
-/** Enum describing various stages of bootstrapping, for use with controller
- * bootstrap status events. The values range from 0 to 100. */
-typedef enum {
-  BOOTSTRAP_STATUS_UNDEF=-1,
-  BOOTSTRAP_STATUS_STARTING=0,
-  BOOTSTRAP_STATUS_CONN_DIR=5,
-  BOOTSTRAP_STATUS_HANDSHAKE=-2,
-  BOOTSTRAP_STATUS_HANDSHAKE_DIR=10,
-  BOOTSTRAP_STATUS_ONEHOP_CREATE=15,
-  BOOTSTRAP_STATUS_REQUESTING_STATUS=20,
-  BOOTSTRAP_STATUS_LOADING_STATUS=25,
-  BOOTSTRAP_STATUS_LOADING_KEYS=40,
-  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
-  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
-  BOOTSTRAP_STATUS_CONN_OR=80,
-  BOOTSTRAP_STATUS_HANDSHAKE_OR=85,
-  BOOTSTRAP_STATUS_CIRCUIT_CREATE=90,
-  BOOTSTRAP_STATUS_DONE=100
-} bootstrap_status_t;
-
 control_connection_t *TO_CONTROL_CONN(connection_t *);
 
 #define CONTROL_CONN_STATE_MIN_ 1
@@ -92,18 +22,6 @@ control_connection_t *TO_CONTROL_CONN(connection_t *);
 #define CONTROL_CONN_STATE_NEEDAUTH 2
 #define CONTROL_CONN_STATE_MAX_ 2
 
-/** Reason for remapping an AP connection's address: we have a cached
- * answer. */
-#define REMAP_STREAM_SOURCE_CACHE 1
-/** Reason for remapping an AP connection's address: the exit node told us an
- * answer. */
-#define REMAP_STREAM_SOURCE_EXIT 2
-
-void control_initialize_event_queue(void);
-
-void control_update_global_event_mask(void);
-void control_adjust_event_log_severity(void);
-
 void control_ports_write_to_file(void);
 
 /** Log information about the connection <b>conn</b>, protecting it as with
@@ -124,294 +42,28 @@ void connection_control_closed(control_connection_t *conn);
 
 int connection_control_process_inbuf(control_connection_t *conn);
 
-#define EVENT_NS 0x000F
-int control_event_is_interesting(int event);
-
-void control_per_second_events(void);
-int control_any_per_second_event_enabled(void);
-
-int control_event_circuit_status(origin_circuit_t *circ,
-                                 circuit_status_event_t e, int reason);
-int control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                          int old_purpose);
-int control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                       int old_purpose,
-                                       const struct timeval *old_tv_created);
-int control_event_stream_status(entry_connection_t *conn,
-                                stream_status_event_t e,
-                                int reason);
-int control_event_or_conn_status(or_connection_t *conn,
-                                 or_conn_status_event_t e, int reason);
-int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
-int control_event_stream_bandwidth(edge_connection_t *edge_conn);
-int control_event_stream_bandwidth_used(void);
-int control_event_circ_bandwidth_used(void);
-int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
-int control_event_conn_bandwidth(connection_t *conn);
-int control_event_conn_bandwidth_used(void);
-int control_event_circuit_cell_stats(void);
-void control_event_logmsg(int severity, uint32_t domain, const char *msg);
-void control_event_logmsg_pending(void);
-int control_event_descriptors_changed(smartlist_t *routers);
-int control_event_address_mapped(const char *from, const char *to,
-                                 time_t expires, const char *error,
-                                 const int cached);
-int control_event_my_descriptor_changed(void);
-int control_event_network_liveness_update(int liveness);
-int control_event_networkstatus_changed(smartlist_t *statuses);
-
-int control_event_newconsensus(const networkstatus_t *consensus);
-int control_event_networkstatus_changed_single(const routerstatus_t *rs);
-int control_event_general_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_client_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_server_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-
-int control_event_general_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_client_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_server_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-
-int control_event_guard(const char *nickname, const char *digest,
-                        const char *status);
-int control_event_conf_changed(const smartlist_t *elements);
-int control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                                   const char *args);
-int control_event_signal(uintptr_t signal);
-
-int init_control_cookie_authentication(int enabled);
-char *get_controller_cookie_file_name(void);
-struct config_line_t;
-smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
 void disable_control_logging(void);
 void enable_control_logging(void);
 
 void monitor_owning_controller_process(const char *process_spec);
 
-void control_event_bootstrap(bootstrap_status_t status, int progress);
-MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
-                                                 int reason,
-                                                 or_connection_t *or_conn));
-void control_event_boot_dir(bootstrap_status_t status, int progress);
-void control_event_boot_first_orconn(void);
-void control_event_bootstrap_problem(const char *warn, const char *reason,
-                                     const connection_t *conn, int dowarn);
-
-void control_event_clients_seen(const char *controller_str);
-void control_event_transport_launched(const char *mode,
-                                      const char *transport_name,
-                                      tor_addr_t *addr, uint16_t port);
 const char *rend_auth_type_to_string(rend_auth_type_t auth_type);
-MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
-void control_event_hs_descriptor_requested(const char *onion_address,
-                                           rend_auth_type_t auth_type,
-                                           const char *id_digest,
-                                           const char *desc_id,
-                                           const char *hsdir_index);
-void control_event_hs_descriptor_created(const char *onion_address,
-                                         const char *desc_id,
-                                         int replica);
-void control_event_hs_descriptor_upload(const char *onion_address,
-                                        const char *desc_id,
-                                        const char *hs_dir,
-                                        const char *hsdir_index);
-void control_event_hs_descriptor_upload_end(const char *action,
-                                            const char *onion_address,
-                                            const char *hs_dir,
-                                            const char *reason);
-void control_event_hs_descriptor_uploaded(const char *hs_dir,
-                                          const char *onion_address);
-/* Hidden service v2 HS_DESC specific. */
-void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                          const char *id_digest,
-                                          const char *reason);
-void control_event_hsv2_descriptor_received(const char *onion_address,
-                                            const rend_data_t *rend_data,
-                                            const char *id_digest);
-/* Hidden service v3 HS_DESC specific. */
-void control_event_hsv3_descriptor_failed(const char *onion_address,
-                                          const char *desc_id,
-                                          const char *hsdir_id_digest,
-                                          const char *reason);
-void control_event_hsv3_descriptor_received(const char *onion_address,
-                                            const char *desc_id,
-                                            const char *hsdir_id_digest);
-void control_event_hs_descriptor_upload_failed(const char *hs_dir,
-                                               const char *onion_address,
-                                               const char *reason);
-void control_event_hs_descriptor_content(const char *onion_address,
-                                         const char *desc_id,
-                                         const char *hsdir_fp,
-                                         const char *content);
 void control_free_all(void);
 
-#ifdef CONTROL_PRIVATE
-#include "lib/crypt_ops/crypto_ed25519.h"
-
-/* Recognized asynchronous event types.  It's okay to expand this list
- * because it is used both as a list of v0 event types, and as indices
- * into the bitfield to determine which controllers want which events.
- */
-/* This bitfield has no event zero    0x0000 */
-#define EVENT_MIN_                    0x0001
-#define EVENT_CIRCUIT_STATUS          0x0001
-#define EVENT_STREAM_STATUS           0x0002
-#define EVENT_OR_CONN_STATUS          0x0003
-#define EVENT_BANDWIDTH_USED          0x0004
-#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
-#define EVENT_NEW_DESC                0x0006
-#define EVENT_DEBUG_MSG               0x0007
-#define EVENT_INFO_MSG                0x0008
-#define EVENT_NOTICE_MSG              0x0009
-#define EVENT_WARN_MSG                0x000A
-#define EVENT_ERR_MSG                 0x000B
-#define EVENT_ADDRMAP                 0x000C
-/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
-   can reclaim 0x000D. */
-#define EVENT_DESCCHANGED             0x000E
-/* Exposed above */
-// #define EVENT_NS                   0x000F
-#define EVENT_STATUS_CLIENT           0x0010
-#define EVENT_STATUS_SERVER           0x0011
-#define EVENT_STATUS_GENERAL          0x0012
-#define EVENT_GUARD                   0x0013
-#define EVENT_STREAM_BANDWIDTH_USED   0x0014
-#define EVENT_CLIENTS_SEEN            0x0015
-#define EVENT_NEWCONSENSUS            0x0016
-#define EVENT_BUILDTIMEOUT_SET        0x0017
-#define EVENT_GOT_SIGNAL              0x0018
-#define EVENT_CONF_CHANGED            0x0019
-#define EVENT_CONN_BW                 0x001A
-#define EVENT_CELL_STATS              0x001B
-/* UNUSED :                           0x001C */
-#define EVENT_CIRC_BANDWIDTH_USED     0x001D
-#define EVENT_TRANSPORT_LAUNCHED      0x0020
-#define EVENT_HS_DESC                 0x0021
-#define EVENT_HS_DESC_CONTENT         0x0022
-#define EVENT_NETWORK_LIVENESS        0x0023
-#define EVENT_MAX_                    0x0023
+#ifdef CONTROL_MODULE_PRIVATE
+struct signal_name_t {
+  int sig;
+  const char *signal_name;
+};
+extern const struct signal_name_t signal_table[];
+int get_cached_network_liveness(void);
+void set_cached_network_liveness(int liveness);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
 
-/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
-#define EVENT_CAPACITY_               0x0040
-
-/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
- * different structure, as it can only handle a maximum left shift of 1<<63. */
-
-#if EVENT_MAX_ >= EVENT_CAPACITY_
-#error control_connection_t.event_mask has an event greater than its capacity
+#ifdef CONTROL_PRIVATE
+STATIC char *control_split_incoming_command(char *incoming_cmd,
+                                            size_t *data_len,
+                                            char **current_cmd_out);
 #endif
 
-#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
-
-#define EVENT_MASK_NONE_             ((uint64_t)0x0)
-
-#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
-#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
-                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
-                                          - EVENT_MIN_))
-
-#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
-                                      & EVENT_MASK_BELOW_MAX_)
-
-/* Used only by control.c and test.c */
-STATIC size_t write_escaped_data(const char *data, size_t len, char **out);
-STATIC size_t read_escaped_data(const char *data, size_t len, char **out);
-
-#ifdef TOR_UNIT_TESTS
-MOCK_DECL(STATIC void,
-          send_control_event_string,(uint16_t event, const char *msg));
-
-MOCK_DECL(STATIC void,
-          queue_control_event_string,(uint16_t event, char *msg));
-
-void control_testing_set_global_event_mask(uint64_t mask);
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Helper structure: temporarily stores cell statistics for a circuit. */
-typedef struct cell_stats_t {
-  /** Number of cells added in app-ward direction by command. */
-  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells added in exit-ward direction by command. */
-  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in app-ward direction by command. */
-  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in exit-ward direction by command. */
-  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in app-ward direction by command. */
-  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in exit-ward direction by command. */
-  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
-} cell_stats_t;
-void sum_up_cell_stats_by_command(circuit_t *circ,
-                                  cell_stats_t *cell_stats);
-void append_cell_stats_by_command(smartlist_t *event_parts,
-                                  const char *key,
-                                  const uint64_t *include_if_non_zero,
-                                  const uint64_t *number_to_include);
-void format_cell_stats(char **event_string, circuit_t *circ,
-                       cell_stats_t *cell_stats);
-STATIC char *get_bw_samples(void);
-
-/* ADD_ONION secret key to create an ephemeral service. The command supports
- * multiple versions so this union stores the key and passes it to the HS
- * subsystem depending on the requested version. */
-typedef union add_onion_secret_key_t {
-  /* Hidden service v2 secret key. */
-  crypto_pk_t *v2;
-  /* Hidden service v3 secret key. */
-  ed25519_secret_key_t *v3;
-} add_onion_secret_key_t;
-
-STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
-                                   const char **key_new_alg_out,
-                                   char **key_new_blob_out,
-                                   add_onion_secret_key_t *decoded_key,
-                                   int *hs_version, char **err_msg_out);
-
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
-
-STATIC int getinfo_helper_onions(
-    control_connection_t *control_conn,
-    const char *question,
-    char **answer,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_networkstatus(
-    const char *flavor,
-    download_status_t **dl_to_emit,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_cert(
-    const char *fp_sk_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_desc(
-    const char *desc_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_bridge(
-    const char *bridge_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC int getinfo_helper_downloads(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_dir(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_current_time(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-
-#endif /* defined(CONTROL_PRIVATE) */
-
 #endif /* !defined(TOR_CONTROL_H) */
diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c
new file mode 100644
index 0000000000..a574d07b33
--- /dev/null
+++ b/src/feature/control/control_auth.c
@@ -0,0 +1,441 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.c
+ * \brief Authentication for Tor's control-socket interface.
+ **/
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_proto.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+
+#include "lib/crypt_ops/crypto_s2k.h"
+
+/** If we're using cookie-type authentication, how long should our cookies be?
+ */
+#define AUTHENTICATION_COOKIE_LEN 32
+
+/** If true, we've set authentication_cookie to a secret code and
+ * stored it to disk. */
+static int authentication_cookie_is_set = 0;
+/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
+ * and which we're using to authenticate controllers.  (If the controller can
+ * read it off disk, it has permission to connect.) */
+static uint8_t *authentication_cookie = NULL;
+
+#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
+  "Tor safe cookie authentication server-to-controller hash"
+#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
+  "Tor safe cookie authentication controller-to-server hash"
+#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
+
+/** Helper: Return a newly allocated string containing a path to the
+ * file where we store our authentication cookie. */
+char *
+get_controller_cookie_file_name(void)
+{
+  const or_options_t *options = get_options();
+  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
+    return tor_strdup(options->CookieAuthFile);
+  } else {
+    return get_datadir_fname("control_auth_cookie");
+  }
+}
+
+/* Initialize the cookie-based authentication system of the
+ * ControlPort. If <b>enabled</b> is 0, then disable the cookie
+ * authentication system.  */
+int
+init_control_cookie_authentication(int enabled)
+{
+  char *fname = NULL;
+  int retval;
+
+  if (!enabled) {
+    authentication_cookie_is_set = 0;
+    return 0;
+  }
+
+  fname = get_controller_cookie_file_name();
+  retval = init_cookie_authentication(fname, "", /* no header */
+                                      AUTHENTICATION_COOKIE_LEN,
+                                   get_options()->CookieAuthFileGroupReadable,
+                                      &authentication_cookie,
+                                      &authentication_cookie_is_set);
+  tor_free(fname);
+  return retval;
+}
+
+/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
+ * Return a smartlist of acceptable passwords (unterminated strings of
+ * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
+ * failure.
+ */
+smartlist_t *
+decode_hashed_passwords(config_line_t *passwords)
+{
+  char decoded[64];
+  config_line_t *cl;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(passwords);
+
+  for (cl = passwords; cl; cl = cl->next) {
+    const char *hashed = cl->value;
+
+    if (!strcmpstart(hashed, "16:")) {
+      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
+                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
+          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
+        goto err;
+      }
+    } else {
+        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
+            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
+          goto err;
+        }
+    }
+    smartlist_add(sl,
+                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
+  }
+
+  return sl;
+
+ err:
+  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
+  smartlist_free(sl);
+  return NULL;
+}
+
+const control_cmd_syntax_t authchallenge_syntax = {
+   .min_args = 1,
+   .max_args = 1,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHCHALLENGE command. */
+int
+handle_control_authchallenge(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  char *client_nonce;
+  size_t client_nonce_len;
+  char server_hash[DIGEST256_LEN];
+  char server_hash_encoded[HEX_DIGEST256_LEN+1];
+  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
+  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
+
+  if (strcasecmp(smartlist_get(args->args, 0), "SAFECOOKIE")) {
+    control_write_endreply(conn, 513,
+                           "AUTHCHALLENGE only supports SAFECOOKIE "
+                           "authentication");
+    goto fail;
+  }
+  if (!authentication_cookie_is_set) {
+    control_write_endreply(conn, 515, "Cookie authentication is disabled");
+    goto fail;
+  }
+  if (args->kwargs == NULL || args->kwargs->next != NULL) {
+    control_write_endreply(conn, 512,
+                           "Wrong number of arguments for AUTHCHALLENGE");
+    goto fail;
+  }
+  if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHCHALLENGE does not accept keyword "
+                           "arguments.");
+    goto fail;
+  }
+
+  bool contains_quote = strchr(args->raw_body, '\"');
+  if (contains_quote) {
+    /* The nonce was quoted */
+    client_nonce = tor_strdup(args->kwargs->value);
+    client_nonce_len = strlen(client_nonce);
+  } else {
+    /* The nonce was should be in hex. */
+    const char *hex_nonce = args->kwargs->value;
+    client_nonce_len = strlen(hex_nonce) / 2;
+    client_nonce = tor_malloc(client_nonce_len);
+    if (base16_decode(client_nonce, client_nonce_len, hex_nonce,
+                      strlen(hex_nonce)) != (int)client_nonce_len) {
+      control_write_endreply(conn, 513, "Invalid base16 client nonce");
+      tor_free(client_nonce);
+      goto fail;
+    }
+  }
+
+  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+  /* Now compute and send the server-to-controller response, and the
+   * server's nonce. */
+  tor_assert(authentication_cookie != NULL);
+
+  {
+    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
+                      client_nonce_len +
+                      SAFECOOKIE_SERVER_NONCE_LEN);
+    char *tmp = tor_malloc_zero(tmp_len);
+    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
+    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
+           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+    crypto_hmac_sha256(server_hash,
+                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
+                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    crypto_hmac_sha256(client_hash,
+                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
+                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    conn->safecookie_client_hash = client_hash;
+
+    tor_free(tmp);
+  }
+
+  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
+                server_hash, sizeof(server_hash));
+  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
+                server_nonce, sizeof(server_nonce));
+
+  control_printf_endreply(conn, 250,
+                          "AUTHCHALLENGE SERVERHASH=%s SERVERNONCE=%s",
+                          server_hash_encoded,
+                          server_nonce_encoded);
+
+  tor_free(client_nonce);
+  return 0;
+ fail:
+  connection_mark_for_close(TO_CONN(conn));
+  return -1;
+}
+
+const control_cmd_syntax_t authenticate_syntax = {
+   .max_args = 0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHENTICATE message.  Check whether the
+ * authentication is valid, and if so, update the connection's state to
+ * OPEN.  Reply with DONE or ERROR.
+ */
+int
+handle_control_authenticate(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  bool used_quoted_string = false;
+  const or_options_t *options = get_options();
+  const char *errstr = "Unknown error";
+  char *password;
+  size_t password_len;
+  int bad_cookie=0, bad_password=0;
+  smartlist_t *sl = NULL;
+
+  if (args->kwargs == NULL) {
+    password = tor_strdup("");
+    password_len = 0;
+  } else if (args->kwargs->next) {
+    control_write_endreply(conn, 512, "Too many arguments to AUTHENTICATE.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHENTICATE does not accept keyword arguments.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strchr(args->raw_body, '\"')) {
+    used_quoted_string = true;
+    password = tor_strdup(args->kwargs->value);
+    password_len = strlen(password);
+  } else {
+    const char *hex_passwd = args->kwargs->value;
+    password_len = strlen(hex_passwd) / 2;
+    password = tor_malloc(password_len+1);
+    if (base16_decode(password, password_len+1, hex_passwd, strlen(hex_passwd))
+                      != (int) password_len) {
+      control_write_endreply(conn, 551,
+            "Invalid hexadecimal encoding.  Maybe you tried a plain text "
+            "password?  If so, the standard requires that you put it in "
+            "double quotes.");
+      connection_mark_for_close(TO_CONN(conn));
+      tor_free(password);
+      return 0;
+    }
+  }
+
+  if (conn->safecookie_client_hash != NULL) {
+    /* The controller has chosen safe cookie authentication; the only
+     * acceptable authentication value is the controller-to-server
+     * response. */
+
+    tor_assert(authentication_cookie_is_set);
+
+    if (password_len != DIGEST256_LEN) {
+      log_warn(LD_CONTROL,
+               "Got safe cookie authentication response with wrong length "
+               "(%d)", (int)password_len);
+      errstr = "Wrong length for safe cookie response.";
+      goto err;
+    }
+
+    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
+      log_warn(LD_CONTROL,
+               "Got incorrect safe cookie authentication response");
+      errstr = "Safe cookie response did not match expected value.";
+      goto err;
+    }
+
+    tor_free(conn->safecookie_client_hash);
+    goto ok;
+  }
+
+  if (!options->CookieAuthentication && !options->HashedControlPassword &&
+      !options->HashedControlSessionPassword) {
+    /* if Tor doesn't demand any stronger authentication, then
+     * the controller can get in with anything. */
+    goto ok;
+  }
+
+  if (options->CookieAuthentication) {
+    int also_password = options->HashedControlPassword != NULL ||
+      options->HashedControlSessionPassword != NULL;
+    if (password_len != AUTHENTICATION_COOKIE_LEN) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
+                 "(%d)", (int)password_len);
+        errstr = "Wrong length on authentication cookie.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else if (tor_memneq(authentication_cookie, password, password_len)) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
+        errstr = "Authentication cookie did not match expected value.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else {
+      goto ok;
+    }
+  }
+
+  if (options->HashedControlPassword ||
+      options->HashedControlSessionPassword) {
+    int bad = 0;
+    smartlist_t *sl_tmp;
+    char received[DIGEST_LEN];
+    int also_cookie = options->CookieAuthentication;
+    sl = smartlist_new();
+    if (options->HashedControlPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (options->HashedControlSessionPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (bad) {
+      if (!also_cookie) {
+        log_warn(LD_BUG,
+                 "Couldn't decode HashedControlPassword: invalid base16");
+        errstr="Couldn't decode HashedControlPassword value in configuration.";
+        goto err;
+      }
+      bad_password = 1;
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+    } else {
+      SMARTLIST_FOREACH(sl, char *, expected,
+      {
+        secret_to_key_rfc2440(received,DIGEST_LEN,
+                              password,password_len,expected);
+        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
+                      received, DIGEST_LEN))
+          goto ok;
+      });
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+
+      if (used_quoted_string)
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration";
+      else
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration. Maybe you tried a plain text password? "
+          "If so, the standard requires that you put it in double quotes.";
+      bad_password = 1;
+      if (!also_cookie)
+        goto err;
+    }
+  }
+
+  /** We only get here if both kinds of authentication failed. */
+  tor_assert(bad_password && bad_cookie);
+  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
+  errstr = "Password did not match HashedControlPassword *or* authentication "
+    "cookie.";
+
+ err:
+  tor_free(password);
+  control_printf_endreply(conn, 515, "Authentication failed: %s", errstr);
+  connection_mark_for_close(TO_CONN(conn));
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+ ok:
+  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
+           ")", conn->base_.s);
+  send_control_done(conn);
+  conn->base_.state = CONTROL_CONN_STATE_OPEN;
+  tor_free(password);
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+}
+
+void
+control_auth_free_all(void)
+{
+  if (authentication_cookie) /* Free the auth cookie */
+    tor_free(authentication_cookie);
+  authentication_cookie_is_set = 0;
+}
diff --git a/src/feature/control/control_auth.h b/src/feature/control/control_auth.h
new file mode 100644
index 0000000000..246e18ccbc
--- /dev/null
+++ b/src/feature/control/control_auth.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.h
+ * \brief Header file for control_auth.c.
+ **/
+
+#ifndef TOR_CONTROL_AUTH_H
+#define TOR_CONTROL_AUTH_H
+
+struct control_cmd_args_t;
+struct control_cmd_syntax_t;
+
+int init_control_cookie_authentication(int enabled);
+char *get_controller_cookie_file_name(void);
+struct config_line_t;
+smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
+
+int handle_control_authchallenge(control_connection_t *conn,
+                                 const struct control_cmd_args_t *args);
+int handle_control_authenticate(control_connection_t *conn,
+                                const struct control_cmd_args_t *args);
+void control_auth_free_all(void);
+
+extern const struct control_cmd_syntax_t authchallenge_syntax;
+extern const struct control_cmd_syntax_t authenticate_syntax;
+
+#endif /* !defined(TOR_CONTROL_AUTH_H) */
diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c
new file mode 100644
index 0000000000..098e24682e
--- /dev/null
+++ b/src/feature/control/control_bootstrap.c
@@ -0,0 +1,383 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_bootstrap.c
+ * \brief Provide bootstrap progress events for the control port.
+ */
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "core/or/connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/reasons.h"
+#include "feature/control/control_events.h"
+#include "feature/hibernate/hibernate.h"
+#include "lib/malloc/malloc.h"
+
+/** A sufficiently large size to record the last bootstrap phase string. */
+#define BOOTSTRAP_MSG_LEN 1024
+
+/** What was the last bootstrap phase message we sent? We keep track
+ * of this so we can respond to getinfo status/bootstrap-phase queries. */
+static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
+
+/** Table to convert bootstrap statuses to strings.  */
+static const struct {
+  bootstrap_status_t status;
+  const char *tag;
+  const char *summary;
+} boot_to_str_tab[] = {
+  { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" },
+  { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" },
+
+  /* Initial connection to any relay */
+
+  { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt",
+    "Connected to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy",
+    "Connected to proxy" },
+  { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" },
+  { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE, "handshake",
+    "Handshaking with a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done",
+    "Handshake with a relay done" },
+
+  /* Loading directory info */
+
+  { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create",
+    "Establishing an encrypted directory connection" },
+  { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status",
+    "Asking for networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status",
+    "Loading networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys",
+    "Loading authority key certs" },
+  { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors",
+    "Asking for relay descriptors" },
+  { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors",
+    "Loading relay descriptors" },
+  { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo",
+    "Loaded enough directory info to build circuits" },
+
+  /* Connecting to a relay for AP circuits */
+
+  { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt",
+    "Connecting to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt",
+    "Connected to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy",
+    "Connecting to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy",
+    "Connected to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN, "ap_conn",
+    "Connecting to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done",
+    "Connected to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake",
+    "Finishing handshake with a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done",
+    "Handshake finished with a relay to build circuits" },
+
+  /* Creating AP circuits */
+
+  { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create",
+    "Establishing a Tor circuit" },
+  { BOOTSTRAP_STATUS_DONE, "done", "Done" },
+};
+#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0]))
+
+/** Convert the name of a bootstrapping phase <b>s</b> into strings
+ * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
+static int
+bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
+                           const char **summary)
+{
+  for (size_t i = 0; i < N_BOOT_TO_STR; i++) {
+    if (s == boot_to_str_tab[i].status) {
+      *tag = boot_to_str_tab[i].tag;
+      *summary = boot_to_str_tab[i].summary;
+      return 0;
+    }
+  }
+
+  *tag = *summary = "unknown";
+  return -1;
+}
+
+/** What percentage through the bootstrap process are we? We remember
+ * this so we can avoid sending redundant bootstrap status events, and
+ * so we can guess context for the bootstrap messages which are
+ * ambiguous. It starts at 'undef', but gets set to 'starting' while
+ * Tor initializes. */
+static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_percent, but only takes on the enumerated values in
+ * bootstrap_status_t.
+ */
+static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** As bootstrap_percent, but holds the bootstrapping level at which we last
+ * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
+ * to avoid flooding the log with a new message every time we get a few more
+ * microdescriptors */
+static int notice_bootstrap_percent = 0;
+
+/** How many problems have we had getting to the next bootstrapping phase?
+ * These include failure to establish a connection to a Tor relay,
+ * failures to finish the TLS handshake, failures to validate the
+ * consensus document, etc. */
+static int bootstrap_problems = 0;
+
+/** We only tell the controller once we've hit a threshold of problems
+ * for the current phase. */
+#define BOOTSTRAP_PROBLEM_THRESHOLD 10
+
+/** When our bootstrapping progress level changes, but our bootstrapping
+ * status has not advanced, we only log at NOTICE when we have made at least
+ * this much progress.
+ */
+#define BOOTSTRAP_PCT_INCREMENT 5
+
+/** Do the actual logging and notifications for
+ * control_event_bootstrap().  Doesn't change any state beyond that.
+ */
+static void
+control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
+                             int progress)
+{
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *tag, *summary;
+
+  bootstrap_status_to_string(status, &tag, &summary);
+  /* Locally reset status if there's incremental progress */
+  if (progress)
+    status = progress;
+
+  tor_log(loglevel, LD_CONTROL,
+          "Bootstrapped %d%% (%s): %s", status, tag, summary);
+  tor_snprintf(buf, sizeof(buf),
+               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
+               status, tag, summary);
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "NOTICE %s", buf);
+  control_event_client_status(LOG_NOTICE, "%s", buf);
+}
+
+/** Called when Tor has made progress at bootstrapping its directory
+ * information and initial circuits.
+ *
+ * <b>status</b> is the new status, that is, what task we will be doing
+ * next. <b>progress</b> is zero if we just started this task, else it
+ * represents progress on the task.
+ */
+void
+control_event_bootstrap(bootstrap_status_t status, int progress)
+{
+  int loglevel = LOG_NOTICE;
+
+  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  if (status <= bootstrap_percent) {
+    /* If there's no new progress, return early. */
+    if (!progress || progress <= bootstrap_percent)
+      return;
+    /* Log at INFO if not enough progress happened. */
+    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
+      loglevel = LOG_INFO;
+  }
+
+  control_event_bootstrap_core(loglevel, status, progress);
+
+  if (status > bootstrap_percent) {
+    bootstrap_phase = status; /* new milestone reached */
+    bootstrap_percent = status;
+  }
+  if (progress > bootstrap_percent) {
+    /* incremental progress within a milestone */
+    bootstrap_percent = progress;
+    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
+  }
+  if (loglevel == LOG_NOTICE &&
+      bootstrap_percent > notice_bootstrap_percent) {
+    /* Remember that we gave a notice at this level. */
+    notice_bootstrap_percent = bootstrap_percent;
+  }
+}
+
+/** Flag whether we've opened an OR_CONN yet  */
+static int bootstrap_first_orconn = 0;
+
+/** Like bootstrap_phase, but for (possibly deferred) directory progress */
+static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
+static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+
+/** Defer directory info bootstrap events until we have successfully
+ * completed our first connection to a router.  */
+void
+control_event_boot_dir(bootstrap_status_t status, int progress)
+{
+  if (status > bootstrap_dir_progress) {
+    bootstrap_dir_progress = status;
+    bootstrap_dir_phase = status;
+  }
+  if (progress && progress >= bootstrap_dir_progress) {
+    bootstrap_dir_progress = progress;
+  }
+
+  /* Don't report unless we have successfully opened at least one OR_CONN */
+  if (!bootstrap_first_orconn)
+    return;
+
+  control_event_bootstrap(status, progress);
+}
+
+/** Set a flag to allow reporting of directory bootstrap progress.
+ * (Code that reports completion of an OR_CONN calls this.)  Also,
+ * report directory progress so far. */
+void
+control_event_boot_first_orconn(void)
+{
+  bootstrap_first_orconn = 1;
+  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a human-readable hint
+ * as to why, and <b>reason</b> provides a controller-facing short
+ * tag.  <b>conn</b> is the connection that caused this problem and
+ * can be NULL if a connection cannot be easily identified.
+ */
+void
+control_event_bootstrap_problem(const char *warn, const char *reason,
+                                const connection_t *conn, int dowarn)
+{
+  int status = bootstrap_percent;
+  const char *tag = "", *summary = "";
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *recommendation = "ignore";
+  int severity;
+  char *or_id = NULL, *hostaddr = NULL;
+  or_connection_t *or_conn = NULL;
+
+  /* bootstrap_percent must not be in "undefined" state here. */
+  tor_assert(status >= 0);
+
+  if (bootstrap_percent == 100)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  bootstrap_problems++;
+
+  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
+    dowarn = 1;
+
+  /* Don't warn about our bootstrapping status if we are hibernating or
+   * shutting down. */
+  if (we_are_hibernating())
+    dowarn = 0;
+
+  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
+
+  severity = dowarn ? LOG_WARN : LOG_INFO;
+
+  if (dowarn)
+    recommendation = "warn";
+
+  if (conn && conn->type == CONN_TYPE_OR) {
+    /* XXX TO_OR_CONN can't deal with const */
+    or_conn = TO_OR_CONN((connection_t *)conn);
+    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
+  } else {
+    or_id = tor_strdup("?");
+  }
+
+  if (conn)
+    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
+  else
+    hostaddr = tor_strdup("?");
+
+  log_fn(severity,
+         LD_CONTROL, "Problem bootstrapping. Stuck at %d%% (%s): %s. (%s; %s; "
+         "count %d; recommendation %s; host %s at %s)",
+         status, tag, summary, warn, reason,
+         bootstrap_problems, recommendation,
+         or_id, hostaddr);
+
+  connection_or_report_broken_states(severity, LD_HANDSHAKE);
+
+  tor_snprintf(buf, sizeof(buf),
+      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
+      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
+      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
+      recommendation,
+      or_id, hostaddr);
+
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "WARN %s", buf);
+  control_event_client_status(LOG_WARN, "%s", buf);
+
+  tor_free(hostaddr);
+  tor_free(or_id);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a hint as to why, and
+ * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
+ * is the connection that caused this problem.
+ */
+MOCK_IMPL(void,
+control_event_bootstrap_prob_or, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
+{
+  int dowarn = 0;
+
+  if (or_conn->have_noted_bootstrap_problem)
+    return;
+
+  or_conn->have_noted_bootstrap_problem = 1;
+
+  if (reason == END_OR_CONN_REASON_NO_ROUTE)
+    dowarn = 1;
+
+  /* If we are using bridges and all our OR connections are now
+     closed, it means that we totally failed to connect to our
+     bridges. Throw a warning. */
+  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
+    dowarn = 1;
+
+  control_event_bootstrap_problem(warn,
+                                  orconn_end_reason_to_control_string(reason),
+                                  TO_CONN(or_conn), dowarn);
+}
+
+/** Return a copy of the last sent bootstrap message. */
+char *
+control_event_boot_last_msg(void)
+{
+  return tor_strdup(last_sent_bootstrap_message);
+}
+
+/** Reset bootstrap tracking state. */
+void
+control_event_bootstrap_reset(void)
+{
+  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+  notice_bootstrap_percent = 0;
+  bootstrap_problems = 0;
+  bootstrap_first_orconn = 0;
+  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
+}
diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c
new file mode 100644
index 0000000000..68d05abb73
--- /dev/null
+++ b/src/feature/control/control_cmd.c
@@ -0,0 +1,2399 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.c
+ * \brief Implement various commands for Tor's control-socket interface.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/confmgt/confparse.h"
+#include "app/main/main.h"
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/dnsserv.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/hs/hs_control.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "feature/rend/rendservice.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/rend/rend_authorized_client_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+
+static int control_setconf_helper(control_connection_t *conn,
+                                  const control_cmd_args_t *args,
+                                  int use_defaults);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/**
+ * Release all storage held in <b>args</b>
+ **/
+void
+control_cmd_args_free_(control_cmd_args_t *args)
+{
+  if (! args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, tor_free(c));
+    smartlist_free(args->args);
+  }
+  config_free_lines(args->kwargs);
+  tor_free(args->cmddata);
+
+  tor_free(args);
+}
+
+/** Erase all memory held in <b>args</b>. */
+void
+control_cmd_args_wipe(control_cmd_args_t *args)
+{
+  if (!args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, memwipe(c, 0, strlen(c)));
+  }
+  for (config_line_t *line = args->kwargs; line; line = line->next) {
+    memwipe(line->key, 0, strlen(line->key));
+    memwipe(line->value, 0, strlen(line->value));
+  }
+  if (args->cmddata)
+    memwipe(args->cmddata, 0, args->cmddata_len);
+}
+
+/**
+ * Return true iff any element of the NULL-terminated <b>array</b> matches
+ * <b>kwd</b>. Case-insensitive.
+ **/
+static bool
+string_array_contains_keyword(const char **array, const char *kwd)
+{
+  for (unsigned i = 0; array[i]; ++i) {
+    if (! strcasecmp(array[i], kwd))
+      return true;
+  }
+  return false;
+}
+
+/** Helper for argument parsing: check whether the keyword arguments just
+ * parsed in <b>result</b> were well-formed according to <b>syntax</b>.
+ *
+ * On success, return 0.  On failure, return -1 and set *<b>error_out</b>
+ * to a newly allocated error string.
+ **/
+static int
+kvline_check_keyword_args(const control_cmd_args_t *result,
+                          const control_cmd_syntax_t *syntax,
+                          char **error_out)
+{
+  if (result->kwargs == NULL) {
+    tor_asprintf(error_out, "Cannot parse keyword argument(s)");
+    return -1;
+  }
+
+  if (! syntax->allowed_keywords) {
+    /* All keywords are permitted. */
+    return 0;
+  }
+
+  /* Check for unpermitted arguments */
+  const config_line_t *line;
+  for (line = result->kwargs; line; line = line->next) {
+    if (! string_array_contains_keyword(syntax->allowed_keywords,
+                                        line->key)) {
+      tor_asprintf(error_out, "Unrecognized keyword argument %s",
+                   escaped(line->key));
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Helper: parse the arguments to a command according to <b>syntax</b>.  On
+ * success, set *<b>error_out</b> to NULL and return a newly allocated
+ * control_cmd_args_t.  On failure, set *<b>error_out</b> to newly allocated
+ * error string, and return NULL.
+ **/
+STATIC control_cmd_args_t *
+control_cmd_parse_args(const char *command,
+                       const control_cmd_syntax_t *syntax,
+                       size_t body_len,
+                       const char *body,
+                       char **error_out)
+{
+  *error_out = NULL;
+  control_cmd_args_t *result = tor_malloc_zero(sizeof(control_cmd_args_t));
+  const char *cmdline;
+  char *cmdline_alloc = NULL;
+  tor_assert(syntax->max_args < INT_MAX || syntax->max_args == UINT_MAX);
+
+  result->command = command;
+
+  if (syntax->store_raw_body) {
+    tor_assert(body[body_len] == 0);
+    result->raw_body = body;
+  }
+
+  const char *eol = memchr(body, '\n', body_len);
+  if (syntax->want_cmddata) {
+    if (! eol || (eol+1) == body+body_len) {
+      *error_out = tor_strdup("Empty body");
+      goto err;
+    }
+    cmdline_alloc = tor_memdup_nulterm(body, eol-body);
+    cmdline = cmdline_alloc;
+    ++eol;
+    result->cmddata_len = read_escaped_data(eol, (body+body_len)-eol,
+                                           &result->cmddata);
+  } else {
+    if (eol && (eol+1) != body+body_len) {
+      *error_out = tor_strdup("Unexpected body");
+      goto err;
+    }
+    cmdline = body;
+  }
+
+  result->args = smartlist_new();
+  smartlist_split_string(result->args, cmdline, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK,
+                         (int)(syntax->max_args+1));
+  size_t n_args = smartlist_len(result->args);
+  if (n_args < syntax->min_args) {
+    tor_asprintf(error_out, "Need at least %u argument(s)",
+                 syntax->min_args);
+    goto err;
+  } else if (n_args > syntax->max_args && ! syntax->accept_keywords) {
+    tor_asprintf(error_out, "Cannot accept more than %u argument(s)",
+                 syntax->max_args);
+    goto err;
+  }
+
+  if (n_args > syntax->max_args) {
+    /* We have extra arguments after the positional arguments, and we didn't
+       treat them as an error, so they must count as keyword arguments: Either
+       K=V pairs, or flags, or both. */
+    tor_assert(n_args == syntax->max_args + 1);
+    tor_assert(syntax->accept_keywords);
+    char *remainder = smartlist_pop_last(result->args);
+    result->kwargs = kvline_parse(remainder, syntax->kvline_flags);
+    tor_free(remainder);
+    if (kvline_check_keyword_args(result, syntax, error_out) < 0) {
+      goto err;
+    }
+  }
+
+  tor_assert_nonfatal(*error_out == NULL);
+  goto done;
+ err:
+  tor_assert_nonfatal(*error_out != NULL);
+  control_cmd_args_free(result);
+ done:
+  tor_free(cmdline_alloc);
+  return result;
+}
+
+/**
+ * Return true iff <b>lines</b> contains <b>flags</b> as a no-value
+ * (keyword-only) entry.
+ **/
+static bool
+config_lines_contain_flag(const config_line_t *lines, const char *flag)
+{
+  const config_line_t *line = config_line_find_case(lines, flag);
+  return line && !strcmp(line->value, "");
+}
+
+static const control_cmd_syntax_t setconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a SETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body.*/
+static int
+handle_control_setconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 0);
+}
+
+static const control_cmd_syntax_t resetconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a RESETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body. */
+static int
+handle_control_resetconf(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 1);
+}
+
+static const control_cmd_syntax_t getconf_syntax = {
+  .max_args=UINT_MAX
+};
+
+/** Called when we receive a GETCONF message.  Parse the request, and
+ * reply with a CONFVALUE or an ERROR message */
+static int
+handle_control_getconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *msg = NULL;
+  size_t msg_len;
+  const or_options_t *options = get_options();
+  int i, len;
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    if (!option_is_recognized(q)) {
+      smartlist_add(unrecognized, (char*) q);
+    } else {
+      config_line_t *answer = option_get_assignment(options,q);
+      if (!answer) {
+        const char *name = option_get_canonical_name(q);
+        smartlist_add_asprintf(answers, "250-%s\r\n", name);
+      }
+
+      while (answer) {
+        config_line_t *next;
+        smartlist_add_asprintf(answers, "250-%s=%s\r\n",
+                     answer->key, answer->value);
+
+        next = answer->next;
+        tor_free(answer->key);
+        tor_free(answer->value);
+        tor_free(answer);
+        answer = next;
+      }
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if ((len = smartlist_len(unrecognized))) {
+    for (i=0; i < len-1; ++i)
+      control_printf_midreply(conn, 552,
+                              "Unrecognized configuration key \"%s\"",
+                              (char*)smartlist_get(unrecognized, i));
+    control_printf_endreply(conn, 552,
+                            "Unrecognized configuration key \"%s\"",
+                            (char*)smartlist_get(unrecognized, len-1));
+  } else if ((len = smartlist_len(answers))) {
+    char *tmp = smartlist_get(answers, len-1);
+    tor_assert(strlen(tmp)>4);
+    tmp[3] = ' ';
+    msg = smartlist_join_strings(answers, "", 0, &msg_len);
+    connection_buf_add(msg, msg_len, TO_CONN(conn));
+  } else {
+    send_control_done(conn);
+  }
+
+  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
+  smartlist_free(answers);
+  smartlist_free(unrecognized);
+
+  tor_free(msg);
+
+  return 0;
+}
+
+static const control_cmd_syntax_t loadconf_syntax = {
+  .want_cmddata = true
+};
+
+/** Called when we get a +LOADCONF message. */
+static int
+handle_control_loadconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  setopt_err_t retval;
+  char *errstring = NULL;
+
+  retval = options_init_from_string(NULL, args->cmddata,
+                                    CMD_RUN_TOR, NULL, &errstring);
+
+  if (retval != SETOPT_OK)
+    log_warn(LD_CONTROL,
+             "Controller gave us config file that didn't validate: %s",
+             errstring);
+
+#define SEND_ERRMSG(code, msg)                          \
+  control_printf_endreply(conn, code, msg "%s%s",       \
+                          errstring ? ": " : "",        \
+                          errstring ? errstring : "")
+  switch (retval) {
+  case SETOPT_ERR_PARSE:
+    SEND_ERRMSG(552, "Invalid config file");
+    break;
+  case SETOPT_ERR_TRANSITION:
+    SEND_ERRMSG(553, "Transition not allowed");
+    break;
+  case SETOPT_ERR_SETTING:
+    SEND_ERRMSG(553, "Unable to set option");
+    break;
+  case SETOPT_ERR_MISC:
+  default:
+    SEND_ERRMSG(550, "Unable to load config");
+    break;
+  case SETOPT_OK:
+    send_control_done(conn);
+    break;
+  }
+#undef SEND_ERRMSG
+  tor_free(errstring);
+  return 0;
+}
+
+static const control_cmd_syntax_t setevents_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a SETEVENTS message: update conn->event_mask,
+ * and reply with DONE or ERROR. */
+static int
+handle_control_setevents(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  int event_code;
+  event_mask_t event_mask = 0;
+  const smartlist_t *events = args->args;
+
+  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
+    {
+      if (!strcasecmp(ev, "EXTENDED") ||
+          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
+        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
+                 "supported.", ev);
+        continue;
+      } else {
+        int i;
+        event_code = -1;
+
+        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+          if (!strcasecmp(ev, control_event_table[i].event_name)) {
+            event_code = control_event_table[i].event_code;
+            break;
+          }
+        }
+
+        if (event_code == -1) {
+          control_printf_endreply(conn, 552, "Unrecognized event \"%s\"", ev);
+          return 0;
+        }
+      }
+      event_mask |= (((event_mask_t)1) << event_code);
+    }
+  SMARTLIST_FOREACH_END(ev);
+
+  conn->event_mask = event_mask;
+
+  control_update_global_event_mask();
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t saveconf_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a SAVECONF command. Try to flush the current options to
+ * disk, and report success or failure. */
+static int
+handle_control_saveconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  bool force = config_lines_contain_flag(args->kwargs, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
+    control_write_endreply(conn, 551,
+                           "Unable to write configuration to disk.");
+  } else {
+    send_control_done(conn);
+  }
+  return 0;
+}
+
+static const control_cmd_syntax_t signal_syntax = {
+  .min_args = 1,
+  .max_args = 1,
+};
+
+/** Called when we get a SIGNAL command. React to the provided signal, and
+ * report success or failure. (If the signal results in a shutdown, success
+ * may not be reported.) */
+static int
+handle_control_signal(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  int sig = -1;
+  int i;
+
+  tor_assert(smartlist_len(args->args) == 1);
+  const char *s = smartlist_get(args->args, 0);
+
+  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if (!strcasecmp(s, signal_table[i].signal_name)) {
+      sig = signal_table[i].sig;
+      break;
+    }
+  }
+
+  if (sig < 0)
+    control_printf_endreply(conn, 552, "Unrecognized signal code \"%s\"", s);
+  if (sig < 0)
+    return 0;
+
+  send_control_done(conn);
+  /* Flush the "done" first if the signal might make us shut down. */
+  if (sig == SIGTERM || sig == SIGINT)
+    connection_flush(TO_CONN(conn));
+
+  activate_signal(sig);
+
+  return 0;
+}
+
+static const control_cmd_syntax_t takeownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
+ * as an owning connection, so that we will exit if the connection
+ * closes. */
+static int
+handle_control_takeownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 1;
+
+  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t dropownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a DROPOWNERSHIP command.  Mark this connection
+ * as a non-owning connection, so that we will not exit if the connection
+ * closes. */
+static int
+handle_control_dropownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 0;
+
+  log_info(LD_CONTROL, "Control connection %d has dropped ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+/** Given a text circuit <b>id</b>, return the corresponding circuit. */
+static origin_circuit_t *
+get_circ(const char *id)
+{
+  uint32_t n_id;
+  int ok;
+  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  return circuit_get_by_global_id(n_id);
+}
+
+/** Given a text stream <b>id</b>, return the corresponding AP connection. */
+static entry_connection_t *
+get_stream(const char *id)
+{
+  uint64_t n_id;
+  int ok;
+  connection_t *conn;
+  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  conn = connection_get_by_global_id(n_id);
+  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
+    return NULL;
+  return TO_ENTRY_CONN(conn);
+}
+
+/** Helper for setconf and resetconf. Acts like setconf, except
+ * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
+ * contents of body.
+ */
+static int
+control_setconf_helper(control_connection_t *conn,
+                       const control_cmd_args_t *args,
+                       int use_defaults)
+{
+  setopt_err_t opt_err;
+  char *errstring = NULL;
+  const unsigned flags =
+    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
+
+  // We need a copy here, since confparse.c wants to canonicalize cases.
+  config_line_t *lines = config_lines_dup(args->kwargs);
+
+  opt_err = options_trial_assign(lines, flags, &errstring);
+  {
+#define SEND_ERRMSG(code, msg)                                  \
+    control_printf_endreply(conn, code, msg ": %s", errstring);
+
+    switch (opt_err) {
+      case SETOPT_ERR_MISC:
+        SEND_ERRMSG(552, "Unrecognized option");
+        break;
+      case SETOPT_ERR_PARSE:
+        SEND_ERRMSG(513, "Unacceptable option value");
+        break;
+      case SETOPT_ERR_TRANSITION:
+        SEND_ERRMSG(553, "Transition not allowed");
+        break;
+      case SETOPT_ERR_SETTING:
+      default:
+        SEND_ERRMSG(553, "Unable to set option");
+        break;
+      case SETOPT_OK:
+        config_free_lines(lines);
+        send_control_done(conn);
+        return 0;
+    }
+#undef SEND_ERRMSG
+    log_warn(LD_CONTROL,
+             "Controller gave us config lines that didn't validate: %s",
+             errstring);
+    config_free_lines(lines);
+    tor_free(errstring);
+    return 0;
+  }
+}
+
+/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
+ * containing funny characters. */
+static int
+address_is_invalid_mapaddress_target(const char *addr)
+{
+  if (!strcmpstart(addr, "*."))
+    return address_is_invalid_destination(addr+2, 1);
+  else
+    return address_is_invalid_destination(addr, 1);
+}
+
+static const control_cmd_syntax_t mapaddress_syntax = {
+  // no positional arguments are expected
+  .max_args=0,
+  // an arbitrary number of K=V entries are supported.
+  .accept_keywords=true,
+};
+
+/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
+ * and report success or failure. */
+static int
+handle_control_mapaddress(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  smartlist_t *reply;
+  char *r;
+  size_t sz;
+
+  reply = smartlist_new();
+  const config_line_t *line;
+  for (line = args->kwargs; line; line = line->next) {
+    const char *from = line->key;
+    const char *to = line->value;
+    {
+      if (address_is_invalid_mapaddress_target(to)) {
+        smartlist_add_asprintf(reply,
+                     "512-syntax error: invalid address '%s'", to);
+        log_warn(LD_CONTROL,
+                 "Skipping invalid argument '%s' in MapAddress msg", to);
+      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
+                 !strcmp(from, "::")) {
+        const char type =
+          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
+          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
+        const char *address = addressmap_register_virtual_address(
+                                                     type, tor_strdup(to));
+        if (!address) {
+          smartlist_add_asprintf(reply,
+                   "451-resource exhausted: skipping '%s=%s'", from,to);
+          log_warn(LD_CONTROL,
+                   "Unable to allocate address for '%s' in MapAddress msg",
+                   safe_str_client(to));
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
+        }
+      } else {
+        const char *msg;
+        if (addressmap_register_auto(from, to, 1,
+                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
+          smartlist_add_asprintf(reply,
+                                 "512-syntax error: invalid address mapping "
+                                 " '%s=%s': %s", from, to, msg);
+          log_warn(LD_CONTROL,
+                   "Skipping invalid argument '%s=%s' in MapAddress msg: %s",
+                   from, to, msg);
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", from, to);
+        }
+      }
+    }
+  }
+
+  if (smartlist_len(reply)) {
+    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
+    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
+    connection_buf_add(r, sz, TO_CONN(conn));
+    tor_free(r);
+  } else {
+    control_write_endreply(conn, 512, "syntax error: "
+                           "not enough arguments to mapaddress.");
+  }
+
+  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
+  smartlist_free(reply);
+  return 0;
+}
+
+/** Given a string, convert it to a circuit purpose. */
+static uint8_t
+circuit_purpose_from_string(const char *string)
+{
+  if (!strcasecmpstart(string, "purpose="))
+    string += strlen("purpose=");
+
+  if (!strcasecmp(string, "general"))
+    return CIRCUIT_PURPOSE_C_GENERAL;
+  else if (!strcasecmp(string, "controller"))
+    return CIRCUIT_PURPOSE_CONTROLLER;
+  else
+    return CIRCUIT_PURPOSE_UNKNOWN;
+}
+
+static const control_cmd_syntax_t extendcircuit_syntax = {
+  .min_args=1,
+  .max_args=1, // see note in function
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS
+};
+
+/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
+ * circuit, and report success or failure. */
+static int
+handle_control_extendcircuit(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  smartlist_t *router_nicknames=smartlist_new(), *nodes=NULL;
+  origin_circuit_t *circ = NULL;
+  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  const config_line_t *kwargs = args->kwargs;
+  const char *circ_id = smartlist_get(args->args, 0);
+  const char *path_str = NULL;
+  char *path_str_alloc = NULL;
+
+  /* The syntax for this command is unfortunate. The second argument is
+     optional, and is a comma-separated list long-format fingerprints, which
+     can (historically!) contain an equals sign.
+
+     Here we check the second argument to see if it's a path, and if so we
+     remove it from the kwargs list and put it in path_str.
+  */
+  if (kwargs) {
+    const config_line_t *arg1 = kwargs;
+    if (!strcmp(arg1->value, "")) {
+      path_str = arg1->key;
+      kwargs = kwargs->next;
+    } else if (arg1->key[0] == '$') {
+      tor_asprintf(&path_str_alloc, "%s=%s", arg1->key, arg1->value);
+      path_str = path_str_alloc;
+      kwargs = kwargs->next;
+    }
+  }
+
+  const config_line_t *purpose_line = config_line_find_case(kwargs, "PURPOSE");
+  bool zero_circ = !strcmp("0", circ_id);
+
+  if (purpose_line) {
+    intended_purpose = circuit_purpose_from_string(purpose_line->value);
+    if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purpose_line->value);
+      goto done;
+    }
+  }
+
+  if (zero_circ) {
+    if (!path_str) {
+      // "EXTENDCIRCUIT 0" with no path.
+      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
+      if (!circ) {
+        control_write_endreply(conn, 551, "Couldn't start circuit");
+      } else {
+        control_printf_endreply(conn, 250, "EXTENDED %lu",
+                                (unsigned long)circ->global_identifier);
+      }
+      goto done;
+    }
+  }
+
+  if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  if (!path_str) {
+    control_write_endreply(conn, 512, "syntax error: path required.");
+    goto done;
+  }
+
+  smartlist_split_string(router_nicknames, path_str, ",", 0, 0);
+
+  nodes = smartlist_new();
+  bool first_node = zero_circ;
+  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
+    const node_t *node = node_get_by_nickname(n, 0);
+    if (!node) {
+      control_printf_endreply(conn, 552, "No such router \"%s\"", n);
+      goto done;
+    }
+    if (!node_has_preferred_descriptor(node, first_node)) {
+      control_printf_endreply(conn, 552, "No descriptor for \"%s\"", n);
+      goto done;
+    }
+    smartlist_add(nodes, (void*)node);
+    first_node = false;
+  } SMARTLIST_FOREACH_END(n);
+
+  if (!smartlist_len(nodes)) {
+    control_write_endreply(conn, 512, "No router names provided");
+    goto done;
+  }
+
+  if (zero_circ) {
+    /* start a new circuit */
+    circ = origin_circuit_init(intended_purpose, 0);
+  }
+
+  /* now circ refers to something that is ready to be extended */
+  first_node = zero_circ;
+  SMARTLIST_FOREACH(nodes, const node_t *, node,
+  {
+    extend_info_t *info = extend_info_from_node(node, first_node);
+    if (!info) {
+      tor_assert_nonfatal(first_node);
+      log_warn(LD_CONTROL,
+               "controller tried to connect to a node that lacks a suitable "
+               "descriptor, or which doesn't have any "
+               "addresses that are allowed by the firewall configuration; "
+               "circuit marked for closing.");
+      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+    circuit_append_new_exit(circ, info);
+    if (circ->build_state->desired_path_len > 1) {
+      circ->build_state->onehop_tunnel = 0;
+    }
+    extend_info_free(info);
+    first_node = 0;
+  });
+
+  /* now that we've populated the cpath, start extending */
+  if (zero_circ) {
+    int err_reason = 0;
+    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
+      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+  } else {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
+      int err_reason = 0;
+      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
+      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
+        log_info(LD_CONTROL,
+                 "send_next_onion_skin failed; circuit marked for closing.");
+        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+        control_write_endreply(conn, 551, "Couldn't send onion skin");
+        goto done;
+      }
+    }
+  }
+
+  control_printf_endreply(conn, 250, "EXTENDED %lu",
+                          (unsigned long)circ->global_identifier);
+  if (zero_circ) /* send a 'launched' event, for completeness */
+    circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
+ done:
+  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
+  smartlist_free(router_nicknames);
+  smartlist_free(nodes);
+  tor_free(path_str_alloc);
+  return 0;
+}
+
+static const control_cmd_syntax_t setcircuitpurpose_syntax = {
+  .max_args=1,
+  .accept_keywords=true,
+};
+
+/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
+ * circuit and it's a valid purpose, change it. */
+static int
+handle_control_setcircuitpurpose(control_connection_t *conn,
+                                 const control_cmd_args_t *args)
+{
+  origin_circuit_t *circ = NULL;
+  uint8_t new_purpose;
+  const char *circ_id = smartlist_get(args->args,0);
+
+  if (!(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  {
+    const config_line_t *purp = config_line_find_case(args->kwargs, "PURPOSE");
+    if (!purp) {
+      control_write_endreply(conn, 552, "No purpose given");
+      goto done;
+    }
+    new_purpose = circuit_purpose_from_string(purp->value);
+    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purp->value);
+      goto done;
+    }
+  }
+
+  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
+  send_control_done(conn);
+
+ done:
+  return 0;
+}
+
+static const char *attachstream_keywords[] = {
+  "HOP", NULL
+};
+static const control_cmd_syntax_t attachstream_syntax = {
+  .min_args=2, .max_args=2,
+  .accept_keywords=true,
+  .allowed_keywords=attachstream_keywords
+};
+
+/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
+ * stream, and report success or failure. */
+static int
+handle_control_attachstream(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  entry_connection_t *ap_conn = NULL;
+  origin_circuit_t *circ = NULL;
+  crypt_path_t *cpath=NULL;
+  int hop=0, hop_line_ok=1;
+  const char *stream_id = smartlist_get(args->args, 0);
+  const char *circ_id = smartlist_get(args->args, 1);
+  int zero_circ = !strcmp(circ_id, "0");
+  const config_line_t *hoparg = config_line_find_case(args->kwargs, "HOP");
+
+  if (!(ap_conn = get_stream(stream_id))) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"", stream_id);
+    return 0;
+  } else if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  } else if (circ) {
+    if (hoparg) {
+      hop = (int) tor_parse_ulong(hoparg->value, 10, 0, INT_MAX,
+                                  &hop_line_ok, NULL);
+      if (!hop_line_ok) { /* broken hop line */
+        control_printf_endreply(conn, 552, "Bad value hop=%s",
+                                hoparg->value);
+        return 0;
+      }
+    }
+  }
+
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
+    control_write_endreply(conn, 555,
+                           "Connection is not managed by controller.");
+    return 0;
+  }
+
+  /* Do we need to detach it first? */
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
+    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
+    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
+    /* Un-mark it as ending, since we're going to reuse it. */
+    edge_conn->edge_has_sent_end = 0;
+    edge_conn->end_reason = 0;
+    if (tmpcirc)
+      circuit_detach_stream(tmpcirc, edge_conn);
+    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
+    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
+  }
+
+  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to non-open origin circuit");
+    return 0;
+  }
+  /* Is this a single hop circuit? */
+  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to this one-hop circuit.");
+    return 0;
+  }
+
+  if (circ && hop>0) {
+    /* find this hop in the circuit, and set cpath */
+    cpath = circuit_get_cpath_hop(circ, hop);
+    if (!cpath) {
+      control_printf_endreply(conn, 551, "Circuit doesn't have %d hops.", hop);
+      return 0;
+    }
+  }
+  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
+    control_write_endreply(conn, 551, "Unable to attach stream");
+    return 0;
+  }
+  send_control_done(conn);
+  return 0;
+}
+
+static const char *postdescriptor_keywords[] = {
+  "cache", "purpose", NULL,
+};
+
+static const control_cmd_syntax_t postdescriptor_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .allowed_keywords = postdescriptor_keywords,
+  .want_cmddata = true,
+};
+
+/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
+ * descriptor, and report success or failure. */
+static int
+handle_control_postdescriptor(control_connection_t *conn,
+                              const control_cmd_args_t *args)
+{
+  const char *msg=NULL;
+  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
+  int cache = 0; /* eventually, we may switch this to 1 */
+  const config_line_t *line;
+
+  line = config_line_find_case(args->kwargs, "purpose");
+  if (line) {
+    purpose = router_purpose_from_string(line->value);
+    if (purpose == ROUTER_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+  line = config_line_find_case(args->kwargs, "cache");
+  if (line) {
+    if (!strcasecmp(line->value, "no"))
+      cache = 0;
+    else if (!strcasecmp(line->value, "yes"))
+      cache = 1;
+    else {
+      control_printf_endreply(conn, 552, "Unknown cache request \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+
+  switch (router_load_single_router(args->cmddata, purpose, cache, &msg)) {
+  case -1:
+    if (!msg) msg = "Could not parse descriptor";
+    control_write_endreply(conn, 554, msg);
+    break;
+  case 0:
+    if (!msg) msg = "Descriptor not added";
+    control_write_endreply(conn, 251, msg);
+    break;
+  case 1:
+    send_control_done(conn);
+    break;
+  }
+
+ done:
+  return 0;
+}
+
+static const control_cmd_syntax_t redirectstream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, // XXX should be 3.
+};
+
+/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
+ * address of the named AP stream, and report success or failure. */
+static int
+handle_control_redirectstream(control_connection_t *conn,
+                              const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn = NULL;
+  char *new_addr = NULL;
+  uint16_t new_port = 0;
+  const smartlist_t *args = cmd_args->args;
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
+           || !ap_conn->socks_request) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  } else {
+    int ok = 1;
+    if (smartlist_len(args) > 2) { /* they included a port too */
+      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
+                                            10, 1, 65535, &ok, NULL);
+    }
+    if (!ok) {
+      control_printf_endreply(conn, 512, "Cannot parse port \"%s\"",
+                              (char*)smartlist_get(args, 2));
+    } else {
+      new_addr = tor_strdup(smartlist_get(args, 1));
+    }
+  }
+
+  if (!new_addr)
+    return 0;
+
+  strlcpy(ap_conn->socks_request->address, new_addr,
+          sizeof(ap_conn->socks_request->address));
+  if (new_port)
+    ap_conn->socks_request->port = new_port;
+  tor_free(new_addr);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closestream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, /* XXXX This is the original behavior, but
+                         * maybe we should change the spec. */
+};
+
+/** Called when we get a CLOSESTREAM command; try to close the named stream
+ * and report success or failure. */
+static int
+handle_control_closestream(control_connection_t *conn,
+                           const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn=NULL;
+  uint8_t reason=0;
+  int ok;
+  const smartlist_t *args = cmd_args->args;
+
+  tor_assert(smartlist_len(args) >= 2);
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0))))
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  else {
+    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
+                                       &ok, NULL);
+    if (!ok) {
+      control_printf_endreply(conn, 552, "Unrecognized reason \"%s\"",
+                              (char*)smartlist_get(args, 1));
+      ap_conn = NULL;
+    }
+  }
+  if (!ap_conn)
+    return 0;
+
+  connection_mark_unattached_ap(ap_conn, reason);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closecircuit_syntax = {
+  .min_args=1, .max_args=1,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+  // XXXX we might want to exclude unrecognized flags, but for now we
+  // XXXX just ignore them for backward compatibility.
+};
+
+/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
+ * and report success or failure. */
+static int
+handle_control_closecircuit(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  const char *circ_id = smartlist_get(args->args, 0);
+  origin_circuit_t *circ = NULL;
+
+  if (!(circ=get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  }
+
+  bool safe =  config_lines_contain_flag(args->kwargs, "IfUnused");
+
+  if (!safe || !circ->p_streams) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
+  }
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t resolve_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a RESOLVE command: start trying to resolve
+ * the listed addresses. */
+static int
+handle_control_resolve(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  smartlist_t *failed;
+  int is_reverse = 0;
+
+  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
+    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
+             "isn't listening for ADDRMAP events.  It probably won't see "
+             "the answer.");
+  }
+
+  {
+    const config_line_t *modearg = config_line_find_case(args->kwargs, "mode");
+    if (modearg && !strcasecmp(modearg->value, "reverse"))
+      is_reverse = 1;
+  }
+  failed = smartlist_new();
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strlen(line->value)) {
+      const char *addr = line->key;
+      if (dnsserv_launch_request(addr, is_reverse, conn)<0)
+        smartlist_add(failed, (char*)addr);
+    } else {
+      // XXXX arguably we should reject unrecognized keyword arguments,
+      // XXXX but the old implementation didn't do that.
+    }
+  }
+
+  send_control_done(conn);
+  SMARTLIST_FOREACH(failed, const char *, arg, {
+      control_event_address_mapped(arg, arg, time(NULL),
+                                   "internal", 0);
+  });
+
+  smartlist_free(failed);
+  return 0;
+}
+
+static const control_cmd_syntax_t protocolinfo_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a PROTOCOLINFO command: send back a reply. */
+static int
+handle_control_protocolinfo(control_connection_t *conn,
+                            const control_cmd_args_t *cmd_args)
+{
+  const char *bad_arg = NULL;
+  const smartlist_t *args = cmd_args->args;
+
+  conn->have_sent_protocolinfo = 1;
+
+  SMARTLIST_FOREACH(args, const char *, arg, {
+      int ok;
+      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
+      if (!ok) {
+        bad_arg = arg;
+        break;
+      }
+    });
+  if (bad_arg) {
+    control_printf_endreply(conn, 513, "No such version %s",
+                            escaped(bad_arg));
+    /* Don't tolerate bad arguments when not authenticated. */
+    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
+      connection_mark_for_close(TO_CONN(conn));
+    goto done;
+  } else {
+    const or_options_t *options = get_options();
+    int cookies = options->CookieAuthentication;
+    char *cfile = get_controller_cookie_file_name();
+    char *abs_cfile;
+    char *esc_cfile;
+    char *methods;
+    abs_cfile = make_path_absolute(cfile);
+    esc_cfile = esc_for_log(abs_cfile);
+    {
+      int passwd = (options->HashedControlPassword != NULL ||
+                    options->HashedControlSessionPassword != NULL);
+      smartlist_t *mlist = smartlist_new();
+      if (cookies) {
+        smartlist_add(mlist, (char*)"COOKIE");
+        smartlist_add(mlist, (char*)"SAFECOOKIE");
+      }
+      if (passwd)
+        smartlist_add(mlist, (char*)"HASHEDPASSWORD");
+      if (!cookies && !passwd)
+        smartlist_add(mlist, (char*)"NULL");
+      methods = smartlist_join_strings(mlist, ",", 0, NULL);
+      smartlist_free(mlist);
+    }
+
+    control_write_midreply(conn, 250, "PROTOCOLINFO 1");
+    control_printf_midreply(conn, 250, "AUTH METHODS=%s%s%s", methods,
+                            cookies?" COOKIEFILE=":"",
+                            cookies?esc_cfile:"");
+    control_printf_midreply(conn, 250, "VERSION Tor=%s", escaped(VERSION));
+    send_control_done(conn);
+
+    tor_free(methods);
+    tor_free(cfile);
+    tor_free(abs_cfile);
+    tor_free(esc_cfile);
+  }
+ done:
+  return 0;
+}
+
+static const control_cmd_syntax_t usefeature_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a USEFEATURE command: parse the feature list, and
+ * set up the control_connection's options properly. */
+static int
+handle_control_usefeature(control_connection_t *conn,
+                          const control_cmd_args_t *cmd_args)
+{
+  const smartlist_t *args = cmd_args->args;
+  int bad = 0;
+  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
+      if (!strcasecmp(arg, "VERBOSE_NAMES"))
+        ;
+      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
+        ;
+      else {
+        control_printf_endreply(conn, 552, "Unrecognized feature \"%s\"",
+                                arg);
+        bad = 1;
+        break;
+      }
+  } SMARTLIST_FOREACH_END(arg);
+
+  if (!bad) {
+    send_control_done(conn);
+  }
+
+  return 0;
+}
+
+static const control_cmd_syntax_t dropguards_syntax = {
+  .max_args = 0,
+};
+
+/** Implementation for the DROPGUARDS command. */
+static int
+handle_control_dropguards(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  (void) args; /* We don't take arguments. */
+
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
+  remove_all_entry_guards();
+  send_control_done(conn);
+
+  return 0;
+}
+
+static const char *hsfetch_keywords[] = {
+  "SERVER", NULL,
+};
+static const control_cmd_syntax_t hsfetch_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = hsfetch_keywords,
+};
+
+/** Implementation for the HSFETCH command. */
+static int
+handle_control_hsfetch(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+
+{
+  char digest[DIGEST_LEN], *desc_id = NULL;
+  smartlist_t *hsdirs = NULL;
+  static const char *v2_str = "v2-";
+  const size_t v2_str_len = strlen(v2_str);
+  rend_data_t *rend_query = NULL;
+  ed25519_public_key_t v3_pk;
+  uint32_t version;
+  const char *hsaddress = NULL;
+
+  /* Extract the first argument (either HSAddress or DescID). */
+  const char *arg1 = smartlist_get(args->args, 0);
+  /* Test if it's an HS address without the .onion part. */
+  if (rend_valid_v2_service_id(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_TWO;
+  } else if (strcmpstart(arg1, v2_str) == 0 &&
+             rend_valid_descriptor_id(arg1 + v2_str_len) &&
+             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
+                           REND_DESC_ID_V2_LEN_BASE32) ==
+                REND_DESC_ID_V2_LEN_BASE32) {
+    /* We have a well formed version 2 descriptor ID. Keep the decoded value
+     * of the id. */
+    desc_id = digest;
+    version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_THREE;
+    hs_parse_address(hsaddress, &v3_pk, NULL, NULL);
+  } else {
+    control_printf_endreply(conn, 513, "Invalid argument \"%s\"", arg1);
+    goto done;
+  }
+
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strcasecmp(line->key, "SERVER")) {
+      const char *server = line->value;
+
+      const node_t *node = node_get_by_hex_id(server, 0);
+      if (!node) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found", server);
+        goto done;
+      }
+      if (!hsdirs) {
+        /* Stores routerstatus_t cmddata for each specified server. */
+        hsdirs = smartlist_new();
+      }
+      /* Valid server, add it to our local list. */
+      smartlist_add(hsdirs, node->rs);
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  if (version == HS_VERSION_TWO) {
+    rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
+                                         REND_NO_AUTH);
+    if (rend_query == NULL) {
+      control_write_endreply(conn, 551, "Error creating the HS query");
+      goto done;
+    }
+  }
+
+  /* Using a descriptor ID, we force the user to provide at least one
+   * hsdir server using the SERVER= option. */
+  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
+    control_write_endreply(conn, 512, "SERVER option is required");
+    goto done;
+  }
+
+  /* We are about to trigger HSDir fetch so send the OK now because after
+   * that 650 event(s) are possible so better to have the 250 OK before them
+   * to avoid out of order replies. */
+  send_control_done(conn);
+
+  /* Trigger the fetch using the built rend query and possibly a list of HS
+   * directory to use. This function ignores the client cache thus this will
+   * always send a fetch command. */
+  if (version == HS_VERSION_TWO) {
+    rend_client_fetch_v2_desc(rend_query, hsdirs);
+  } else if (version == HS_VERSION_THREE) {
+    hs_control_hsfetch_command(&v3_pk, hsdirs);
+  }
+
+ done:
+  /* Contains data pointer that we don't own thus no cleanup. */
+  smartlist_free(hsdirs);
+  rend_data_free(rend_query);
+  return 0;
+}
+
+static const char *hspost_keywords[] = {
+  "SERVER", "HSADDRESS", NULL
+};
+static const control_cmd_syntax_t hspost_syntax = {
+  .min_args = 0, .max_args = 0,
+  .accept_keywords = true,
+  .want_cmddata = true,
+  .allowed_keywords = hspost_keywords
+};
+
+/** Implementation for the HSPOST command. */
+static int
+handle_control_hspost(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  smartlist_t *hs_dirs = NULL;
+  const char *encoded_desc = args->cmddata;
+  size_t encoded_desc_len = args->cmddata_len;
+  const char *onion_address = NULL;
+  const config_line_t *line;
+
+  for (line = args->kwargs; line; line = line->next) {
+    if (!strcasecmpstart(line->key, "SERVER")) {
+      const char *server = line->value;
+      const node_t *node = node_get_by_hex_id(server, 0);
+
+      if (!node || !node->rs) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found",
+                                server);
+        goto done;
+      }
+      /* Valid server, add it to our local list. */
+      if (!hs_dirs)
+        hs_dirs = smartlist_new();
+      smartlist_add(hs_dirs, node->rs);
+    } else if (!strcasecmpstart(line->key, "HSADDRESS")) {
+      const char *address = line->value;
+      if (!hs_address_is_valid(address)) {
+        control_write_endreply(conn, 512, "Malformed onion address");
+        goto done;
+      }
+      onion_address = address;
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  /* Handle the v3 case. */
+  if (onion_address) {
+    if (hs_control_hspost_command(encoded_desc, onion_address, hs_dirs) < 0) {
+      control_write_endreply(conn, 554, "Invalid descriptor");
+    } else {
+      send_control_done(conn);
+    }
+    goto done;
+  }
+
+  /* From this point on, it is only v2. */
+
+  /*  parse it. */
+  rend_encoded_v2_service_descriptor_t *desc =
+      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
+  desc->desc_str = tor_memdup_nulterm(encoded_desc, encoded_desc_len);
+
+  rend_service_descriptor_t *parsed = NULL;
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
+                                        &intro_size, &encoded_size,
+                                        &next_desc, desc->desc_str, 1)) {
+    /* Post the descriptor. */
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    if (!rend_get_service_id(parsed->pk, serviceid)) {
+      smartlist_t *descs = smartlist_new();
+      smartlist_add(descs, desc);
+
+      /* We are about to trigger HS descriptor upload so send the OK now
+       * because after that 650 event(s) are possible so better to have the
+       * 250 OK before them to avoid out of order replies. */
+      send_control_done(conn);
+
+      /* Trigger the descriptor upload */
+      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
+      smartlist_free(descs);
+    }
+
+    rend_service_descriptor_free(parsed);
+  } else {
+    control_write_endreply(conn, 554, "Invalid descriptor");
+  }
+
+  tor_free(intro_content);
+  rend_encoded_v2_service_descriptor_free(desc);
+ done:
+  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
+  return 0;
+}
+
+/* Helper function for ADD_ONION that adds an ephemeral service depending on
+ * the given hs_version.
+ *
+ * The secret key in pk depends on the hs_version. The ownership of the key
+ * used in pk is given to the HS subsystem so the caller must stop accessing
+ * it after.
+ *
+ * The port_cfgs is a list of service port. Ownership transferred to service.
+ * The max_streams refers to the MaxStreams= key.
+ * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
+ * The auth_type is the authentication type of the clients in auth_clients.
+ * The ownership of that list is transferred to the service.
+ *
+ * On success (RSAE_OKAY), the address_out points to a newly allocated string
+ * containing the onion address without the .onion part. On error, address_out
+ * is untouched. */
+static hs_service_add_ephemeral_status_t
+add_onion_helper_add_service(int hs_version,
+                             add_onion_secret_key_t *pk,
+                             smartlist_t *port_cfgs, int max_streams,
+                             int max_streams_close_circuit, int auth_type,
+                             smartlist_t *auth_clients, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+
+  tor_assert(pk);
+  tor_assert(port_cfgs);
+  tor_assert(address_out);
+
+  switch (hs_version) {
+  case HS_VERSION_TWO:
+    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
+                                     max_streams_close_circuit, auth_type,
+                                     auth_clients, address_out);
+    break;
+  case HS_VERSION_THREE:
+    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
+                                   max_streams_close_circuit, address_out);
+    break;
+  default:
+    tor_assert_unreached();
+  }
+
+  return ret;
+}
+
+/** The list of onion services that have been added via ADD_ONION that do not
+ * belong to any particular control connection.
+ */
+static smartlist_t *detached_onion_services = NULL;
+
+/**
+ * Return a list of detached onion services, or NULL if none exist.
+ **/
+smartlist_t *
+get_detached_onion_services(void)
+{
+  return detached_onion_services;
+}
+
+static const char *add_onion_keywords[] = {
+   "Port", "Flags", "MaxStreams", "ClientAuth", NULL
+};
+static const control_cmd_syntax_t add_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = add_onion_keywords
+};
+
+/** Called when we get a ADD_ONION command; parse the body, and set up
+ * the new ephemeral Onion Service. */
+static int
+handle_control_add_onion(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  /* Parse all of the arguments that do not involve handling cryptographic
+   * material first, since there's no reason to touch that at all if any of
+   * the other arguments are malformed.
+   */
+  smartlist_t *port_cfgs = smartlist_new();
+  smartlist_t *auth_clients = NULL;
+  smartlist_t *auth_created_clients = NULL;
+  int discard_pk = 0;
+  int detach = 0;
+  int max_streams = 0;
+  int max_streams_close_circuit = 0;
+  rend_auth_type_t auth_type = REND_NO_AUTH;
+  int non_anonymous = 0;
+  const config_line_t *arg;
+
+  for (arg = args->kwargs; arg; arg = arg->next) {
+    if (!strcasecmp(arg->key, "Port")) {
+      /* "Port=VIRTPORT[,TARGET]". */
+      rend_service_port_config_t *cfg =
+          rend_service_parse_port_config(arg->value, ",", NULL);
+      if (!cfg) {
+        control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+        goto out;
+      }
+      smartlist_add(port_cfgs, cfg);
+    } else if (!strcasecmp(arg->key, "MaxStreams")) {
+      /* "MaxStreams=[0..65535]". */
+      int ok = 0;
+      max_streams = (int)tor_parse_long(arg->value, 10, 0, 65535, &ok, NULL);
+      if (!ok) {
+        control_write_endreply(conn, 512, "Invalid MaxStreams");
+        goto out;
+      }
+    } else if (!strcasecmp(arg->key, "Flags")) {
+      /* "Flags=Flag[,Flag]", where Flag can be:
+       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
+       *                   the response.
+       *   * 'Detach' - Do not tie this onion service to any particular control
+       *                connection.
+       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
+       *                                exceeded.
+       *   * 'BasicAuth' - Client authorization using the 'basic' method.
+       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
+       *                      flag is present, tor must be in non-anonymous
+       *                      hidden service mode. If this flag is absent,
+       *                      tor must be in anonymous hidden service mode.
+       */
+      static const char *discard_flag = "DiscardPK";
+      static const char *detach_flag = "Detach";
+      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
+      static const char *basicauth_flag = "BasicAuth";
+      static const char *non_anonymous_flag = "NonAnonymous";
+
+      smartlist_t *flags = smartlist_new();
+      int bad = 0;
+
+      smartlist_split_string(flags, arg->value, ",", SPLIT_IGNORE_BLANK, 0);
+      if (smartlist_len(flags) < 1) {
+        control_write_endreply(conn, 512, "Invalid 'Flags' argument");
+        bad = 1;
+      }
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
+      {
+        if (!strcasecmp(flag, discard_flag)) {
+          discard_pk = 1;
+        } else if (!strcasecmp(flag, detach_flag)) {
+          detach = 1;
+        } else if (!strcasecmp(flag, max_s_close_flag)) {
+          max_streams_close_circuit = 1;
+        } else if (!strcasecmp(flag, basicauth_flag)) {
+          auth_type = REND_BASIC_AUTH;
+        } else if (!strcasecmp(flag, non_anonymous_flag)) {
+          non_anonymous = 1;
+        } else {
+          control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s",
+                                  escaped(flag));
+          bad = 1;
+          break;
+        }
+      } SMARTLIST_FOREACH_END(flag);
+      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
+      smartlist_free(flags);
+      if (bad)
+        goto out;
+
+    } else if (!strcasecmp(arg->key, "ClientAuth")) {
+      int created = 0;
+      rend_authorized_client_t *client =
+        add_onion_helper_clientauth(arg->value, &created, conn);
+      if (!client) {
+        goto out;
+      }
+
+      if (auth_clients != NULL) {
+        int bad = 0;
+        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
+          if (strcmp(ac->client_name, client->client_name) == 0) {
+            bad = 1;
+            break;
+          }
+        } SMARTLIST_FOREACH_END(ac);
+        if (bad) {
+          control_write_endreply(conn, 512, "Duplicate name in ClientAuth");
+          rend_authorized_client_free(client);
+          goto out;
+        }
+      } else {
+        auth_clients = smartlist_new();
+        auth_created_clients = smartlist_new();
+      }
+      smartlist_add(auth_clients, client);
+      if (created) {
+        smartlist_add(auth_created_clients, client);
+      }
+    } else {
+      tor_assert_nonfatal_unreached();
+      goto out;
+    }
+  }
+  if (smartlist_len(port_cfgs) == 0) {
+    control_write_endreply(conn, 512, "Missing 'Port' argument");
+    goto out;
+  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
+    control_write_endreply(conn, 512, "No auth type specified");
+    goto out;
+  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
+    control_write_endreply(conn, 512, "No auth clients specified");
+    goto out;
+  } else if ((auth_type == REND_BASIC_AUTH &&
+              smartlist_len(auth_clients) > 512) ||
+             (auth_type == REND_STEALTH_AUTH &&
+              smartlist_len(auth_clients) > 16)) {
+    control_write_endreply(conn, 512, "Too many auth clients");
+    goto out;
+  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
+                                                              get_options())) {
+    /* If we failed, and the non-anonymous flag is set, Tor must be in
+     * anonymous hidden service mode.
+     * The error message changes based on the current Tor config:
+     * 512 Tor is in anonymous hidden service mode
+     * 512 Tor is in non-anonymous hidden service mode
+     * (I've deliberately written them out in full here to aid searchability.)
+     */
+    control_printf_endreply(conn, 512,
+                            "Tor is in %sanonymous hidden service " "mode",
+                            non_anonymous ? "" : "non-");
+    goto out;
+  }
+
+  /* Parse the "keytype:keyblob" argument. */
+  int hs_version = 0;
+  add_onion_secret_key_t pk = { NULL };
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+
+  const char *onionkey = smartlist_get(args->args, 0);
+  if (add_onion_helper_keyarg(onionkey, discard_pk,
+                              &key_new_alg, &key_new_blob, &pk, &hs_version,
+                              conn) < 0) {
+    goto out;
+  }
+
+  /* Hidden service version 3 don't have client authentication support so if
+   * ClientAuth was given, send back an error. */
+  if (hs_version == HS_VERSION_THREE && auth_clients) {
+    control_write_endreply(conn, 513, "ClientAuth not supported");
+    goto out;
+  }
+
+  /* Create the HS, using private key pk, client authentication auth_type,
+   * the list of auth_clients, and port config port_cfg.
+   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
+   * regardless of success/failure.
+   */
+  char *service_id = NULL;
+  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
+                                         max_streams,
+                                         max_streams_close_circuit, auth_type,
+                                         auth_clients, &service_id);
+  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
+  auth_clients = NULL; /* so is auth_clients */
+  switch (ret) {
+  case RSAE_OKAY:
+  {
+    if (detach) {
+      if (!detached_onion_services)
+        detached_onion_services = smartlist_new();
+      smartlist_add(detached_onion_services, service_id);
+    } else {
+      if (!conn->ephemeral_onion_services)
+        conn->ephemeral_onion_services = smartlist_new();
+      smartlist_add(conn->ephemeral_onion_services, service_id);
+    }
+
+    tor_assert(service_id);
+    control_printf_midreply(conn, 250, "ServiceID=%s", service_id);
+    if (key_new_alg) {
+      tor_assert(key_new_blob);
+      control_printf_midreply(conn, 250, "PrivateKey=%s:%s",
+                              key_new_alg, key_new_blob);
+    }
+    if (auth_created_clients) {
+      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
+        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
+                                                auth_type);
+        tor_assert(encoded);
+        control_printf_midreply(conn, 250, "ClientAuth=%s:%s",
+                                ac->client_name, encoded);
+        memwipe(encoded, 0, strlen(encoded));
+        tor_free(encoded);
+      });
+    }
+
+    send_control_done(conn);
+    break;
+  }
+  case RSAE_BADPRIVKEY:
+    control_write_endreply(conn, 551, "Failed to generate onion address");
+    break;
+  case RSAE_ADDREXISTS:
+    control_write_endreply(conn, 550, "Onion address collision");
+    break;
+  case RSAE_BADVIRTPORT:
+    control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+    break;
+  case RSAE_BADAUTH:
+    control_write_endreply(conn, 512, "Invalid client authorization");
+    break;
+  case RSAE_INTERNAL: FALLTHROUGH;
+  default:
+    control_write_endreply(conn, 551, "Failed to add Onion Service");
+  }
+  if (key_new_blob) {
+    memwipe(key_new_blob, 0, strlen(key_new_blob));
+    tor_free(key_new_blob);
+  }
+
+ out:
+  if (port_cfgs) {
+    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
+                      rend_service_port_config_free(p));
+    smartlist_free(port_cfgs);
+  }
+
+  if (auth_clients) {
+    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
+                      rend_authorized_client_free(ac));
+    smartlist_free(auth_clients);
+  }
+  if (auth_created_clients) {
+    // Do not free entries; they are the same as auth_clients
+    smartlist_free(auth_created_clients);
+  }
+  return 0;
+}
+
+/** Helper function to handle parsing the KeyType:KeyBlob argument to the
+ * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
+ * and the private key not discarded, the algorithm and serialized private key,
+ * or NULL and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned key_new_blob.
+ *
+ * Note: The error messages returned are deliberately vague to avoid echoing
+ * key material.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC int
+add_onion_helper_keyarg(const char *arg, int discard_pk,
+                        const char **key_new_alg_out, char **key_new_blob_out,
+                        add_onion_secret_key_t *decoded_key, int *hs_version,
+                        control_connection_t *conn)
+{
+  smartlist_t *key_args = smartlist_new();
+  crypto_pk_t *pk = NULL;
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+  int ret = -1;
+
+  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(key_args) != 2) {
+    control_write_endreply(conn, 512, "Invalid key type/blob");
+    goto err;
+  }
+
+  /* The format is "KeyType:KeyBlob". */
+  static const char *key_type_new = "NEW";
+  static const char *key_type_best = "BEST";
+  static const char *key_type_rsa1024 = "RSA1024";
+  static const char *key_type_ed25519_v3 = "ED25519-V3";
+
+  const char *key_type = smartlist_get(key_args, 0);
+  const char *key_blob = smartlist_get(key_args, 1);
+
+  if (!strcasecmp(key_type_rsa1024, key_type)) {
+    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
+    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
+    if (!pk) {
+      control_write_endreply(conn, 512, "Failed to decode RSA key");
+      goto err;
+    }
+    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
+      crypto_pk_free(pk);
+      control_write_endreply(conn, 512, "Invalid RSA key size");
+      goto err;
+    }
+    decoded_key->v2 = pk;
+    *hs_version = HS_VERSION_TWO;
+  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
+    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
+    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
+                      strlen(key_blob)) != sizeof(sk->seckey)) {
+      tor_free(sk);
+      control_write_endreply(conn, 512, "Failed to decode ED25519-V3 key");
+      goto err;
+    }
+    decoded_key->v3 = sk;
+    *hs_version = HS_VERSION_THREE;
+  } else if (!strcasecmp(key_type_new, key_type)) {
+    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
+    if (!strcasecmp(key_type_rsa1024, key_blob)) {
+      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
+      pk = crypto_pk_new();
+      if (crypto_pk_generate_key(pk)) {
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_rsa1024);
+        goto err;
+      }
+      if (!discard_pk) {
+        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
+          crypto_pk_free(pk);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_rsa1024);
+          goto err;
+        }
+        key_new_alg = key_type_rsa1024;
+      }
+      decoded_key->v2 = pk;
+      *hs_version = HS_VERSION_TWO;
+    } else if (!strcasecmp(key_type_ed25519_v3, key_blob) ||
+               !strcasecmp(key_type_best, key_blob)) {
+      /* "ED25519-V3", ed25519 key, also currently "BEST" by default. */
+      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+      if (ed25519_secret_key_generate(sk, 1) < 0) {
+        tor_free(sk);
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_ed25519_v3);
+        goto err;
+      }
+      if (!discard_pk) {
+        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
+        key_new_blob = tor_malloc_zero(len);
+        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
+                          sizeof(sk->seckey), 0) != (len - 1)) {
+          tor_free(sk);
+          tor_free(key_new_blob);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_ed25519_v3);
+          goto err;
+        }
+        key_new_alg = key_type_ed25519_v3;
+      }
+      decoded_key->v3 = sk;
+      *hs_version = HS_VERSION_THREE;
+    } else {
+      control_write_endreply(conn, 513, "Invalid key type");
+      goto err;
+    }
+  } else {
+    control_write_endreply(conn, 513, "Invalid key type");
+    goto err;
+  }
+
+  /* Succeeded in loading or generating a private key. */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(key_args, char *, cp, {
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+  });
+  smartlist_free(key_args);
+
+  *key_new_alg_out = key_new_alg;
+  *key_new_blob_out = key_new_blob;
+
+  return ret;
+}
+
+/** Helper function to handle parsing a ClientAuth argument to the
+ * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
+ * and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned auth_client.
+ *
+ * If 'created' is specified, it will be set to 1 when a new cookie has
+ * been generated.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC rend_authorized_client_t *
+add_onion_helper_clientauth(const char *arg, int *created,
+                            control_connection_t *conn)
+{
+  int ok = 0;
+
+  tor_assert(arg);
+  tor_assert(created);
+
+  smartlist_t *auth_args = smartlist_new();
+  rend_authorized_client_t *client =
+    tor_malloc_zero(sizeof(rend_authorized_client_t));
+  smartlist_split_string(auth_args, arg, ":", 0, 0);
+  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
+    control_write_endreply(conn, 512, "Invalid ClientAuth syntax");
+    goto err;
+  }
+  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
+  if (smartlist_len(auth_args) == 2) {
+    char *decode_err_msg = NULL;
+    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
+                                client->descriptor_cookie,
+                                NULL, &decode_err_msg) < 0) {
+      tor_assert(decode_err_msg);
+      control_write_endreply(conn, 512, decode_err_msg);
+      tor_free(decode_err_msg);
+      goto err;
+    }
+    *created = 0;
+  } else {
+    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
+    *created = 1;
+  }
+
+  if (!rend_valid_client_name(client->client_name)) {
+    control_write_endreply(conn, 512, "Invalid name in ClientAuth");
+    goto err;
+  }
+
+  ok = 1;
+ err:
+  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
+  smartlist_free(auth_args);
+  if (!ok) {
+    rend_authorized_client_free(client);
+    client = NULL;
+  }
+  return client;
+}
+
+static const control_cmd_syntax_t del_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+};
+
+/** Called when we get a DEL_ONION command; parse the body, and remove
+ * the existing ephemeral Onion Service. */
+static int
+handle_control_del_onion(control_connection_t *conn,
+                         const control_cmd_args_t *cmd_args)
+{
+  int hs_version = 0;
+  smartlist_t *args = cmd_args->args;
+  tor_assert(smartlist_len(args) == 1);
+
+  const char *service_id = smartlist_get(args, 0);
+  if (rend_valid_v2_service_id(service_id)) {
+    hs_version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(service_id)) {
+    hs_version = HS_VERSION_THREE;
+  } else {
+    control_write_endreply(conn, 512, "Malformed Onion Service id");
+    goto out;
+  }
+
+  /* Determine if the onion service belongs to this particular control
+   * connection, or if it is in the global list of detached services.  If it
+   * is in neither, either the service ID is invalid in some way, or it
+   * explicitly belongs to a different control connection, and an error
+   * should be returned.
+   */
+  smartlist_t *services[2] = {
+    conn->ephemeral_onion_services,
+    detached_onion_services
+  };
+  smartlist_t *onion_services = NULL;
+  int idx = -1;
+  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
+    idx = smartlist_string_pos(services[i], service_id);
+    if (idx != -1) {
+      onion_services = services[i];
+      break;
+    }
+  }
+  if (onion_services == NULL) {
+    control_write_endreply(conn, 552, "Unknown Onion Service id");
+  } else {
+    int ret = -1;
+    switch (hs_version) {
+    case HS_VERSION_TWO:
+      ret = rend_service_del_ephemeral(service_id);
+      break;
+    case HS_VERSION_THREE:
+      ret = hs_service_del_ephemeral(service_id);
+      break;
+    default:
+      /* The ret value will be -1 thus hitting the warning below. This should
+       * never happen because of the check at the start of the function. */
+      break;
+    }
+    if (ret < 0) {
+      /* This should *NEVER* fail, since the service is on either the
+       * per-control connection list, or the global one.
+       */
+      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
+               escaped(service_id));
+      tor_fragile_assert();
+    }
+
+    /* Remove/scrub the service_id from the appropriate list. */
+    char *cp = smartlist_get(onion_services, idx);
+    smartlist_del(onion_services, idx);
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+
+    send_control_done(conn);
+  }
+
+ out:
+  return 0;
+}
+
+static const control_cmd_syntax_t obsolete_syntax = {
+  .max_args = UINT_MAX
+};
+
+/**
+ * Called when we get an obsolete command: tell the controller that it is
+ * obsolete.
+ */
+static int
+handle_control_obsolete(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  (void)args;
+  char *command = tor_strdup(conn->current_cmd);
+  tor_strupper(command);
+  control_printf_endreply(conn, 511, "%s is obsolete.", command);
+  tor_free(command);
+  return 0;
+}
+
+/**
+ * Function pointer to a handler function for a controller command.
+ **/
+typedef int (*handler_fn_t) (control_connection_t *conn,
+                             const control_cmd_args_t *args);
+
+/**
+ * Definition for a controller command.
+ */
+typedef struct control_cmd_def_t {
+  /**
+   * The name of the command. If the command is multiline, the name must
+   * begin with "+".  This is not case-sensitive. */
+  const char *name;
+  /**
+   * A function to execute the command.
+   */
+  handler_fn_t handler;
+  /**
+   * Zero or more CMD_FL_* flags, or'd together.
+   */
+  unsigned flags;
+  /**
+   * For parsed command: a syntax description.
+   */
+  const control_cmd_syntax_t *syntax;
+} control_cmd_def_t;
+
+/**
+ * Indicates that the command's arguments are sensitive, and should be
+ * memwiped after use.
+ */
+#define CMD_FL_WIPE (1u<<0)
+
+/** Macro: declare a command with a one-line argument, a given set of flags,
+ * and a syntax definition.
+ **/
+#define ONE_LINE(name, flags)                                   \
+  {                                                             \
+    #name,                                                      \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax,                                           \
+   }
+
+/**
+ * Macro: declare a command with a multi-line argument and a given set of
+ * flags.
+ **/
+#define MULTLINE(name, flags)                                   \
+  { "+"#name,                                                   \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax                                            \
+  }
+
+/**
+ * Macro: declare an obsolete command. (Obsolete commands give a different
+ * error than non-existent ones.)
+ **/
+#define OBSOLETE(name)                          \
+  { #name,                                      \
+      handle_control_obsolete,                  \
+      0,                                        \
+      &obsolete_syntax,                         \
+  }
+
+/**
+ * An array defining all the recognized controller commands.
+ **/
+static const control_cmd_def_t CONTROL_COMMANDS[] =
+{
+  ONE_LINE(setconf, 0),
+  ONE_LINE(resetconf, 0),
+  ONE_LINE(getconf, 0),
+  MULTLINE(loadconf, 0),
+  ONE_LINE(setevents, 0),
+  ONE_LINE(authenticate, CMD_FL_WIPE),
+  ONE_LINE(saveconf, 0),
+  ONE_LINE(signal, 0),
+  ONE_LINE(takeownership, 0),
+  ONE_LINE(dropownership, 0),
+  ONE_LINE(mapaddress, 0),
+  ONE_LINE(getinfo, 0),
+  ONE_LINE(extendcircuit, 0),
+  ONE_LINE(setcircuitpurpose, 0),
+  OBSOLETE(setrouterpurpose),
+  ONE_LINE(attachstream, 0),
+  MULTLINE(postdescriptor, 0),
+  ONE_LINE(redirectstream, 0),
+  ONE_LINE(closestream, 0),
+  ONE_LINE(closecircuit, 0),
+  ONE_LINE(usefeature, 0),
+  ONE_LINE(resolve, 0),
+  ONE_LINE(protocolinfo, 0),
+  ONE_LINE(authchallenge, CMD_FL_WIPE),
+  ONE_LINE(dropguards, 0),
+  ONE_LINE(hsfetch, 0),
+  MULTLINE(hspost, 0),
+  ONE_LINE(add_onion, CMD_FL_WIPE),
+  ONE_LINE(del_onion, CMD_FL_WIPE),
+};
+
+/**
+ * The number of entries in CONTROL_COMMANDS.
+ **/
+static const size_t N_CONTROL_COMMANDS = ARRAY_LENGTH(CONTROL_COMMANDS);
+
+/**
+ * Run a single control command, as defined by a control_cmd_def_t,
+ * with a given set of arguments.
+ */
+static int
+handle_single_control_command(const control_cmd_def_t *def,
+                              control_connection_t *conn,
+                              uint32_t cmd_data_len,
+                              char *args)
+{
+  int rv = 0;
+
+  control_cmd_args_t *parsed_args;
+  char *err=NULL;
+  tor_assert(def->syntax);
+  parsed_args = control_cmd_parse_args(conn->current_cmd,
+                                       def->syntax,
+                                       cmd_data_len, args,
+                                       &err);
+  if (!parsed_args) {
+    control_printf_endreply(conn, 512, "Bad arguments to %s: %s",
+                            conn->current_cmd, err?err:"");
+    tor_free(err);
+  } else {
+    if (BUG(err))
+      tor_free(err);
+    if (def->handler(conn, parsed_args))
+      rv = 0;
+
+    if (def->flags & CMD_FL_WIPE)
+      control_cmd_args_wipe(parsed_args);
+
+    control_cmd_args_free(parsed_args);
+  }
+
+  if (def->flags & CMD_FL_WIPE)
+    memwipe(args, 0, cmd_data_len);
+
+  return rv;
+}
+
+/**
+ * Run a given controller command, as selected by the current_cmd field of
+ * <b>conn</b>.
+ */
+int
+handle_control_command(control_connection_t *conn,
+                       uint32_t cmd_data_len,
+                       char *args)
+{
+  tor_assert(conn);
+  tor_assert(args);
+  tor_assert(args[cmd_data_len] == '\0');
+
+  for (unsigned i = 0; i < N_CONTROL_COMMANDS; ++i) {
+    const control_cmd_def_t *def = &CONTROL_COMMANDS[i];
+    if (!strcasecmp(conn->current_cmd, def->name)) {
+      return handle_single_control_command(def, conn, cmd_data_len, args);
+    }
+  }
+
+  control_printf_endreply(conn, 510, "Unrecognized command \"%s\"",
+                          conn->current_cmd);
+
+  return 0;
+}
+
+void
+control_cmd_free_all(void)
+{
+  if (detached_onion_services) { /* Free the detached onion services */
+    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
+    smartlist_free(detached_onion_services);
+  }
+}
diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h
new file mode 100644
index 0000000000..4b6d54abe7
--- /dev/null
+++ b/src/feature/control/control_cmd.h
@@ -0,0 +1,113 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.h
+ * \brief Header file for control_cmd.c.
+ **/
+
+#ifndef TOR_CONTROL_CMD_H
+#define TOR_CONTROL_CMD_H
+
+#include "lib/malloc/malloc.h"
+
+int handle_control_command(control_connection_t *conn,
+                           uint32_t cmd_data_len,
+                           char *args);
+void control_cmd_free_all(void);
+
+typedef struct control_cmd_args_t control_cmd_args_t;
+void control_cmd_args_free_(control_cmd_args_t *args);
+void control_cmd_args_wipe(control_cmd_args_t *args);
+
+#define control_cmd_args_free(v) \
+  FREE_AND_NULL(control_cmd_args_t, control_cmd_args_free_, (v))
+
+/**
+ * Definition for the syntax of a controller command, as parsed by
+ * control_cmd_parse_args.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+typedef struct control_cmd_syntax_t {
+  /**
+   * Lowest number of positional arguments that this command accepts.
+   * 0 for "it's okay not to have positional arguments."
+   **/
+  unsigned int min_args;
+  /**
+   * Highest number of positional arguments that this command accepts.
+   * UINT_MAX for no limit.
+   **/
+  unsigned int max_args;
+  /**
+   * If true, we should parse options after the positional arguments
+   * as a set of unordered flags and key=value arguments.
+   *
+   * Requires that max_args is not UINT_MAX.
+   **/
+  bool accept_keywords;
+  /**
+   * If accept_keywords is true, then only the keywords listed in this
+   * (NULL-terminated) array are valid keywords for this command.
+   **/
+  const char **allowed_keywords;
+  /**
+   * If accept_keywords is true, this option is passed to kvline_parse() as
+   * its flags.
+   **/
+  unsigned kvline_flags;
+  /**
+   * True iff this command wants to be followed by a multiline object.
+   **/
+  bool want_cmddata;
+  /**
+   * True iff this command needs access to the raw body of the input.
+   *
+   * This should not be needed for pure commands; it is purely a legacy
+   * option.
+   **/
+  bool store_raw_body;
+} control_cmd_syntax_t;
+
+#ifdef CONTROL_CMD_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/* ADD_ONION secret key to create an ephemeral service. The command supports
+ * multiple versions so this union stores the key and passes it to the HS
+ * subsystem depending on the requested version. */
+typedef union add_onion_secret_key_t {
+  /* Hidden service v2 secret key. */
+  crypto_pk_t *v2;
+  /* Hidden service v3 secret key. */
+  ed25519_secret_key_t *v3;
+} add_onion_secret_key_t;
+
+STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
+                                   const char **key_new_alg_out,
+                                   char **key_new_blob_out,
+                                   add_onion_secret_key_t *decoded_key,
+                                   int *hs_version,
+                                   control_connection_t *conn);
+
+STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg,
+                                   int *created, control_connection_t *conn);
+
+STATIC control_cmd_args_t *control_cmd_parse_args(
+                                   const char *command,
+                                   const control_cmd_syntax_t *syntax,
+                                   size_t body_len,
+                                   const char *body,
+                                   char **error_out);
+
+#endif /* defined(CONTROL_CMD_PRIVATE) */
+
+#ifdef CONTROL_MODULE_PRIVATE
+smartlist_t * get_detached_onion_services(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_CMD_H) */
diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h
new file mode 100644
index 0000000000..8d7a4f55b3
--- /dev/null
+++ b/src/feature/control/control_cmd_args_st.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd_args_st.h
+ * \brief Definition for control_cmd_args_t
+ **/
+
+#ifndef TOR_CONTROL_CMD_ST_H
+#define TOR_CONTROL_CMD_ST_H
+
+struct smartlist_t;
+struct config_line_t;
+
+/**
+ * Parsed arguments for a control command.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+struct control_cmd_args_t {
+  /**
+   * The command itself, as provided by the controller.  Not owned by this
+   * structure.
+   **/
+  const char *command;
+  /**
+   * Positional arguments to the command.
+   **/
+  struct smartlist_t *args;
+  /**
+   * Keyword arguments to the command.
+   **/
+  struct config_line_t *kwargs;
+  /**
+   * Number of bytes in <b>cmddata</b>; 0 if <b>cmddata</b> is not set.
+   **/
+  size_t cmddata_len;
+  /**
+   * A multiline object passed with this command.
+   **/
+  char *cmddata;
+  /**
+   * If set, a nul-terminated string containing the raw unparsed arguments.
+   **/
+  const char *raw_body;
+};
+
+#endif /* !defined(TOR_CONTROL_CMD_ST_H) */
diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h
index 177a916257..c9164f03b3 100644
--- a/src/feature/control/control_connection_st.h
+++ b/src/feature/control/control_connection_st.h
@@ -40,7 +40,8 @@ struct control_connection_t {
   /** A control command that we're reading from the inbuf, but which has not
    * yet arrived completely. */
   char *incoming_cmd;
+  /** The control command that we are currently processing. */
+  char *current_cmd;
 };
 
-#endif
-
+#endif /* !defined(CONTROL_CONNECTION_ST_H) */
diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c
new file mode 100644
index 0000000000..8cf6d6de0b
--- /dev/null
+++ b/src/feature/control/control_events.c
@@ -0,0 +1,2306 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.c
+ * \brief Implement the event-reporting part of the controller API.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/command.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/reasons.h"
+#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/control/control_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "lib/evloop/compat_libevent.h"
+
+static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
+static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/** An event mask of all the events that any controller is interested in
+ * receiving. */
+static event_mask_t global_event_mask = 0;
+
+/** True iff we have disabled log messages from being sent to the controller */
+static int disable_log_messages = 0;
+
+/** Macro: true if any control connection is interested in events of type
+ * <b>e</b>. */
+#define EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & EVENT_MASK_(e)))
+
+/** Macro: true if any event from the bitfield 'e' is interesting. */
+#define ANY_EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & (e)))
+
+static void send_control_event_impl(uint16_t event,
+                                    const char *format, va_list ap)
+  CHECK_PRINTF(2,0);
+static int control_event_status(int type, int severity, const char *format,
+                                va_list args)
+  CHECK_PRINTF(3,0);
+
+static void send_control_event(uint16_t event,
+                               const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+/** Table mapping event values to their names.  Used to implement SETEVENTS
+ * and GETINFO events/names, and to keep they in sync. */
+const struct control_event_t control_event_table[] = {
+  { EVENT_CIRCUIT_STATUS, "CIRC" },
+  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
+  { EVENT_STREAM_STATUS, "STREAM" },
+  { EVENT_OR_CONN_STATUS, "ORCONN" },
+  { EVENT_BANDWIDTH_USED, "BW" },
+  { EVENT_DEBUG_MSG, "DEBUG" },
+  { EVENT_INFO_MSG, "INFO" },
+  { EVENT_NOTICE_MSG, "NOTICE" },
+  { EVENT_WARN_MSG, "WARN" },
+  { EVENT_ERR_MSG, "ERR" },
+  { EVENT_NEW_DESC, "NEWDESC" },
+  { EVENT_ADDRMAP, "ADDRMAP" },
+  { EVENT_DESCCHANGED, "DESCCHANGED" },
+  { EVENT_NS, "NS" },
+  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
+  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
+  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
+  { EVENT_GUARD, "GUARD" },
+  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
+  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
+  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
+  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
+  { EVENT_GOT_SIGNAL, "SIGNAL" },
+  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
+  { EVENT_CONN_BW, "CONN_BW" },
+  { EVENT_CELL_STATS, "CELL_STATS" },
+  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
+  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
+  { EVENT_HS_DESC, "HS_DESC" },
+  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
+  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
+  { 0, NULL },
+};
+
+/** Given a log severity, return the corresponding control event code. */
+static inline int
+log_severity_to_event(int severity)
+{
+  switch (severity) {
+    case LOG_DEBUG: return EVENT_DEBUG_MSG;
+    case LOG_INFO: return EVENT_INFO_MSG;
+    case LOG_NOTICE: return EVENT_NOTICE_MSG;
+    case LOG_WARN: return EVENT_WARN_MSG;
+    case LOG_ERR: return EVENT_ERR_MSG;
+    default: return -1;
+  }
+}
+
+/** Helper: clear bandwidth counters of all origin circuits. */
+static void
+clear_circ_bw_fields(void)
+{
+  origin_circuit_t *ocirc;
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+  }
+  SMARTLIST_FOREACH_END(circ);
+}
+
+/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
+ * connection's event_mask field. */
+void
+control_update_global_event_mask(void)
+{
+  smartlist_t *conns = get_connection_array();
+  event_mask_t old_mask, new_mask;
+  old_mask = global_event_mask;
+  int any_old_per_sec_events = control_any_per_second_event_enabled();
+
+  global_event_mask = 0;
+  SMARTLIST_FOREACH(conns, connection_t *, _conn,
+  {
+    if (_conn->type == CONN_TYPE_CONTROL &&
+        STATE_IS_OPEN(_conn->state)) {
+      control_connection_t *conn = TO_CONTROL_CONN(_conn);
+      global_event_mask |= conn->event_mask;
+    }
+  });
+
+  new_mask = global_event_mask;
+
+  /* Handle the aftermath.  Set up the log callback to tell us only what
+   * we want to hear...*/
+  control_adjust_event_log_severity();
+
+  /* Macro: true if ev was false before and is true now. */
+#define NEWLY_ENABLED(ev) \
+  (! (old_mask & (ev)) && (new_mask & (ev)))
+
+  /* ...then, if we've started logging stream or circ bw, clear the
+   * appropriate fields. */
+  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
+    SMARTLIST_FOREACH(conns, connection_t *, conn,
+    {
+      if (conn->type == CONN_TYPE_AP) {
+        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
+        edge_conn->n_written = edge_conn->n_read = 0;
+      }
+    });
+  }
+  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
+    clear_circ_bw_fields();
+  }
+  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
+    uint64_t r, w;
+    control_get_bytes_rw_last_sec(&r, &w);
+  }
+  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
+    rescan_periodic_events(get_options());
+  }
+
+#undef NEWLY_ENABLED
+}
+
+/** Given a control event code for a message event, return the corresponding
+ * log severity. */
+static inline int
+event_to_log_severity(int event)
+{
+  switch (event) {
+    case EVENT_DEBUG_MSG: return LOG_DEBUG;
+    case EVENT_INFO_MSG: return LOG_INFO;
+    case EVENT_NOTICE_MSG: return LOG_NOTICE;
+    case EVENT_WARN_MSG: return LOG_WARN;
+    case EVENT_ERR_MSG: return LOG_ERR;
+    default: return -1;
+  }
+}
+
+/** Adjust the log severities that result in control_event_logmsg being called
+ * to match the severity of log messages that any controllers are interested
+ * in. */
+void
+control_adjust_event_log_severity(void)
+{
+  int i;
+  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
+
+  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      min_log_event = i;
+      break;
+    }
+  }
+  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      max_log_event = i;
+      break;
+    }
+  }
+  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
+    if (min_log_event > EVENT_NOTICE_MSG)
+      min_log_event = EVENT_NOTICE_MSG;
+    if (max_log_event < EVENT_ERR_MSG)
+      max_log_event = EVENT_ERR_MSG;
+  }
+  if (min_log_event <= max_log_event)
+    change_callback_log_severity(event_to_log_severity(min_log_event),
+                                 event_to_log_severity(max_log_event),
+                                 control_event_logmsg);
+  else
+    change_callback_log_severity(LOG_ERR, LOG_ERR,
+                                 control_event_logmsg);
+}
+
+/** Return true iff the event with code <b>c</b> is being sent to any current
+ * control connection.  This is useful if the amount of work needed to prepare
+ * to call the appropriate control_event_...() function is high.
+ */
+int
+control_event_is_interesting(int event)
+{
+  return EVENT_IS_INTERESTING(event);
+}
+
+/** Return true if any event that needs to fire once a second is enabled. */
+int
+control_any_per_second_event_enabled(void)
+{
+  return ANY_EVENT_IS_INTERESTING(
+      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CELL_STATS) |
+      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CONN_BW) |
+      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
+  );
+}
+
+/* The value of 'get_bytes_read()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_read = 0;
+/* The value of 'get_bytes_written()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_written = 0;
+
+/**
+ * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
+ * and written by Tor since the last call to this function.
+ *
+ * Call this only from the main thread.
+ */
+static void
+control_get_bytes_rw_last_sec(uint64_t *n_read,
+                              uint64_t *n_written)
+{
+  const uint64_t stats_n_bytes_read = get_bytes_read();
+  const uint64_t stats_n_bytes_written = get_bytes_written();
+
+  *n_read = stats_n_bytes_read - stats_prev_n_read;
+  *n_written = stats_n_bytes_written - stats_prev_n_written;
+  stats_prev_n_read = stats_n_bytes_read;
+  stats_prev_n_written = stats_n_bytes_written;
+}
+
+/**
+ * Run all the controller events (if any) that are scheduled to trigger once
+ * per second.
+ */
+void
+control_per_second_events(void)
+{
+  if (!control_any_per_second_event_enabled())
+    return;
+
+  uint64_t bytes_read, bytes_written;
+  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
+  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
+
+  control_event_stream_bandwidth_used();
+  control_event_conn_bandwidth_used();
+  control_event_circ_bandwidth_used();
+  control_event_circuit_cell_stats();
+}
+
+/** Represents an event that's queued to be sent to one or more
+ * controllers. */
+typedef struct queued_event_s {
+  uint16_t event;
+  char *msg;
+} queued_event_t;
+
+/** Pointer to int. If this is greater than 0, we don't allow new events to be
+ * queued. */
+static tor_threadlocal_t block_event_queue_flag;
+
+/** Holds a smartlist of queued_event_t objects that may need to be sent
+ * to one or more controllers */
+static smartlist_t *queued_control_events = NULL;
+
+/** True if the flush_queued_events_event is pending. */
+static int flush_queued_event_pending = 0;
+
+/** Lock to protect the above fields. */
+static tor_mutex_t *queued_control_events_lock = NULL;
+
+/** An event that should fire in order to flush the contents of
+ * queued_control_events. */
+static mainloop_event_t *flush_queued_events_event = NULL;
+
+void
+control_initialize_event_queue(void)
+{
+  if (queued_control_events == NULL) {
+    queued_control_events = smartlist_new();
+  }
+
+  if (flush_queued_events_event == NULL) {
+    struct event_base *b = tor_libevent_get_base();
+    if (b) {
+      flush_queued_events_event =
+        mainloop_event_new(flush_queued_events_cb, NULL);
+      tor_assert(flush_queued_events_event);
+    }
+  }
+
+  if (queued_control_events_lock == NULL) {
+    queued_control_events_lock = tor_mutex_new();
+    tor_threadlocal_init(&block_event_queue_flag);
+  }
+}
+
+static int *
+get_block_event_queue(void)
+{
+  int *val = tor_threadlocal_get(&block_event_queue_flag);
+  if (PREDICT_UNLIKELY(val == NULL)) {
+    val = tor_malloc_zero(sizeof(int));
+    tor_threadlocal_set(&block_event_queue_flag, val);
+  }
+  return val;
+}
+
+/** Helper: inserts an event on the list of events queued to be sent to
+ * one or more controllers, and schedules the events to be flushed if needed.
+ *
+ * This function takes ownership of <b>msg</b>, and may free it.
+ *
+ * We queue these events rather than send them immediately in order to break
+ * the dependency in our callgraph from code that generates events for the
+ * controller, and the network layer at large.  Otherwise, nearly every
+ * interesting part of Tor would potentially call every other interesting part
+ * of Tor.
+ */
+MOCK_IMPL(STATIC void,
+queue_control_event_string,(uint16_t event, char *msg))
+{
+  /* This is redundant with checks done elsewhere, but it's a last-ditch
+   * attempt to avoid queueing something we shouldn't have to queue. */
+  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
+    tor_free(msg);
+    return;
+  }
+
+  int *block_event_queue = get_block_event_queue();
+  if (*block_event_queue) {
+    tor_free(msg);
+    return;
+  }
+
+  queued_event_t *ev = tor_malloc(sizeof(*ev));
+  ev->event = event;
+  ev->msg = msg;
+
+  /* No queueing an event while queueing an event */
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  tor_assert(queued_control_events);
+  smartlist_add(queued_control_events, ev);
+
+  int activate_event = 0;
+  if (! flush_queued_event_pending && in_main_thread()) {
+    activate_event = 1;
+    flush_queued_event_pending = 1;
+  }
+
+  tor_mutex_release(queued_control_events_lock);
+
+  --*block_event_queue;
+
+  /* We just put an event on the queue; mark the queue to be
+   * flushed.  We only do this from the main thread for now; otherwise,
+   * we'd need to incur locking overhead in Libevent or use a socket.
+   */
+  if (activate_event) {
+    tor_assert(flush_queued_events_event);
+    mainloop_event_activate(flush_queued_events_event);
+  }
+}
+
+#define queued_event_free(ev) \
+  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
+
+/** Release all storage held by <b>ev</b>. */
+static void
+queued_event_free_(queued_event_t *ev)
+{
+  if (ev == NULL)
+    return;
+
+  tor_free(ev->msg);
+  tor_free(ev);
+}
+
+/** Send every queued event to every controller that's interested in it,
+ * and remove the events from the queue.  If <b>force</b> is true,
+ * then make all controllers send their data out immediately, since we
+ * may be about to shut down. */
+static void
+queued_events_flush_all(int force)
+{
+  /* Make sure that we get all the pending log events, if there are any. */
+  flush_pending_log_callbacks();
+
+  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
+    return;
+  }
+  smartlist_t *all_conns = get_connection_array();
+  smartlist_t *controllers = smartlist_new();
+  smartlist_t *queued_events;
+
+  int *block_event_queue = get_block_event_queue();
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  /* No queueing an event while flushing events. */
+  flush_queued_event_pending = 0;
+  queued_events = queued_control_events;
+  queued_control_events = smartlist_new();
+  tor_mutex_release(queued_control_events_lock);
+
+  /* Gather all the controllers that will care... */
+  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_CONTROL &&
+        !conn->marked_for_close &&
+        conn->state == CONTROL_CONN_STATE_OPEN) {
+      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
+
+      smartlist_add(controllers, control_conn);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
+    const event_mask_t bit = ((event_mask_t)1) << ev->event;
+    const size_t msg_len = strlen(ev->msg);
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      if (control_conn->event_mask & bit) {
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
+      }
+    } SMARTLIST_FOREACH_END(control_conn);
+
+    queued_event_free(ev);
+  } SMARTLIST_FOREACH_END(ev);
+
+  if (force) {
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      connection_flush(TO_CONN(control_conn));
+    } SMARTLIST_FOREACH_END(control_conn);
+  }
+
+  smartlist_free(queued_events);
+  smartlist_free(controllers);
+
+  --*block_event_queue;
+}
+
+/** Libevent callback: Flushes pending events to controllers that are
+ * interested in them. */
+static void
+flush_queued_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void) event;
+  (void) arg;
+  queued_events_flush_all(0);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is given by <b>msg</b>.
+ *
+ * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
+ * respect to the EXTENDED_EVENTS feature. */
+MOCK_IMPL(STATIC void,
+send_control_event_string,(uint16_t event,
+                           const char *msg))
+{
+  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
+  queue_control_event_string(event, tor_strdup(msg));
+}
+
+/** Helper for send_control_event and control_event_status:
+ * Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event_impl(uint16_t event,
+                        const char *format, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, format, ap);
+  if (len < 0) {
+    log_warn(LD_BUG, "Unable to format event for controller.");
+    return;
+  }
+
+  queue_control_event_string(event, buf);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event(uint16_t event,
+                   const char *format, ...)
+{
+  va_list ap;
+  va_start(ap, format);
+  send_control_event_impl(event, format, ap);
+  va_end(ap);
+}
+
+/** Something major has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+int
+control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                             int reason_code)
+{
+  const char *status;
+  char reasons[64] = "";
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
+    return 0;
+  tor_assert(circ);
+
+  switch (tp)
+    {
+    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case CIRC_EVENT_BUILT: status = "BUILT"; break;
+    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
+    case CIRC_EVENT_FAILED: status = "FAILED"; break;
+    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
+    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
+    char unk_reason_buf[16];
+    if (!reason_str) {
+      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
+      reason_str = unk_reason_buf;
+    }
+    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
+    } else {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=%s", reason_str);
+    }
+  }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS,
+                                "650 CIRC %lu %s%s%s%s\r\n",
+                                (unsigned long)circ->global_identifier,
+                                status, sp,
+                                circdesc,
+                                reasons);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/** Something minor has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+static int
+control_event_circuit_status_minor(origin_circuit_t *circ,
+                                   circuit_status_minor_event_t e,
+                                   int purpose, const struct timeval *tv)
+{
+  const char *event_desc;
+  char event_tail[160] = "";
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
+    return 0;
+  tor_assert(circ);
+
+  switch (e)
+    {
+    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
+      event_desc = "PURPOSE_CHANGED";
+
+      {
+        /* event_tail can currently be up to 68 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "");
+      }
+
+      break;
+    case CIRC_MINOR_EVENT_CANNIBALIZED:
+      event_desc = "CANNIBALIZED";
+
+      {
+        /* event_tail can currently be up to 130 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        const struct timeval *old_timestamp_began = tv;
+        char tbuf[ISO_TIME_USEC_LEN+1];
+        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
+
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "",
+                     tbuf);
+      }
+
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
+                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
+                       (unsigned long)circ->global_identifier,
+                       event_desc, sp,
+                       circdesc,
+                       event_tail);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
+ * interested controllers.
+ */
+int
+control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                      int old_purpose)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+                                            old_purpose,
+                                            NULL);
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
+ * created-time from <b>old_tv_created</b>: tell any interested controllers.
+ */
+int
+control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                   int old_purpose,
+                                   const struct timeval *old_tv_created)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_CANNIBALIZED,
+                                            old_purpose,
+                                            old_tv_created);
+}
+
+/** Something has happened to the stream associated with AP connection
+ * <b>conn</b>: tell any interested control connections. */
+int
+control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
+                            int reason_code)
+{
+  char reason_buf[64];
+  char addrport_buf[64];
+  const char *status;
+  circuit_t *circ;
+  origin_circuit_t *origin_circ = NULL;
+  char buf[256];
+  const char *purpose = "";
+  tor_assert(conn->socks_request);
+
+  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
+    return 0;
+
+  if (tp == STREAM_EVENT_CLOSED &&
+      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
+    return 0;
+
+  write_stream_target_to_buf(conn, buf, sizeof(buf));
+
+  reason_buf[0] = '\0';
+  switch (tp)
+    {
+    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
+    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
+    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
+    case STREAM_EVENT_FAILED: status = "FAILED"; break;
+    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
+    case STREAM_EVENT_NEW: status = "NEW"; break;
+    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
+    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
+    case STREAM_EVENT_REMAP: status = "REMAP"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (reason_code && (tp == STREAM_EVENT_FAILED ||
+                      tp == STREAM_EVENT_CLOSED ||
+                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
+    const char *reason_str = stream_end_reason_to_control_string(reason_code);
+    char *r = NULL;
+    if (!reason_str) {
+      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
+      reason_str = r;
+    }
+    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=END REMOTE_REASON=%s", reason_str);
+    else
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=%s", reason_str);
+    tor_free(r);
+  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
+    switch (reason_code) {
+    case REMAP_STREAM_SOURCE_CACHE:
+      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
+      break;
+    case REMAP_STREAM_SOURCE_EXIT:
+      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
+      break;
+    default:
+      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
+                   reason_code);
+      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
+      break;
+    }
+  }
+
+  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
+    /*
+     * When the control conn is an AF_UNIX socket and we have no address,
+     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
+     * dnsserv.c.
+     */
+    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
+      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
+                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
+    } else {
+      /*
+       * else leave it blank so control on AF_UNIX doesn't need to make
+       * something up.
+       */
+      addrport_buf[0] = '\0';
+    }
+  } else {
+    addrport_buf[0] = '\0';
+  }
+
+  if (tp == STREAM_EVENT_NEW_RESOLVE) {
+    purpose = " PURPOSE=DNS_REQUEST";
+  } else if (tp == STREAM_EVENT_NEW) {
+    if (conn->use_begindir) {
+      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
+      int linked_dir_purpose = -1;
+      if (linked && linked->type == CONN_TYPE_DIR)
+        linked_dir_purpose = linked->purpose;
+      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
+        purpose = " PURPOSE=DIR_UPLOAD";
+      else
+        purpose = " PURPOSE=DIR_FETCH";
+    } else
+      purpose = " PURPOSE=USER";
+  }
+
+  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+  if (circ && CIRCUIT_IS_ORIGIN(circ))
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+  send_control_event(EVENT_STREAM_STATUS,
+                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
+                     (ENTRY_TO_CONN(conn)->global_identifier),
+                     status,
+                        origin_circ?
+                           (unsigned long)origin_circ->global_identifier : 0ul,
+                        buf, reason_buf, addrport_buf, purpose);
+
+  /* XXX need to specify its intended exit, etc? */
+
+  return 0;
+}
+
+/** Called when the status of an OR connection <b>conn</b> changes: tell any
+ * interested control connections. <b>tp</b> is the new status for the
+ * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
+ * may be the reason why.
+ */
+int
+control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
+                             int reason)
+{
+  int ncircs = 0;
+  const char *status;
+  char name[128];
+  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
+
+  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
+    return 0;
+
+  switch (tp)
+    {
+    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
+    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
+    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
+    case OR_CONN_EVENT_NEW: status = "NEW"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (conn->chan) {
+    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
+  } else {
+    ncircs = 0;
+  }
+  ncircs += connection_or_get_num_circuits(conn);
+  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
+    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
+  }
+
+  orconn_target_get_name(name, sizeof(name), conn);
+  send_control_event(EVENT_OR_CONN_STATUS,
+                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
+                              name, status,
+                              reason ? " REASON=" : "",
+                              orconn_end_reason_to_control_string(reason),
+                              ncircs_buf,
+                              (conn->base_.global_identifier));
+
+  return 0;
+}
+
+/**
+ * Print out STREAM_BW event for a single conn
+ */
+int
+control_event_stream_bandwidth(edge_connection_t *edge_conn)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    if (!edge_conn->n_read && !edge_conn->n_written)
+      return 0;
+
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
+    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                       (edge_conn->base_.global_identifier),
+                       (unsigned long)edge_conn->n_read,
+                       (unsigned long)edge_conn->n_written,
+                       tbuf);
+
+    edge_conn->n_written = edge_conn->n_read = 0;
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth streams have used. */
+int
+control_event_stream_bandwidth_used(void)
+{
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    smartlist_t *conns = get_connection_array();
+    edge_connection_t *edge_conn;
+    struct timeval now;
+    char tbuf[ISO_TIME_USEC_LEN+1];
+
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
+    {
+        if (conn->type != CONN_TYPE_AP)
+          continue;
+        edge_conn = TO_EDGE_CONN(conn);
+        if (!edge_conn->n_read && !edge_conn->n_written)
+          continue;
+
+        tor_gettimeofday(&now);
+        format_iso_time_nospace_usec(tbuf, &now);
+        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                           (edge_conn->base_.global_identifier),
+                           (unsigned long)edge_conn->n_read,
+                           (unsigned long)edge_conn->n_written,
+                           tbuf);
+
+        edge_conn->n_written = edge_conn->n_read = 0;
+    }
+    SMARTLIST_FOREACH_END(conn);
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control connections
+ * how much bandwidth origin circuits have used. */
+int
+control_event_circ_bandwidth_used(void)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
+  }
+  SMARTLIST_FOREACH_END(circ);
+
+  return 0;
+}
+
+/**
+ * Emit a CIRC_BW event line for a specific circuit.
+ *
+ * This function sets the values it emits to 0, and does not emit
+ * an event if there is no new data to report since the last call.
+ *
+ * Therefore, it may be called at any frequency.
+ */
+int
+control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+
+  tor_assert(ocirc);
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  /* n_read_circ_bw and n_written_circ_bw are always updated
+   * when there is any new cell on a circuit, and set to 0 after
+   * the event, below.
+   *
+   * Therefore, checking them is sufficient to determine if there
+   * is new data to report. */
+  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
+    return 0;
+
+  tor_gettimeofday(&now);
+  format_iso_time_nospace_usec(tbuf, &now);
+  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
+                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
+                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
+                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
+                     ocirc->global_identifier,
+                     (unsigned long)ocirc->n_read_circ_bw,
+                     (unsigned long)ocirc->n_written_circ_bw,
+                     tbuf,
+                     (unsigned long)ocirc->n_delivered_read_circ_bw,
+                     (unsigned long)ocirc->n_overhead_read_circ_bw,
+                     (unsigned long)ocirc->n_delivered_written_circ_bw,
+                     (unsigned long)ocirc->n_overhead_written_circ_bw);
+  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+
+  return 0;
+}
+
+/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
+  * bandwidth counters. */
+int
+control_event_conn_bandwidth(connection_t *conn)
+{
+  const char *conn_type_str;
+  if (!get_options()->TestingEnableConnBwEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
+    return 0;
+  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
+    return 0;
+  switch (conn->type) {
+    case CONN_TYPE_OR:
+      conn_type_str = "OR";
+      break;
+    case CONN_TYPE_DIR:
+      conn_type_str = "DIR";
+      break;
+    case CONN_TYPE_EXIT:
+      conn_type_str = "EXIT";
+      break;
+    default:
+      return 0;
+  }
+  send_control_event(EVENT_CONN_BW,
+                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
+                     "READ=%lu WRITTEN=%lu\r\n",
+                     (conn->global_identifier),
+                     conn_type_str,
+                     (unsigned long)conn->n_read_conn_bw,
+                     (unsigned long)conn->n_written_conn_bw);
+  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth connections have used. */
+int
+control_event_conn_bandwidth_used(void)
+{
+  if (get_options()->TestingEnableConnBwEvent &&
+      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
+    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
+                      control_event_conn_bandwidth(conn));
+  }
+  return 0;
+}
+
+/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
+ * cells, removed cells, and waiting times by cell command and direction.
+ * Store results in <b>cell_stats</b>.  Free cell statistics of the
+ * circuit afterwards. */
+void
+sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
+{
+  memset(cell_stats, 0, sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
+                          const testing_cell_stats_entry_t *, ent) {
+    tor_assert(ent->command <= CELL_COMMAND_MAX_);
+    if (!ent->removed && !ent->exitward) {
+      cell_stats->added_cells_appward[ent->command] += 1;
+    } else if (!ent->removed && ent->exitward) {
+      cell_stats->added_cells_exitward[ent->command] += 1;
+    } else if (!ent->exitward) {
+      cell_stats->removed_cells_appward[ent->command] += 1;
+      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
+    } else {
+      cell_stats->removed_cells_exitward[ent->command] += 1;
+      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+  circuit_clear_testing_cell_stats(circ);
+}
+
+/** Helper: append a cell statistics string to <code>event_parts</code>,
+ * prefixed with <code>key</code>=.  Statistics consist of comma-separated
+ * key:value pairs with lower-case command strings as keys and cell
+ * numbers or total waiting times as values.  A key:value pair is included
+ * if the entry in <code>include_if_non_zero</code> is not zero, but with
+ * the (possibly zero) entry from <code>number_to_include</code>.  Both
+ * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
+ * entry in <code>include_if_non_zero</code> is positive, no string will
+ * be added to <code>event_parts</code>. */
+void
+append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
+                             const uint64_t *include_if_non_zero,
+                             const uint64_t *number_to_include)
+{
+  smartlist_t *key_value_strings = smartlist_new();
+  int i;
+  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
+    if (include_if_non_zero[i] > 0) {
+      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
+                             cell_command_to_string(i),
+                             (number_to_include[i]));
+    }
+  }
+  if (smartlist_len(key_value_strings) > 0) {
+    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
+    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
+    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
+    tor_free(joined);
+  }
+  smartlist_free(key_value_strings);
+}
+
+/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
+ * CELL_STATS event and write result string to <b>event_string</b>. */
+void
+format_cell_stats(char **event_string, circuit_t *circ,
+                  cell_stats_t *cell_stats)
+{
+  smartlist_t *event_parts = smartlist_new();
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "ID=%lu",
+                 (unsigned long)ocirc->global_identifier);
+  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
+                 (unsigned long)or_circ->p_circ_id);
+    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
+                 (or_circ->p_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "InboundAdded",
+                                 cell_stats->added_cells_appward,
+                                 cell_stats->added_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundRemoved",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->removed_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundTime",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->total_time_appward);
+  }
+  if (circ->n_chan) {
+    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
+                     (unsigned long)circ->n_circ_id);
+    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
+                 (circ->n_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "OutboundAdded",
+                                 cell_stats->added_cells_exitward,
+                                 cell_stats->added_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundRemoved",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->removed_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundTime",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->total_time_exitward);
+  }
+  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
+  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
+  smartlist_free(event_parts);
+}
+
+/** A second or more has elapsed: tell any interested control connection
+ * how many cells have been processed for a given circuit. */
+int
+control_event_circuit_cell_stats(void)
+{
+  cell_stats_t *cell_stats;
+  char *event_string;
+  if (!get_options()->TestingEnableCellStatsEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
+    return 0;
+  cell_stats = tor_malloc(sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!circ->testing_cell_stats)
+      continue;
+    sum_up_cell_stats_by_command(circ, cell_stats);
+    format_cell_stats(&event_string, circ, cell_stats);
+    send_control_event(EVENT_CELL_STATS,
+                       "650 CELL_STATS %s\r\n", event_string);
+    tor_free(event_string);
+  }
+  SMARTLIST_FOREACH_END(circ);
+  tor_free(cell_stats);
+  return 0;
+}
+
+/* about 5 minutes worth. */
+#define N_BW_EVENTS_TO_CACHE 300
+/* Index into cached_bw_events to next write. */
+static int next_measurement_idx = 0;
+/* number of entries set in n_measurements */
+static int n_measurements = 0;
+static struct cached_bw_event_s {
+  uint32_t n_read;
+  uint32_t n_written;
+} cached_bw_events[N_BW_EVENTS_TO_CACHE];
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth we used. */
+int
+control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
+{
+  cached_bw_events[next_measurement_idx].n_read = n_read;
+  cached_bw_events[next_measurement_idx].n_written = n_written;
+  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
+    next_measurement_idx = 0;
+  if (n_measurements < N_BW_EVENTS_TO_CACHE)
+    ++n_measurements;
+
+  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
+    send_control_event(EVENT_BANDWIDTH_USED,
+                       "650 BW %lu %lu\r\n",
+                       (unsigned long)n_read,
+                       (unsigned long)n_written);
+  }
+
+  return 0;
+}
+
+char *
+get_bw_samples(void)
+{
+  int i;
+  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
+    % N_BW_EVENTS_TO_CACHE;
+  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+
+  smartlist_t *elements = smartlist_new();
+
+  for (i = 0; i < n_measurements; ++i) {
+    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+    const struct cached_bw_event_s *bwe = &cached_bw_events[idx];
+
+    smartlist_add_asprintf(elements, "%u,%u",
+                           (unsigned)bwe->n_read,
+                           (unsigned)bwe->n_written);
+
+    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/** Called when we are sending a log message to the controllers: suspend
+ * sending further log messages to the controllers until we're done.  Used by
+ * CONN_LOG_PROTECT. */
+void
+disable_control_logging(void)
+{
+  ++disable_log_messages;
+}
+
+/** We're done sending a log message to the controllers: re-enable controller
+ * logging.  Used by CONN_LOG_PROTECT. */
+void
+enable_control_logging(void)
+{
+  if (--disable_log_messages < 0)
+    tor_assert(0);
+}
+
+/** We got a log message: tell any interested control connections. */
+void
+control_event_logmsg(int severity, log_domain_mask_t domain, const char *msg)
+{
+  int event;
+
+  /* Don't even think of trying to add stuff to a buffer from a cpuworker
+   * thread. (See #25987 for plan to fix.) */
+  if (! in_main_thread())
+    return;
+
+  if (disable_log_messages)
+    return;
+
+  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
+      severity <= LOG_NOTICE) {
+    char *esc = esc_for_log(msg);
+    ++disable_log_messages;
+    control_event_general_status(severity, "BUG REASON=%s", esc);
+    --disable_log_messages;
+    tor_free(esc);
+  }
+
+  event = log_severity_to_event(severity);
+  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
+    char *b = NULL;
+    const char *s;
+    if (strchr(msg, '\n')) {
+      char *cp;
+      b = tor_strdup(msg);
+      for (cp = b; *cp; ++cp)
+        if (*cp == '\r' || *cp == '\n')
+          *cp = ' ';
+    }
+    switch (severity) {
+      case LOG_DEBUG: s = "DEBUG"; break;
+      case LOG_INFO: s = "INFO"; break;
+      case LOG_NOTICE: s = "NOTICE"; break;
+      case LOG_WARN: s = "WARN"; break;
+      case LOG_ERR: s = "ERR"; break;
+      default: s = "UnknownLogSeverity"; break;
+    }
+    ++disable_log_messages;
+    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
+    if (severity == LOG_ERR) {
+      /* Force a flush, since we may be about to die horribly */
+      queued_events_flush_all(1);
+    }
+    --disable_log_messages;
+    tor_free(b);
+  }
+}
+
+/**
+ * Logging callback: called when there is a queued pending log callback.
+ */
+void
+control_event_logmsg_pending(void)
+{
+  if (! in_main_thread()) {
+    /* We can't handle this case yet, since we're using a
+     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
+     * use a different mechanism instead: see #25987.
+     **/
+    return;
+  }
+  tor_assert(flush_queued_events_event);
+  mainloop_event_activate(flush_queued_events_event);
+}
+
+/** Called whenever we receive new router descriptors: tell any
+ * interested control connections.  <b>routers</b> is a list of
+ * routerinfo_t's.
+ */
+int
+control_event_descriptors_changed(smartlist_t *routers)
+{
+  char *msg;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
+    return 0;
+
+  {
+    smartlist_t *names = smartlist_new();
+    char *ids;
+    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
+        router_get_verbose_nickname(b, ri);
+        smartlist_add(names, b);
+      });
+    ids = smartlist_join_strings(names, " ", 0, NULL);
+    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
+    send_control_event_string(EVENT_NEW_DESC,  msg);
+    tor_free(ids);
+    tor_free(msg);
+    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
+    smartlist_free(names);
+  }
+  return 0;
+}
+
+/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
+ * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
+ * <b>error</b> is non-NULL, it is an error code describing the failure
+ * mode of the mapping.
+ */
+int
+control_event_address_mapped(const char *from, const char *to, time_t expires,
+                             const char *error, const int cached)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
+    return 0;
+
+  if (expires < 3 || expires == TIME_MAX)
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s NEVER %s%s"
+                                "CACHED=\"%s\"\r\n",
+                                  from, to, error?error:"", error?" ":"",
+                                cached?"YES":"NO");
+  else {
+    char buf[ISO_TIME_LEN+1];
+    char buf2[ISO_TIME_LEN+1];
+    format_local_iso_time(buf,expires);
+    format_iso_time(buf2,expires);
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s \"%s\""
+                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
+                                from, to, buf,
+                                error?error:"", error?" ":"",
+                                buf2, cached?"YES":"NO");
+  }
+
+  return 0;
+}
+/** The network liveness has changed; this is called from circuitstats.c
+ * whenever we receive a cell, or when timeout expires and we assume the
+ * network is down. */
+int
+control_event_network_liveness_update(int liveness)
+{
+  if (liveness > 0) {
+    if (get_cached_network_liveness() <= 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(1);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS UP\r\n");
+    }
+    /* else was already live, no-op */
+  } else {
+    if (get_cached_network_liveness() > 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(0);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS DOWN\r\n");
+    }
+    /* else was already dead, no-op */
+  }
+
+  return 0;
+}
+
+/** Helper function for NS-style events. Constructs and sends an event
+ * of type <b>event</b> with string <b>event_string</b> out of the set of
+ * networkstatuses <b>statuses</b>. Currently it is used for NS events
+ * and NEWCONSENSUS events. */
+static int
+control_event_networkstatus_changed_helper(smartlist_t *statuses,
+                                           uint16_t event,
+                                           const char *event_string)
+{
+  smartlist_t *strs;
+  char *s, *esc = NULL;
+  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
+    return 0;
+
+  strs = smartlist_new();
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
+  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
+    {
+      s = networkstatus_getinfo_helper_single(rs);
+      if (!s) continue;
+      smartlist_add(strs, s);
+    });
+
+  s = smartlist_join_strings(strs, "", 0, NULL);
+  write_escaped_data(s, strlen(s), &esc);
+  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
+  smartlist_free(strs);
+  tor_free(s);
+  send_control_event_string(event,  esc);
+  send_control_event_string(event,
+                            "650 OK\r\n");
+
+  tor_free(esc);
+  return 0;
+}
+
+/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
+ * an NS event to any controller that cares. */
+int
+control_event_networkstatus_changed(smartlist_t *statuses)
+{
+  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
+}
+
+/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
+ * event consisting of an NS-style line for each relay in the consensus. */
+int
+control_event_newconsensus(const networkstatus_t *consensus)
+{
+  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
+    return 0;
+  return control_event_networkstatus_changed_helper(
+           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
+}
+
+/** Called when we compute a new circuitbuildtimeout */
+int
+control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                               const char *args)
+{
+  const char *type_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
+    return 0;
+
+  switch (type) {
+    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
+      type_string = "COMPUTED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESET:
+      type_string = "RESET";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
+      type_string = "SUSPENDED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_DISCARD:
+      type_string = "DISCARD";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESUME:
+      type_string = "RESUME";
+      break;
+    default:
+      type_string = "UNKNOWN";
+      break;
+  }
+
+  send_control_event(EVENT_BUILDTIMEOUT_SET,
+                     "650 BUILDTIMEOUT_SET %s %s\r\n",
+                     type_string, args);
+
+  return 0;
+}
+
+/** Called when a signal has been processed from signal_callback */
+int
+control_event_signal(uintptr_t signal_num)
+{
+  const char *signal_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
+    return 0;
+
+  for (unsigned i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if ((int)signal_num == signal_table[i].sig) {
+      signal_string = signal_table[i].signal_name;
+      break;
+    }
+  }
+
+  if (signal_string == NULL) {
+    log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
+             (unsigned long)signal_num);
+    return -1;
+  }
+
+  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
+                     signal_string);
+  return 0;
+}
+
+/** Called when a single local_routerstatus_t has changed: Sends an NS event
+ * to any controller that cares. */
+int
+control_event_networkstatus_changed_single(const routerstatus_t *rs)
+{
+  smartlist_t *statuses;
+  int r;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NS))
+    return 0;
+
+  statuses = smartlist_new();
+  smartlist_add(statuses, (void*)rs);
+  r = control_event_networkstatus_changed(statuses);
+  smartlist_free(statuses);
+  return r;
+}
+
+/** Our own router descriptor has changed; tell any controllers that care.
+ */
+int
+control_event_my_descriptor_changed(void)
+{
+  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
+  return 0;
+}
+
+/** Helper: sends a status event where <b>type</b> is one of
+ * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
+ * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
+ * string corresponding to <b>args</b>. */
+static int
+control_event_status(int type, int severity, const char *format, va_list args)
+{
+  char *user_buf = NULL;
+  char format_buf[160];
+  const char *status, *sev;
+
+  switch (type) {
+    case EVENT_STATUS_GENERAL:
+      status = "STATUS_GENERAL";
+      break;
+    case EVENT_STATUS_CLIENT:
+      status = "STATUS_CLIENT";
+      break;
+    case EVENT_STATUS_SERVER:
+      status = "STATUS_SERVER";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status type %d", type);
+      return -1;
+  }
+  switch (severity) {
+    case LOG_NOTICE:
+      sev = "NOTICE";
+      break;
+    case LOG_WARN:
+      sev = "WARN";
+      break;
+    case LOG_ERR:
+      sev = "ERR";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
+      return -1;
+  }
+  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
+                   status, sev)<0) {
+    log_warn(LD_BUG, "Format string too long.");
+    return -1;
+  }
+  tor_vasprintf(&user_buf, format, args);
+
+  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
+  tor_free(user_buf);
+  return 0;
+}
+
+#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
+  int r;                                                        \
+  do {                                                          \
+    va_list ap;                                                 \
+    if (!EVENT_IS_INTERESTING(event))                           \
+      return 0;                                                 \
+                                                                \
+    va_start(ap, format);                                       \
+    r = control_event_status((event), (sev), format, ap);       \
+    va_end(ap);                                                 \
+  } while (0)
+
+/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_general_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_general_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_client_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_client_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_server_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_server_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Called when the status of an entry guard with the given <b>nickname</b>
+ * and identity <b>digest</b> has changed to <b>status</b>: tells any
+ * controllers that care. */
+int
+control_event_guard(const char *nickname, const char *digest,
+                    const char *status)
+{
+  char hbuf[HEX_DIGEST_LEN+1];
+  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
+  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
+    return 0;
+
+  {
+    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
+    const node_t *node = node_get_by_id(digest);
+    if (node) {
+      node_get_verbose_nickname(node, buf);
+    } else {
+      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
+    }
+    send_control_event(EVENT_GUARD,
+                       "650 GUARD ENTRY %s %s\r\n", buf, status);
+  }
+  return 0;
+}
+
+/** Called when a configuration option changes. This is generally triggered
+ * by SETCONF requests and RELOAD/SIGHUP signals. The <b>elements</b> is
+ * a smartlist_t containing (key, value, ...) pairs in sequence.
+ * <b>value</b> can be NULL. */
+int
+control_event_conf_changed(const smartlist_t *elements)
+{
+  int i;
+  char *result;
+  smartlist_t *lines;
+  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) ||
+      smartlist_len(elements) == 0) {
+    return 0;
+  }
+  lines = smartlist_new();
+  for (i = 0; i < smartlist_len(elements); i += 2) {
+    char *k = smartlist_get(elements, i);
+    char *v = smartlist_get(elements, i+1);
+    if (v == NULL) {
+      smartlist_add_asprintf(lines, "650-%s", k);
+    } else {
+      smartlist_add_asprintf(lines, "650-%s=%s", k, v);
+    }
+  }
+  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
+  send_control_event(EVENT_CONF_CHANGED,
+    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
+  tor_free(result);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+  return 0;
+}
+
+/** We just generated a new summary of which countries we've seen clients
+ * from recently. Send a copy to the controller in case it wants to
+ * display it for the user. */
+void
+control_event_clients_seen(const char *controller_str)
+{
+  send_control_event(EVENT_CLIENTS_SEEN,
+    "650 CLIENTS_SEEN %s\r\n", controller_str);
+}
+
+/** A new pluggable transport called <b>transport_name</b> was
+ *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
+ *  "server" or "client" depending on the mode of the pluggable
+ *  transport.
+ *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
+ */
+void
+control_event_transport_launched(const char *mode, const char *transport_name,
+                                 tor_addr_t *addr, uint16_t port)
+{
+  send_control_event(EVENT_TRANSPORT_LAUNCHED,
+                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
+                     mode, transport_name, fmt_addr(addr), port);
+}
+
+/** A pluggable transport called <b>pt_name</b> has emitted a log message
+ * found in <b>message</b> at <b>severity</b> log level. */
+void
+control_event_pt_log(const char *log)
+{
+  send_control_event(EVENT_PT_LOG,
+                     "650 PT_LOG %s\r\n",
+                     log);
+}
+
+/** A pluggable transport has emitted a STATUS message found in
+ * <b>status</b>. */
+void
+control_event_pt_status(const char *status)
+{
+  send_control_event(EVENT_PT_STATUS,
+                     "650 PT_STATUS %s\r\n",
+                     status);
+}
+
+/** Convert rendezvous auth type to string for HS_DESC control events
+ */
+const char *
+rend_auth_type_to_string(rend_auth_type_t auth_type)
+{
+  const char *str;
+
+  switch (auth_type) {
+    case REND_NO_AUTH:
+      str = "NO_AUTH";
+      break;
+    case REND_BASIC_AUTH:
+      str = "BASIC_AUTH";
+      break;
+    case REND_STEALTH_AUTH:
+      str = "STEALTH_AUTH";
+      break;
+    default:
+      str = "UNKNOWN";
+  }
+
+  return str;
+}
+
+/** Return either the onion address if the given pointer is a non empty
+ * string else the unknown string. */
+static const char *
+rend_hsaddress_str_or_unknown(const char *onion_address)
+{
+  static const char *str_unknown = "UNKNOWN";
+  const char *str_ret = str_unknown;
+
+  /* No valid pointer, unknown it is. */
+  if (!onion_address) {
+    goto end;
+  }
+  /* Empty onion address thus we don't know, unknown it is. */
+  if (onion_address[0] == '\0') {
+    goto end;
+  }
+  /* All checks are good so return the given onion address. */
+  str_ret = onion_address;
+
+ end:
+  return str_ret;
+}
+
+/** send HS_DESC requested event.
+ *
+ * <b>rend_query</b> is used to fetch requested onion address and auth type.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
+ */
+void
+control_event_hs_descriptor_requested(const char *onion_address,
+                                      rend_auth_type_t auth_type,
+                                      const char *id_digest,
+                                      const char *desc_id,
+                                      const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC CREATED event when a local service generates a descriptor.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>desc_id</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number. If it is negative, it
+ * is ignored.
+ */
+void
+control_event_hs_descriptor_created(const char *onion_address,
+                                    const char *desc_id,
+                                    int replica)
+{
+  char *replica_field = NULL;
+
+  if (BUG(!onion_address || !desc_id)) {
+    return;
+  }
+
+  if (replica >= 0) {
+    tor_asprintf(&replica_field, " REPLICA=%d", replica);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
+                     onion_address, desc_id,
+                     replica_field ? replica_field : "");
+  tor_free(replica_field);
+}
+
+/** send HS_DESC upload event.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id</b> is the ID of requested hs descriptor.
+ */
+void
+control_event_hs_descriptor_upload(const char *onion_address,
+                                   const char *id_digest,
+                                   const char *desc_id,
+                                   const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!onion_address || !id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
+                     onion_address,
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hsv2_descriptor_received
+ * control_event_hsv2_descriptor_failed
+ * control_event_hsv3_descriptor_failed
+ *
+ * So do not call this function directly.
+ */
+static void
+event_hs_descriptor_receive_end(const char *action,
+                                const char *onion_address,
+                                const char *desc_id,
+                                rend_auth_type_t auth_type,
+                                const char *hsdir_id_digest,
+                                const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !onion_address)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s %s %s%s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     desc_id ? desc_id : "",
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hs_descriptor_uploaded
+ * control_event_hs_descriptor_upload_failed
+ *
+ * So do not call this function directly.
+ */
+void
+control_event_hs_descriptor_upload_end(const char *action,
+                                       const char *onion_address,
+                                       const char *id_digest,
+                                       const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !id_digest)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     node_describe_longname_by_id(id_digest),
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** For an HS descriptor query <b>rend_data</b>, using the
+ * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
+ * which descriptor ID in the query is the right one.
+ *
+ * Return a pointer of the binary descriptor ID found in the query's object
+ * or NULL if not found. */
+static const char *
+get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
+{
+  int replica;
+  const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
+
+  /* Possible if the fetch was done using a descriptor ID. This means that
+   * the HSFETCH command was used. */
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
+    goto end;
+  }
+
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
+  /* OK, we have an onion address so now let's find which descriptor ID
+   * is the one associated with the HSDir fingerprint. */
+  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
+       replica++) {
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
+
+    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
+      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
+        /* Found it! This descriptor ID is the right one. */
+        desc_id = digest;
+        goto end;
+      }
+    } SMARTLIST_FOREACH_END(fingerprint);
+  }
+
+ end:
+  return desc_id;
+}
+
+/** send HS_DESC RECEIVED event
+ *
+ * called when we successfully received a hidden service descriptor.
+ */
+void
+control_event_hsv2_descriptor_received(const char *onion_address,
+                                       const rend_data_t *rend_data,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/* Send HS_DESC RECEIVED event
+ *
+ * Called when we successfully received a hidden service descriptor. */
+void
+control_event_hsv3_descriptor_received(const char *onion_address,
+                                       const char *desc_id,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/** send HS_DESC UPLOADED event
+ *
+ * called when we successfully uploaded a hidden service descriptor.
+ */
+void
+control_event_hs_descriptor_uploaded(const char *id_digest,
+                                     const char *onion_address)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
+                                         id_digest, NULL);
+}
+
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
+ */
+void
+control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
+                                  desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC event to inform controller that the query to
+ * <b>onion_address</b> failed to retrieve hidden service descriptor
+ * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
+ * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
+ * field. */
+void
+control_event_hsv3_descriptor_failed(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !reason)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC_CONTENT event after completion of a successful fetch
+ * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
+ * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
+ * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
+ * not trigger the control event. */
+void
+control_event_hs_descriptor_content(const char *onion_address,
+                                    const char *desc_id,
+                                    const char *hsdir_id_digest,
+                                    const char *content)
+{
+  static const char *event_name = "HS_DESC_CONTENT";
+  char *esc_content = NULL;
+
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
+    return;
+  }
+
+  if (content == NULL) {
+    /* Point it to empty content so it can still be escaped. */
+    content = "";
+  }
+  write_escaped_data(content, strlen(content), &esc_content);
+
+  send_control_event(EVENT_HS_DESC_CONTENT,
+                     "650+%s %s %s %s\r\n%s650 OK\r\n",
+                     event_name,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     desc_id,
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     esc_content);
+  tor_free(esc_content);
+}
+
+/** Send HS_DESC event to inform controller upload of hidden service
+ * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
+ * is not NULL, add it to REASON= field.
+ */
+void
+control_event_hs_descriptor_upload_failed(const char *id_digest,
+                                          const char *onion_address,
+                                          const char *reason)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+  control_event_hs_descriptor_upload_end("FAILED", onion_address,
+                                         id_digest, reason);
+}
+
+void
+control_events_free_all(void)
+{
+  smartlist_t *queued_events = NULL;
+
+  stats_prev_n_read = stats_prev_n_written = 0;
+
+  if (queued_control_events_lock) {
+    tor_mutex_acquire(queued_control_events_lock);
+    flush_queued_event_pending = 0;
+    queued_events = queued_control_events;
+    queued_control_events = NULL;
+    tor_mutex_release(queued_control_events_lock);
+  }
+  if (queued_events) {
+    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
+                      queued_event_free(ev));
+    smartlist_free(queued_events);
+  }
+  if (flush_queued_events_event) {
+    mainloop_event_free(flush_queued_events_event);
+    flush_queued_events_event = NULL;
+  }
+  global_event_mask = 0;
+  disable_log_messages = 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* For testing: change the value of global_event_mask */
+void
+control_testing_set_global_event_mask(uint64_t mask)
+{
+  global_event_mask = mask;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control_events.h b/src/feature/control/control_events.h
new file mode 100644
index 0000000000..34986fdb89
--- /dev/null
+++ b/src/feature/control/control_events.h
@@ -0,0 +1,352 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.h
+ * \brief Header file for control_events.c.
+ **/
+
+#ifndef TOR_CONTROL_EVENTS_H
+#define TOR_CONTROL_EVENTS_H
+
+#include "core/or/ocirc_event.h"
+
+/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
+ * The various types are defined in control-spec.txt . */
+typedef enum circuit_status_minor_event_t {
+  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+  CIRC_MINOR_EVENT_CANNIBALIZED,
+} circuit_status_minor_event_t;
+
+#include "core/or/orconn_event.h"
+
+/** Used to indicate the type of a stream event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum stream_status_event_t {
+  STREAM_EVENT_SENT_CONNECT = 0,
+  STREAM_EVENT_SENT_RESOLVE = 1,
+  STREAM_EVENT_SUCCEEDED    = 2,
+  STREAM_EVENT_FAILED       = 3,
+  STREAM_EVENT_CLOSED       = 4,
+  STREAM_EVENT_NEW          = 5,
+  STREAM_EVENT_NEW_RESOLVE  = 6,
+  STREAM_EVENT_FAILED_RETRIABLE = 7,
+  STREAM_EVENT_REMAP        = 8
+} stream_status_event_t;
+
+/** Used to indicate the type of a buildtime event */
+typedef enum buildtimeout_set_event_t {
+  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
+  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
+  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
+  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
+  BUILDTIMEOUT_SET_EVENT_RESUME = 4
+} buildtimeout_set_event_t;
+
+/** Enum describing various stages of bootstrapping, for use with controller
+ * bootstrap status events. The values range from 0 to 100. */
+typedef enum {
+  BOOTSTRAP_STATUS_UNDEF=-1,
+  BOOTSTRAP_STATUS_STARTING=0,
+
+  /* Initial connection to any relay */
+
+  BOOTSTRAP_STATUS_CONN_PT=1,
+  BOOTSTRAP_STATUS_CONN_DONE_PT=2,
+  BOOTSTRAP_STATUS_CONN_PROXY=3,
+  BOOTSTRAP_STATUS_CONN_DONE_PROXY=4,
+  BOOTSTRAP_STATUS_CONN=5,
+  BOOTSTRAP_STATUS_CONN_DONE=10,
+  BOOTSTRAP_STATUS_HANDSHAKE=14,
+  BOOTSTRAP_STATUS_HANDSHAKE_DONE=15,
+
+  /* Loading directory info */
+
+  BOOTSTRAP_STATUS_ONEHOP_CREATE=20,
+  BOOTSTRAP_STATUS_REQUESTING_STATUS=25,
+  BOOTSTRAP_STATUS_LOADING_STATUS=30,
+  BOOTSTRAP_STATUS_LOADING_KEYS=40,
+  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
+  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
+  BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75,
+
+  /* Connecting to a relay for AP circuits */
+
+  BOOTSTRAP_STATUS_AP_CONN_PT=76,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77,
+  BOOTSTRAP_STATUS_AP_CONN_PROXY=78,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79,
+  BOOTSTRAP_STATUS_AP_CONN=80,
+  BOOTSTRAP_STATUS_AP_CONN_DONE=85,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE=89,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90,
+
+  /* Creating AP circuits */
+
+  BOOTSTRAP_STATUS_CIRCUIT_CREATE=95,
+  BOOTSTRAP_STATUS_DONE=100
+} bootstrap_status_t;
+
+/** Reason for remapping an AP connection's address: we have a cached
+ * answer. */
+#define REMAP_STREAM_SOURCE_CACHE 1
+/** Reason for remapping an AP connection's address: the exit node told us an
+ * answer. */
+#define REMAP_STREAM_SOURCE_EXIT 2
+
+void control_initialize_event_queue(void);
+
+void control_update_global_event_mask(void);
+void control_adjust_event_log_severity(void);
+
+#define EVENT_NS 0x000F
+int control_event_is_interesting(int event);
+
+void control_per_second_events(void);
+int control_any_per_second_event_enabled(void);
+
+int control_event_circuit_status(origin_circuit_t *circ,
+                                 circuit_status_event_t e, int reason);
+int control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                          int old_purpose);
+int control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                       int old_purpose,
+                                       const struct timeval *old_tv_created);
+int control_event_stream_status(entry_connection_t *conn,
+                                stream_status_event_t e,
+                                int reason);
+int control_event_or_conn_status(or_connection_t *conn,
+                                 or_conn_status_event_t e, int reason);
+int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
+int control_event_stream_bandwidth(edge_connection_t *edge_conn);
+int control_event_stream_bandwidth_used(void);
+int control_event_circ_bandwidth_used(void);
+int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
+int control_event_conn_bandwidth(connection_t *conn);
+int control_event_conn_bandwidth_used(void);
+int control_event_circuit_cell_stats(void);
+void control_event_logmsg(int severity, log_domain_mask_t domain,
+                          const char *msg);
+void control_event_logmsg_pending(void);
+int control_event_descriptors_changed(smartlist_t *routers);
+int control_event_address_mapped(const char *from, const char *to,
+                                 time_t expires, const char *error,
+                                 const int cached);
+int control_event_my_descriptor_changed(void);
+int control_event_network_liveness_update(int liveness);
+int control_event_networkstatus_changed(smartlist_t *statuses);
+
+int control_event_newconsensus(const networkstatus_t *consensus);
+int control_event_networkstatus_changed_single(const routerstatus_t *rs);
+int control_event_general_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_client_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_server_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+int control_event_general_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_client_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_server_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+
+int control_event_guard(const char *nickname, const char *digest,
+                        const char *status);
+int control_event_conf_changed(const smartlist_t *elements);
+int control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                                   const char *args);
+int control_event_signal(uintptr_t signal);
+
+void control_event_bootstrap(bootstrap_status_t status, int progress);
+MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
+                                                 int reason,
+                                                 or_connection_t *or_conn));
+void control_event_boot_dir(bootstrap_status_t status, int progress);
+void control_event_boot_first_orconn(void);
+void control_event_bootstrap_problem(const char *warn, const char *reason,
+                                     const connection_t *conn, int dowarn);
+char *control_event_boot_last_msg(void);
+void control_event_bootstrap_reset(void);
+
+void control_event_clients_seen(const char *controller_str);
+void control_event_transport_launched(const char *mode,
+                                      const char *transport_name,
+                                      tor_addr_t *addr, uint16_t port);
+void control_event_pt_log(const char *log);
+void control_event_pt_status(const char *status);
+
+void control_event_hs_descriptor_requested(const char *onion_address,
+                                           rend_auth_type_t auth_type,
+                                           const char *id_digest,
+                                           const char *desc_id,
+                                           const char *hsdir_index);
+void control_event_hs_descriptor_created(const char *onion_address,
+                                         const char *desc_id,
+                                         int replica);
+void control_event_hs_descriptor_upload(const char *onion_address,
+                                        const char *desc_id,
+                                        const char *hs_dir,
+                                        const char *hsdir_index);
+void control_event_hs_descriptor_upload_end(const char *action,
+                                            const char *onion_address,
+                                            const char *hs_dir,
+                                            const char *reason);
+void control_event_hs_descriptor_uploaded(const char *hs_dir,
+                                          const char *onion_address);
+/* Hidden service v2 HS_DESC specific. */
+void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                          const char *id_digest,
+                                          const char *reason);
+void control_event_hsv2_descriptor_received(const char *onion_address,
+                                            const rend_data_t *rend_data,
+                                            const char *id_digest);
+/* Hidden service v3 HS_DESC specific. */
+void control_event_hsv3_descriptor_failed(const char *onion_address,
+                                          const char *desc_id,
+                                          const char *hsdir_id_digest,
+                                          const char *reason);
+void control_event_hsv3_descriptor_received(const char *onion_address,
+                                            const char *desc_id,
+                                            const char *hsdir_id_digest);
+void control_event_hs_descriptor_upload_failed(const char *hs_dir,
+                                               const char *onion_address,
+                                               const char *reason);
+void control_event_hs_descriptor_content(const char *onion_address,
+                                         const char *desc_id,
+                                         const char *hsdir_fp,
+                                         const char *content);
+
+void control_events_free_all(void);
+
+#ifdef CONTROL_MODULE_PRIVATE
+char *get_bw_samples(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#ifdef CONTROL_EVENTS_PRIVATE
+/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
+ * connection is interested in events of type <b>e</b>.  We use this
+ * so that we can decide to skip generating event messages that nobody
+ * has interest in without having to walk over the global connection
+ * list to find out.
+ **/
+typedef uint64_t event_mask_t;
+
+/* Recognized asynchronous event types.  It's okay to expand this list
+ * because it is used both as a list of v0 event types, and as indices
+ * into the bitfield to determine which controllers want which events.
+ */
+/* This bitfield has no event zero    0x0000 */
+#define EVENT_MIN_                    0x0001
+#define EVENT_CIRCUIT_STATUS          0x0001
+#define EVENT_STREAM_STATUS           0x0002
+#define EVENT_OR_CONN_STATUS          0x0003
+#define EVENT_BANDWIDTH_USED          0x0004
+#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
+#define EVENT_NEW_DESC                0x0006
+#define EVENT_DEBUG_MSG               0x0007
+#define EVENT_INFO_MSG                0x0008
+#define EVENT_NOTICE_MSG              0x0009
+#define EVENT_WARN_MSG                0x000A
+#define EVENT_ERR_MSG                 0x000B
+#define EVENT_ADDRMAP                 0x000C
+/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
+   can reclaim 0x000D. */
+#define EVENT_DESCCHANGED             0x000E
+/* Exposed above */
+// #define EVENT_NS                   0x000F
+#define EVENT_STATUS_CLIENT           0x0010
+#define EVENT_STATUS_SERVER           0x0011
+#define EVENT_STATUS_GENERAL          0x0012
+#define EVENT_GUARD                   0x0013
+#define EVENT_STREAM_BANDWIDTH_USED   0x0014
+#define EVENT_CLIENTS_SEEN            0x0015
+#define EVENT_NEWCONSENSUS            0x0016
+#define EVENT_BUILDTIMEOUT_SET        0x0017
+#define EVENT_GOT_SIGNAL              0x0018
+#define EVENT_CONF_CHANGED            0x0019
+#define EVENT_CONN_BW                 0x001A
+#define EVENT_CELL_STATS              0x001B
+/* UNUSED :                           0x001C */
+#define EVENT_CIRC_BANDWIDTH_USED     0x001D
+#define EVENT_TRANSPORT_LAUNCHED      0x0020
+#define EVENT_HS_DESC                 0x0021
+#define EVENT_HS_DESC_CONTENT         0x0022
+#define EVENT_NETWORK_LIVENESS        0x0023
+#define EVENT_PT_LOG                  0x0024
+#define EVENT_PT_STATUS               0x0025
+#define EVENT_MAX_                    0x0025
+
+/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
+#define EVENT_CAPACITY_               0x0040
+
+/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
+ * different structure, as it can only handle a maximum left shift of 1<<63. */
+
+#if EVENT_MAX_ >= EVENT_CAPACITY_
+#error control_connection_t.event_mask has an event greater than its capacity
+#endif
+
+#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
+
+#define EVENT_MASK_NONE_             ((uint64_t)0x0)
+
+#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
+#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
+                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
+                                          - EVENT_MIN_))
+
+#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
+                                      & EVENT_MASK_BELOW_MAX_)
+
+/** Helper structure: temporarily stores cell statistics for a circuit. */
+typedef struct cell_stats_t {
+  /** Number of cells added in app-ward direction by command. */
+  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells added in exit-ward direction by command. */
+  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in app-ward direction by command. */
+  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in exit-ward direction by command. */
+  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in app-ward direction by command. */
+  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in exit-ward direction by command. */
+  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
+} cell_stats_t;
+
+void sum_up_cell_stats_by_command(circuit_t *circ,
+                                  cell_stats_t *cell_stats);
+void append_cell_stats_by_command(smartlist_t *event_parts,
+                                  const char *key,
+                                  const uint64_t *include_if_non_zero,
+                                  const uint64_t *number_to_include);
+void format_cell_stats(char **event_string, circuit_t *circ,
+                       cell_stats_t *cell_stats);
+
+/** Helper structure: maps event values to their names. */
+struct control_event_t {
+  uint16_t event_code;
+  const char *event_name;
+};
+
+extern const struct control_event_t control_event_table[];
+
+#ifdef TOR_UNIT_TESTS
+MOCK_DECL(STATIC void,
+          send_control_event_string,(uint16_t event, const char *msg));
+
+MOCK_DECL(STATIC void,
+          queue_control_event_string,(uint16_t event, char *msg));
+
+void control_testing_set_global_event_mask(uint64_t mask);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(CONTROL_EVENTS_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_EVENTS_H) */
diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c
new file mode 100644
index 0000000000..e0e77eb2d0
--- /dev/null
+++ b/src/feature/control/control_fmt.c
@@ -0,0 +1,181 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+
+/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
+ * <b>buf</b>, determine the address:port combination requested on
+ * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
+ * failure. */
+int
+write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
+{
+  char buf2[256];
+  if (conn->chosen_exit_name)
+    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
+      return -1;
+  if (!conn->socks_request)
+    return -1;
+  if (tor_snprintf(buf, len, "%s%s%s:%d",
+               conn->socks_request->address,
+               conn->chosen_exit_name ? buf2 : "",
+               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
+                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
+               conn->socks_request->port)<0)
+    return -1;
+  return 0;
+}
+
+/** Figure out the best name for the target router of an OR connection
+ * <b>conn</b>, and write it into the <b>len</b>-character buffer
+ * <b>name</b>. */
+void
+orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
+{
+  const node_t *node = node_get_by_id(conn->identity_digest);
+  if (node) {
+    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
+    node_get_verbose_nickname(node, name);
+  } else if (! tor_digest_is_zero(conn->identity_digest)) {
+    name[0] = '$';
+    base16_encode(name+1, len-1, conn->identity_digest,
+                  DIGEST_LEN);
+  } else {
+    tor_snprintf(name, len, "%s:%d",
+                 conn->base_.address, conn->base_.port);
+  }
+}
+
+/** Allocate and return a description of <b>circ</b>'s current status,
+ * including its path (if any). */
+char *
+circuit_describe_status_for_controller(origin_circuit_t *circ)
+{
+  char *rv;
+  smartlist_t *descparts = smartlist_new();
+
+  {
+    char *vpath = circuit_list_path_for_controller(circ);
+    if (*vpath) {
+      smartlist_add(descparts, vpath);
+    } else {
+      tor_free(vpath); /* empty path; don't put an extra space in the result */
+    }
+  }
+
+  {
+    cpath_build_state_t *build_state = circ->build_state;
+    smartlist_t *flaglist = smartlist_new();
+    char *flaglist_joined;
+
+    if (build_state->onehop_tunnel)
+      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
+    if (build_state->is_internal)
+      smartlist_add(flaglist, (void *)"IS_INTERNAL");
+    if (build_state->need_capacity)
+      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
+    if (build_state->need_uptime)
+      smartlist_add(flaglist, (void *)"NEED_UPTIME");
+
+    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
+    if (smartlist_len(flaglist)) {
+      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
+
+      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
+
+      tor_free(flaglist_joined);
+    }
+
+    smartlist_free(flaglist);
+  }
+
+  smartlist_add_asprintf(descparts, "PURPOSE=%s",
+                    circuit_purpose_to_controller_string(circ->base_.purpose));
+
+  {
+    const char *hs_state =
+      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
+
+    if (hs_state != NULL) {
+      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
+    }
+  }
+
+  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
+    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    const char *onion_address;
+    if (circ->rend_data) {
+      onion_address = rend_data_get_address(circ->rend_data);
+    } else {
+      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
+      onion_address = addr;
+    }
+    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
+  }
+
+  {
+    char tbuf[ISO_TIME_USEC_LEN+1];
+    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
+
+    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
+  }
+
+  // Show username and/or password if available.
+  if (circ->socks_username_len > 0) {
+    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
+                                     (size_t) circ->socks_username_len);
+    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
+                           socks_username_escaped);
+    tor_free(socks_username_escaped);
+  }
+  if (circ->socks_password_len > 0) {
+    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
+                                     (size_t) circ->socks_password_len);
+    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
+                           socks_password_escaped);
+    tor_free(socks_password_escaped);
+  }
+
+  rv = smartlist_join_strings(descparts, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
+  smartlist_free(descparts);
+
+  return rv;
+}
+
+/** Return a longname the node whose identity is <b>id_digest</b>. If
+ * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
+ * returned instead.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+MOCK_IMPL(const char *,
+node_describe_longname_by_id,(const char *id_digest))
+{
+  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
+  node_get_verbose_nickname_by_id(id_digest, longname);
+  return longname;
+}
diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h
new file mode 100644
index 0000000000..6446e37079
--- /dev/null
+++ b/src/feature/control/control_fmt.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.h
+ * \brief Header file for control_fmt.c.
+ **/
+
+#ifndef TOR_CONTROL_FMT_H
+#define TOR_CONTROL_FMT_H
+
+int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
+                               size_t len);
+void orconn_target_get_name(char *buf, size_t len,
+                            or_connection_t *conn);
+char *circuit_describe_status_for_controller(origin_circuit_t *circ);
+
+MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
+
+#endif /* !defined(TOR_CONTROL_FMT_H) */
diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c
new file mode 100644
index 0000000000..3e31bb9e8f
--- /dev/null
+++ b/src/feature/control/control_getinfo.c
@@ -0,0 +1,1654 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_getinfo.c
+ * \brief Implementation for miscellaneous controller getinfo commands.
+ */
+
+#define CONTROL_EVENTS_PRIVATE
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/policies.h"
+#include "core/or/versions.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/control/fmt_serverstatus.h"
+#include "feature/control/getinfo_geoip.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendcache.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/version/torversion.h"
+
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifndef _WIN32
+#include <pwd.h>
+#endif
+
+static char *list_getinfo_options(void);
+static char *download_status_to_string(const download_status_t *dl);
+
+/** Implementation helper for GETINFO: knows the answers for various
+ * trivial-to-implement questions. */
+static int
+getinfo_helper_misc(control_connection_t *conn, const char *question,
+                    char **answer, const char **errmsg)
+{
+  (void) conn;
+  if (!strcmp(question, "version")) {
+    *answer = tor_strdup(get_version());
+  } else if (!strcmp(question, "bw-event-cache")) {
+    *answer = get_bw_samples();
+  } else if (!strcmp(question, "config-file")) {
+    const char *a = get_torrc_fname(0);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-defaults-file")) {
+    const char *a = get_torrc_fname(1);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-text")) {
+    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
+  } else if (!strcmp(question, "info/names")) {
+    *answer = list_getinfo_options();
+  } else if (!strcmp(question, "dormant")) {
+    int dormant = rep_hist_circbuilding_dormant(time(NULL));
+    *answer = tor_strdup(dormant ? "1" : "0");
+  } else if (!strcmp(question, "events/names")) {
+    int i;
+    smartlist_t *event_names = smartlist_new();
+
+    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+      smartlist_add(event_names, (char *)control_event_table[i].event_name);
+    }
+
+    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
+
+    smartlist_free(event_names);
+  } else if (!strcmp(question, "signal/names")) {
+    smartlist_t *signal_names = smartlist_new();
+    int j;
+    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
+      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
+    }
+
+    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
+
+    smartlist_free(signal_names);
+  } else if (!strcmp(question, "features/names")) {
+    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
+  } else if (!strcmp(question, "address")) {
+    uint32_t addr;
+    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
+      *errmsg = "Address unknown";
+      return -1;
+    }
+    *answer = tor_dup_ip(addr);
+  } else if (!strcmp(question, "traffic/read")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
+  } else if (!strcmp(question, "traffic/written")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
+  } else if (!strcmp(question, "uptime")) {
+    long uptime_secs = get_uptime();
+    tor_asprintf(answer, "%ld", uptime_secs);
+  } else if (!strcmp(question, "process/pid")) {
+    int myPid = -1;
+
+#ifdef _WIN32
+      myPid = _getpid();
+#else
+      myPid = getpid();
+#endif
+
+    tor_asprintf(answer, "%d", myPid);
+  } else if (!strcmp(question, "process/uid")) {
+#ifdef _WIN32
+      *answer = tor_strdup("-1");
+#else
+      int myUid = geteuid();
+      tor_asprintf(answer, "%d", myUid);
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/user")) {
+#ifdef _WIN32
+      *answer = tor_strdup("");
+#else
+      int myUid = geteuid();
+      const struct passwd *myPwEntry = tor_getpwuid(myUid);
+
+      if (myPwEntry) {
+        *answer = tor_strdup(myPwEntry->pw_name);
+      } else {
+        *answer = tor_strdup("");
+      }
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/descriptor-limit")) {
+    int max_fds = get_max_sockets();
+    tor_asprintf(answer, "%d", max_fds);
+  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
+    tor_asprintf(answer, "%"PRIu64,
+                 (get_options()->MaxMemInQueues));
+  } else if (!strcmp(question, "fingerprint")) {
+    crypto_pk_t *server_key;
+    if (!server_mode(get_options())) {
+      *errmsg = "Not running in server mode";
+      return -1;
+    }
+    server_key = get_server_identity_key();
+    *answer = tor_malloc(HEX_DIGEST_LEN+1);
+    crypto_pk_get_fingerprint(server_key, *answer, 0);
+  }
+  return 0;
+}
+
+/** Awful hack: return a newly allocated string based on a routerinfo and
+ * (possibly) an extrainfo, sticking the read-history and write-history from
+ * <b>ei</b> into the resulting string.  The thing you get back won't
+ * necessarily have a valid signature.
+ *
+ * New code should never use this; it's for backward compatibility.
+ *
+ * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
+ * not be NUL-terminated. */
+static char *
+munge_extrainfo_into_routerinfo(const char *ri_body,
+                                const signed_descriptor_t *ri,
+                                const signed_descriptor_t *ei)
+{
+  char *out = NULL, *outp;
+  int i;
+  const char *router_sig;
+  const char *ei_body = signed_descriptor_get_body(ei);
+  size_t ri_len = ri->signed_descriptor_len;
+  size_t ei_len = ei->signed_descriptor_len;
+  if (!ei_body)
+    goto bail;
+
+  outp = out = tor_malloc(ri_len+ei_len+1);
+  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
+    goto bail;
+  ++router_sig;
+  memcpy(out, ri_body, router_sig-ri_body);
+  outp += router_sig-ri_body;
+
+  for (i=0; i < 2; ++i) {
+    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
+    const char *cp, *eol;
+    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
+      continue;
+    ++cp;
+    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
+      continue;
+    memcpy(outp, cp, eol-cp+1);
+    outp += eol-cp+1;
+  }
+  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
+  *outp++ = '\0';
+  tor_assert(outp-out < (int)(ri_len+ei_len+1));
+
+  return out;
+ bail:
+  tor_free(out);
+  return tor_strndup(ri_body, ri->signed_descriptor_len);
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * which ports are bound. */
+static int
+getinfo_helper_listeners(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  int type;
+  smartlist_t *res;
+
+  (void)control_conn;
+  (void)errmsg;
+
+  if (!strcmp(question, "net/listeners/or"))
+    type = CONN_TYPE_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/extor"))
+    type = CONN_TYPE_EXT_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/dir"))
+    type = CONN_TYPE_DIR_LISTENER;
+  else if (!strcmp(question, "net/listeners/socks"))
+    type = CONN_TYPE_AP_LISTENER;
+  else if (!strcmp(question, "net/listeners/trans"))
+    type = CONN_TYPE_AP_TRANS_LISTENER;
+  else if (!strcmp(question, "net/listeners/natd"))
+    type = CONN_TYPE_AP_NATD_LISTENER;
+  else if (!strcmp(question, "net/listeners/httptunnel"))
+    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  else if (!strcmp(question, "net/listeners/dns"))
+    type = CONN_TYPE_AP_DNS_LISTENER;
+  else if (!strcmp(question, "net/listeners/control"))
+    type = CONN_TYPE_CONTROL_LISTENER;
+  else
+    return 0; /* unknown key */
+
+  res = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    struct sockaddr_storage ss;
+    socklen_t ss_len = sizeof(ss);
+
+    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
+      continue;
+
+    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
+      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
+    } else {
+      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
+      smartlist_add(res, esc_for_log(tmp));
+      tor_free(tmp);
+    }
+
+  } SMARTLIST_FOREACH_END(conn);
+
+  *answer = smartlist_join_strings(res, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
+  smartlist_free(res);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * the current time in both local and UTC forms. */
+STATIC int
+getinfo_helper_current_time(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  if (!strcmp(question, "current-time/local"))
+    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else if (!strcmp(question, "current-time/utc"))
+    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else
+    return 0;
+
+  *answer = tor_strdup(timebuf);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * directory information. */
+STATIC int
+getinfo_helper_dir(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  (void) control_conn;
+  if (!strcmpstart(question, "desc/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/id/* "
+                "instead of desc/id/*.";
+      return 0;
+    }
+  } else if (!strcmpstart(question, "desc/name/")) {
+    const routerinfo_t *ri = NULL;
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node =
+      node_get_by_nickname(question+strlen("desc/name/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/name/* "
+                "instead of desc/name/*.";
+      return 0;
+    }
+  } else if (!strcmp(question, "desc/download-enabled")) {
+    int r = we_fetch_router_descriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmp(question, "desc/all-recent")) {
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
+      {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        if (body)
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+      });
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
+    /* XXXX Remove this once Torstat asks for extrainfos. */
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+        if (ei && body) {
+          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
+                                                        &ri->cache_info, ei));
+        } else if (body) {
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+        }
+      } SMARTLIST_FOREACH_END(ri);
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/client/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      rend_cache_entry_t *e = NULL;
+      if (!rend_cache_lookup_entry(question, -1, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      const char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_cache_lookup_encoded_as_client(&service_pk);
+      if (desc) {
+        *answer = tor_strdup(desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/service/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+    rend_cache_entry_t *e = NULL;
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_service_lookup_current_desc(&service_pk);
+      if (desc) {
+        /* Newly allocated string, we have ownership. */
+        *answer = desc;
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "md/all")) {
+    const smartlist_t *nodes = nodelist_get_list();
+    tor_assert(nodes);
+
+    if (smartlist_len(nodes) == 0) {
+      *answer = tor_strdup("");
+      return 0;
+    }
+
+    smartlist_t *microdescs = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
+      if (n->md && n->md->body) {
+        char *copy = tor_strndup(n->md->body, n->md->bodylen);
+        smartlist_add(microdescs, copy);
+      }
+    } SMARTLIST_FOREACH_END(n);
+
+    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
+    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
+    smartlist_free(microdescs);
+  } else if (!strcmpstart(question, "md/id/")) {
+    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmpstart(question, "md/name/")) {
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
+    /* XXXX duplicated code */
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmp(question, "md/download-enabled")) {
+    int r = we_fetch_microdescriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmpstart(question, "desc-annotations/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node =
+      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *annotations =
+        signed_descriptor_get_annotations(&ri->cache_info);
+      if (annotations)
+        *answer = tor_strndup(annotations,
+                              ri->cache_info.annotations_len);
+    }
+  } else if (!strcmpstart(question, "dir/server/")) {
+    size_t answer_len = 0;
+    char *url = NULL;
+    smartlist_t *descs = smartlist_new();
+    const char *msg;
+    int res;
+    char *cp;
+    tor_asprintf(&url, "/tor/%s", question+4);
+    res = dirserv_get_routerdescs(descs, url, &msg);
+    if (res) {
+      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
+      smartlist_free(descs);
+      tor_free(url);
+      *errmsg = msg;
+      return -1;
+    }
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      answer_len += sd->signed_descriptor_len);
+    cp = *answer = tor_malloc(answer_len+1);
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      {
+                        memcpy(cp, signed_descriptor_get_body(sd),
+                               sd->signed_descriptor_len);
+                        cp += sd->signed_descriptor_len;
+                      });
+    *cp = '\0';
+    tor_free(url);
+    smartlist_free(descs);
+  } else if (!strcmpstart(question, "dir/status/")) {
+    *answer = tor_strdup("");
+  } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
+    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
+      const cached_dir_t *consensus = dirserv_get_consensus("ns");
+      if (consensus)
+        *answer = tor_strdup(consensus->dir);
+    }
+    if (!*answer) { /* try loading it from disk */
+      tor_mmap_t *mapped = networkstatus_map_cached_consensus("ns");
+      if (mapped) {
+        *answer = tor_memdup_nulterm(mapped->data, mapped->size);
+        tor_munmap_file(mapped);
+      }
+      if (!*answer) { /* generate an error */
+        *errmsg = "Could not open cached consensus. "
+          "Make sure FetchUselessDescriptors is set to 1.";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
+    routerlist_t *routerlist = router_get_routerlist();
+    if (!routerlist || !routerlist->routers ||
+        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
+      return -1;
+    }
+  } else if (!strcmpstart(question, "extra-info/digest/")) {
+    question += strlen("extra-info/digest/");
+    if (strlen(question) == HEX_DIGEST_LEN) {
+      char d[DIGEST_LEN];
+      signed_descriptor_t *sd = NULL;
+      if (base16_decode(d, sizeof(d), question, strlen(question))
+                        == sizeof(d)) {
+        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
+         * but I don't want to risk affecting other parts of the code,
+         * especially since the rules for using our own extrainfo (including
+         * when it might be freed) are different from those for using one
+         * we have downloaded. */
+        if (router_extrainfo_digest_is_me(d))
+          sd = &(router_get_my_extrainfo()->cache_info);
+        else
+          sd = extrainfo_get_by_descriptor_digest(d);
+      }
+      if (sd) {
+        const char *body = signed_descriptor_get_body(sd);
+        if (body)
+          *answer = tor_strndup(body, sd->signed_descriptor_len);
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** Given a smartlist of 20-byte digests, return a newly allocated string
+ * containing each of those digests in order, formatted in HEX, and terminated
+ * with a newline. */
+static char *
+digest_list_to_string(const smartlist_t *sl)
+{
+  int len;
+  char *result, *s;
+
+  /* Allow for newlines, and a \0 at the end */
+  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
+  result = tor_malloc_zero(len);
+
+  s = result;
+  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
+    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
+    s[HEX_DIGEST_LEN] = '\n';
+    s += HEX_DIGEST_LEN + 1;
+  } SMARTLIST_FOREACH_END(digest);
+  *s = '\0';
+
+  return result;
+}
+
+/** Turn a download_status_t into a human-readable description in a newly
+ * allocated string.  The format is specified in control-spec.txt, under
+ * the documentation for "GETINFO download/..." .  */
+static char *
+download_status_to_string(const download_status_t *dl)
+{
+  char *rv = NULL;
+  char tbuf[ISO_TIME_LEN+1];
+  const char *schedule_str, *want_authority_str;
+  const char *increment_on_str, *backoff_str;
+
+  if (dl) {
+    /* Get some substrings of the eventual output ready */
+    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
+
+    switch (dl->schedule) {
+      case DL_SCHED_GENERIC:
+        schedule_str = "DL_SCHED_GENERIC";
+        break;
+      case DL_SCHED_CONSENSUS:
+        schedule_str = "DL_SCHED_CONSENSUS";
+        break;
+      case DL_SCHED_BRIDGE:
+        schedule_str = "DL_SCHED_BRIDGE";
+        break;
+      default:
+        schedule_str = "unknown";
+        break;
+    }
+
+    switch (dl->want_authority) {
+      case DL_WANT_ANY_DIRSERVER:
+        want_authority_str = "DL_WANT_ANY_DIRSERVER";
+        break;
+      case DL_WANT_AUTHORITY:
+        want_authority_str = "DL_WANT_AUTHORITY";
+        break;
+      default:
+        want_authority_str = "unknown";
+        break;
+    }
+
+    switch (dl->increment_on) {
+      case DL_SCHED_INCREMENT_FAILURE:
+        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
+        break;
+      case DL_SCHED_INCREMENT_ATTEMPT:
+        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
+        break;
+      default:
+        increment_on_str = "unknown";
+        break;
+    }
+
+    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
+
+    /* Now assemble them */
+    tor_asprintf(&rv,
+                 "next-attempt-at %s\n"
+                 "n-download-failures %u\n"
+                 "n-download-attempts %u\n"
+                 "schedule %s\n"
+                 "want-authority %s\n"
+                 "increment-on %s\n"
+                 "backoff %s\n"
+                 "last-backoff-position %u\n"
+                 "last-delay-used %d\n",
+                 tbuf,
+                 dl->n_download_failures,
+                 dl->n_download_attempts,
+                 schedule_str,
+                 want_authority_str,
+                 increment_on_str,
+                 backoff_str,
+                 dl->last_backoff_position,
+                 dl->last_delay_used);
+  }
+
+  return rv;
+}
+
+/** Handle the consensus download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_networkstatus(const char *flavor,
+                                       download_status_t **dl_to_emit,
+                                       const char **errmsg)
+{
+  /*
+   * We get the one for the current bootstrapped status by default, or
+   * take an extra /bootstrap or /running suffix
+   */
+  if (strcmp(flavor, "ns") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
+  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
+  } else if (strcmp(flavor, "ns/running") == 0 ) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
+  } else if (strcmp(flavor, "microdesc") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/running") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
+  } else {
+    *errmsg = "Unknown flavor";
+  }
+}
+
+/** Handle the cert download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_cert(const char *fp_sk_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  const char *sk_req;
+  char id_digest[DIGEST_LEN];
+  char sk_digest[DIGEST_LEN];
+
+  /*
+   * We have to handle four cases; fp_sk_req is the request with
+   * a prefix of "downloads/cert/" snipped off.
+   *
+   * Case 1: fp_sk_req = "fps"
+   *  - We should emit a digest_list with a list of all the identity
+   *    fingerprints that can be queried for certificate download status;
+   *    get it by calling list_authority_ids_with_downloads().
+   *
+   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
+   *  - We want the default certificate for this identity fingerprint's
+   *    download status; this is the download we get from URLs starting
+   *    in /fp/ on the directory server.  We can get it with
+   *    id_only_download_status_for_authority_id().
+   *
+   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
+   *  - We want a list of all signing key digests for this identity
+   *    fingerprint which can be queried for certificate download status.
+   *    Get it with list_sk_digests_for_authority_id().
+   *
+   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
+   *         signing key digest sk
+   *   - We want the download status for the certificate for this specific
+   *     signing key and fingerprint.  These correspond to the ones we get
+   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
+   *     list_sk_digests_for_authority_id().
+   */
+
+  if (strcmp(fp_sk_req, "fps") == 0) {
+    *digest_list = list_authority_ids_with_downloads();
+    if (!(*digest_list)) {
+      *errmsg = "Failed to get list of authority identity digests (!)";
+    }
+  } else if (!strcmpstart(fp_sk_req, "fp/")) {
+    fp_sk_req += strlen("fp/");
+    /* Okay, look for another / to tell the fp from fp-sk cases */
+    sk_req = strchr(fp_sk_req, '/');
+    if (sk_req) {
+      /* okay, split it here and try to parse <fp> */
+      if (base16_decode(id_digest, DIGEST_LEN,
+                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
+        /* Skip past the '/' */
+        ++sk_req;
+        if (strcmp(sk_req, "sks") == 0) {
+          /* We're asking for the list of signing key fingerprints */
+          *digest_list = list_sk_digests_for_authority_id(id_digest);
+          if (!(*digest_list)) {
+            *errmsg = "Failed to get list of signing key digests for this "
+                      "authority identity digest";
+          }
+        } else {
+          /* We've got a signing key digest */
+          if (base16_decode(sk_digest, DIGEST_LEN,
+                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
+            *dl_to_emit =
+              download_status_for_authority_id_and_sk(id_digest, sk_digest);
+            if (!(*dl_to_emit)) {
+              *errmsg = "Failed to get download status for this identity/"
+                        "signing key digest pair";
+            }
+          } else {
+            *errmsg = "That didn't look like a signing key digest";
+          }
+        }
+      } else {
+        *errmsg = "That didn't look like an identity digest";
+      }
+    } else {
+      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
+      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
+        if (base16_decode(id_digest, DIGEST_LEN,
+                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
+          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
+          if (!(*dl_to_emit)) {
+            *errmsg = "Failed to get download status for this authority "
+                      "identity digest";
+          }
+        } else {
+          *errmsg = "That didn't look like a digest";
+        }
+      } else {
+        *errmsg = "That didn't look like a digest";
+      }
+    }
+  } else {
+    *errmsg = "Unknown certificate download status query";
+  }
+}
+
+/** Handle the routerdesc download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_desc(const char *desc_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  char desc_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: desc_req = "descs"
+   *   - Emit a list of all router descriptor digests, which we get by
+   *     calling router_get_descriptor_digests(); this can return NULL
+   *     if we have no current ns-flavor consensus.
+   *
+   * Case 2: desc_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using router_get_dl_status_by_descriptor_digest().
+   */
+
+  if (strcmp(desc_req, "descs") == 0) {
+    *digest_list = router_get_descriptor_digests();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
+    }
+    /*
+     * Microdescs don't use the download_status_t mechanism, so we don't
+     * answer queries about their downloads here; see microdesc.c.
+     */
+  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(desc_digest, DIGEST_LEN,
+                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such descriptor digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown router descriptor download status query";
+  }
+}
+
+/** Handle the bridge download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_bridge(const char *bridge_req,
+                                download_status_t **dl_to_emit,
+                                smartlist_t **digest_list,
+                                const char **errmsg)
+{
+  char bridge_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: bridge_req = "bridges"
+   *   - Emit a list of all bridge identity digests, which we get by
+   *     calling list_bridge_identities(); this can return NULL if we are
+   *     not using bridges.
+   *
+   * Case 2: bridge_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using get_bridge_dl_status_by_id().
+   */
+
+  if (strcmp(bridge_req, "bridges") == 0) {
+    *digest_list = list_bridge_identities();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to be using bridges";
+    }
+  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(bridge_digest, DIGEST_LEN,
+                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such bridge identity digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown bridge descriptor download status query";
+  }
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * download status information. */
+STATIC int
+getinfo_helper_downloads(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  download_status_t *dl_to_emit = NULL;
+  smartlist_t *digest_list = NULL;
+
+  /* Assert args are sane */
+  tor_assert(control_conn != NULL);
+  tor_assert(question != NULL);
+  tor_assert(answer != NULL);
+  tor_assert(errmsg != NULL);
+
+  /* We check for this later to see if we should supply a default */
+  *errmsg = NULL;
+
+  /* Are we after networkstatus downloads? */
+  if (!strcmpstart(question, "downloads/networkstatus/")) {
+    getinfo_helper_downloads_networkstatus(
+        question + strlen("downloads/networkstatus/"),
+        &dl_to_emit, errmsg);
+  /* Certificates? */
+  } else if (!strcmpstart(question, "downloads/cert/")) {
+    getinfo_helper_downloads_cert(
+        question + strlen("downloads/cert/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Router descriptors? */
+  } else if (!strcmpstart(question, "downloads/desc/")) {
+    getinfo_helper_downloads_desc(
+        question + strlen("downloads/desc/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Bridge descriptors? */
+  } else if (!strcmpstart(question, "downloads/bridge/")) {
+    getinfo_helper_downloads_bridge(
+        question + strlen("downloads/bridge/"),
+        &dl_to_emit, &digest_list, errmsg);
+  } else {
+    *errmsg = "Unknown download status query";
+  }
+
+  if (dl_to_emit) {
+    *answer = download_status_to_string(dl_to_emit);
+
+    return 0;
+  } else if (digest_list) {
+    *answer = digest_list_to_string(digest_list);
+    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
+    smartlist_free(digest_list);
+
+    return 0;
+  } else {
+    if (!(*errmsg)) {
+      *errmsg = "Unknown error";
+    }
+
+    return -1;
+  }
+}
+
+/** Implementation helper for GETINFO: knows how to generate summaries of the
+ * current states of things we send events about. */
+static int
+getinfo_helper_events(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  const or_options_t *options = get_options();
+  (void) control_conn;
+  if (!strcmp(question, "circuit-status")) {
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
+      origin_circuit_t *circ;
+      char *circdesc;
+      const char *state;
+      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
+        continue;
+      circ = TO_ORIGIN_CIRCUIT(circ_);
+
+      if (circ->base_.state == CIRCUIT_STATE_OPEN)
+        state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
+      else if (circ->cpath)
+        state = "EXTENDED";
+      else
+        state = "LAUNCHED";
+
+      circdesc = circuit_describe_status_for_controller(circ);
+
+      smartlist_add_asprintf(status, "%lu %s%s%s",
+                   (unsigned long)circ->global_identifier,
+                   state, *circdesc ? " " : "", circdesc);
+      tor_free(circdesc);
+    }
+    SMARTLIST_FOREACH_END(circ_);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "stream-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    char buf[256];
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      entry_connection_t *conn;
+      circuit_t *circ;
+      origin_circuit_t *origin_circ = NULL;
+      if (base_conn->type != CONN_TYPE_AP ||
+          base_conn->marked_for_close ||
+          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
+          base_conn->state == AP_CONN_STATE_NATD_WAIT)
+        continue;
+      conn = TO_ENTRY_CONN(base_conn);
+      switch (base_conn->state)
+        {
+        case AP_CONN_STATE_CONTROLLER_WAIT:
+        case AP_CONN_STATE_CIRCUIT_WAIT:
+          if (conn->socks_request &&
+              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
+            state = "NEWRESOLVE";
+          else
+            state = "NEW";
+          break;
+        case AP_CONN_STATE_RENDDESC_WAIT:
+        case AP_CONN_STATE_CONNECT_WAIT:
+          state = "SENTCONNECT"; break;
+        case AP_CONN_STATE_RESOLVE_WAIT:
+          state = "SENTRESOLVE"; break;
+        case AP_CONN_STATE_OPEN:
+          state = "SUCCEEDED"; break;
+        default:
+          log_warn(LD_BUG, "Asked for stream in unknown state %d",
+                   base_conn->state);
+          continue;
+        }
+      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+      if (circ && CIRCUIT_IS_ORIGIN(circ))
+        origin_circ = TO_ORIGIN_CIRCUIT(circ);
+      write_stream_target_to_buf(conn, buf, sizeof(buf));
+      smartlist_add_asprintf(status, "%lu %s %lu %s",
+                   (unsigned long) base_conn->global_identifier,state,
+                   origin_circ?
+                         (unsigned long)origin_circ->global_identifier : 0ul,
+                   buf);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "orconn-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      char name[128];
+      or_connection_t *conn;
+      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
+        continue;
+      conn = TO_OR_CONN(base_conn);
+      if (conn->base_.state == OR_CONN_STATE_OPEN)
+        state = "CONNECTED";
+      else if (conn->nickname)
+        state = "LAUNCHED";
+      else
+        state = "NEW";
+      orconn_target_get_name(name, sizeof(name), conn);
+      smartlist_add_asprintf(status, "%s %s", name, state);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmpstart(question, "address-mappings/")) {
+    time_t min_e, max_e;
+    smartlist_t *mappings;
+    question += strlen("address-mappings/");
+    if (!strcmp(question, "all")) {
+      min_e = 0; max_e = TIME_MAX;
+    } else if (!strcmp(question, "cache")) {
+      min_e = 2; max_e = TIME_MAX;
+    } else if (!strcmp(question, "config")) {
+      min_e = 0; max_e = 0;
+    } else if (!strcmp(question, "control")) {
+      min_e = 1; max_e = 1;
+    } else {
+      return 0;
+    }
+    mappings = smartlist_new();
+    addressmap_get_mappings(mappings, min_e, max_e, 1);
+    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
+    smartlist_free(mappings);
+  } else if (!strcmpstart(question, "status/")) {
+    /* Note that status/ is not a catch-all for events; there's only supposed
+     * to be a status GETINFO if there's a corresponding STATUS event. */
+    if (!strcmp(question, "status/circuit-established")) {
+      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
+    } else if (!strcmp(question, "status/enough-dir-info")) {
+      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
+    } else if (!strcmp(question, "status/good-server-descriptor") ||
+               !strcmp(question, "status/accepted-server-descriptor")) {
+      /* They're equivalent for now, until we can figure out how to make
+       * good-server-descriptor be what we want. See comment in
+       * control-spec.txt. */
+      *answer = tor_strdup(directories_have_accepted_server_descriptor()
+                           ? "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
+      *answer = tor_strdup(check_whether_orport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
+      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded")) {
+      tor_asprintf(answer, "OR=%d DIR=%d",
+                   check_whether_orport_reachable(options) ? 1 : 0,
+                   check_whether_dirport_reachable(options) ? 1 : 0);
+    } else if (!strcmp(question, "status/bootstrap-phase")) {
+      *answer = control_event_boot_last_msg();
+    } else if (!strcmpstart(question, "status/version/")) {
+      int is_server = server_mode(options);
+      networkstatus_t *c = networkstatus_get_latest_consensus();
+      version_status_t status;
+      const char *recommended;
+      if (c) {
+        recommended = is_server ? c->server_versions : c->client_versions;
+        status = tor_version_is_obsolete(VERSION, recommended);
+      } else {
+        recommended = "?";
+        status = VS_UNKNOWN;
+      }
+
+      if (!strcmp(question, "status/version/recommended")) {
+        *answer = tor_strdup(recommended);
+        return 0;
+      }
+      if (!strcmp(question, "status/version/current")) {
+        switch (status)
+          {
+          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
+          case VS_OLD: *answer = tor_strdup("obsolete"); break;
+          case VS_NEW: *answer = tor_strdup("new"); break;
+          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
+          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
+          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
+          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
+          default: tor_fragile_assert();
+          }
+      }
+    } else if (!strcmp(question, "status/clients-seen")) {
+      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
+      if (!bridge_stats) {
+        *errmsg = "No bridge-client stats available";
+        return -1;
+      }
+      *answer = bridge_stats;
+    } else if (!strcmp(question, "status/fresh-relay-descs")) {
+      if (!server_mode(options)) {
+        *errmsg = "Only relays have descriptors";
+        return -1;
+      }
+      routerinfo_t *r;
+      extrainfo_t *e;
+      if (router_build_fresh_descriptor(&r, &e) < 0) {
+        *errmsg = "Error generating descriptor";
+        return -1;
+      }
+      size_t size = r->cache_info.signed_descriptor_len + 1;
+      if (e) {
+        size += e->cache_info.signed_descriptor_len + 1;
+      }
+      tor_assert(r->cache_info.signed_descriptor_len);
+      char *descs = tor_malloc(size);
+      char *cp = descs;
+      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
+             r->cache_info.signed_descriptor_len);
+      cp += r->cache_info.signed_descriptor_len - 1;
+      if (e) {
+        if (cp[0] == '\0') {
+          cp[0] = '\n';
+        } else if (cp[0] != '\n') {
+          cp[1] = '\n';
+          cp++;
+        }
+        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
+               e->cache_info.signed_descriptor_len);
+        cp += e->cache_info.signed_descriptor_len - 1;
+      }
+      if (cp[0] == '\n') {
+        cp[0] = '\0';
+      } else if (cp[0] != '\0') {
+        cp[1] = '\0';
+      }
+      *answer = descs;
+      routerinfo_free(r);
+      extrainfo_free(e);
+    } else {
+      return 0;
+    }
+  }
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows how to enumerate hidden services
+ * created via the control port. */
+STATIC int
+getinfo_helper_onions(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
+
+  if (control_conn && !strcmp(question, "onions/current")) {
+    onion_list = control_conn->ephemeral_onion_services;
+  } else if (!strcmp(question, "onions/detached")) {
+    onion_list = get_detached_onion_services();
+  } else {
+    return 0;
+  }
+  if (!onion_list || smartlist_len(onion_list) == 0) {
+    if (answer) {
+      *answer = tor_strdup("");
+    }
+  } else {
+    if (answer) {
+      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about network
+ * liveness. */
+static int
+getinfo_helper_liveness(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+  if (strcmp(question, "network-liveness") == 0) {
+    if (get_cached_network_liveness()) {
+      *answer = tor_strdup("up");
+    } else {
+      *answer = tor_strdup("down");
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
+/** Callback function for GETINFO: on a given control connection, try to
+ * answer the question <b>q</b> and store the newly-allocated answer in
+ * *<b>a</b>. If an internal error occurs, return -1 and optionally set
+ * *<b>error_out</b> to point to an error message to be delivered to the
+ * controller. On success, _or if the key is not recognized_, return 0. Do not
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
+ */
+typedef int (*getinfo_helper_t)(control_connection_t *,
+                                const char *q, char **a,
+                                const char **error_out);
+
+/** A single item for the GETINFO question-to-answer-function table. */
+typedef struct getinfo_item_t {
+  const char *varname; /**< The value (or prefix) of the question. */
+  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
+                        * this entry is documentation-only. */
+  const char *desc; /**< Description of the variable. */
+  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
+} getinfo_item_t;
+
+#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
+#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
+#define DOC(name, desc) { name, NULL, desc, 0 }
+
+/** Table mapping questions accepted by GETINFO to the functions that know how
+ * to answer them. */
+static const getinfo_item_t getinfo_items[] = {
+  ITEM("version", misc, "The current version of Tor."),
+  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
+  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
+  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
+  ITEM("config-text", misc,
+       "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
+  ITEM("accounting/bytes", accounting,
+       "Number of bytes read/written so far in the accounting interval."),
+  ITEM("accounting/bytes-left", accounting,
+      "Number of bytes left to write/read so far in the accounting interval."),
+  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
+  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
+  ITEM("accounting/interval-start", accounting,
+       "Time when the accounting period starts."),
+  ITEM("accounting/interval-end", accounting,
+       "Time when the accounting period ends."),
+  ITEM("accounting/interval-wake", accounting,
+       "Time to wake up in this accounting period."),
+  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
+  ITEM("entry-guards", entry_guards,
+       "Which nodes are we using as entry guards?"),
+  ITEM("fingerprint", misc, NULL),
+  PREFIX("config/", config, "Current configuration values."),
+  DOC("config/names",
+      "List of configuration options, types, and documentation."),
+  DOC("config/defaults",
+      "List of default values for configuration options. "
+      "See also config/names"),
+  PREFIX("current-time/", current_time, "Current time."),
+  DOC("current-time/local", "Current time on the local system."),
+  DOC("current-time/utc", "Current UTC time."),
+  PREFIX("downloads/networkstatus/", downloads,
+         "Download statuses for networkstatus objects"),
+  DOC("downloads/networkstatus/ns",
+      "Download status for current-mode networkstatus download"),
+  DOC("downloads/networkstatus/ns/bootstrap",
+      "Download status for bootstrap-time networkstatus download"),
+  DOC("downloads/networkstatus/ns/running",
+      "Download status for run-time networkstatus download"),
+  DOC("downloads/networkstatus/microdesc",
+      "Download status for current-mode microdesc download"),
+  DOC("downloads/networkstatus/microdesc/bootstrap",
+      "Download status for bootstrap-time microdesc download"),
+  DOC("downloads/networkstatus/microdesc/running",
+      "Download status for run-time microdesc download"),
+  PREFIX("downloads/cert/", downloads,
+         "Download statuses for certificates, by id fingerprint and "
+         "signing key"),
+  DOC("downloads/cert/fps",
+      "List of authority fingerprints for which any download statuses "
+      "exist"),
+  DOC("downloads/cert/fp/<fp>",
+      "Download status for <fp> with the default signing key; corresponds "
+      "to /fp/ URLs on directory server."),
+  DOC("downloads/cert/fp/<fp>/sks",
+      "List of signing keys for which specific download statuses are "
+      "available for this id fingerprint"),
+  DOC("downloads/cert/fp/<fp>/<sk>",
+      "Download status for <fp> with signing key <sk>; corresponds "
+      "to /fp-sk/ URLs on directory server."),
+  PREFIX("downloads/desc/", downloads,
+         "Download statuses for router descriptors, by descriptor digest"),
+  DOC("downloads/desc/descs",
+      "Return a list of known router descriptor digests"),
+  DOC("downloads/desc/<desc>",
+      "Return a download status for a given descriptor digest"),
+  PREFIX("downloads/bridge/", downloads,
+         "Download statuses for bridge descriptors, by bridge identity "
+         "digest"),
+  DOC("downloads/bridge/bridges",
+      "Return a list of configured bridge identity digests with download "
+      "statuses"),
+  DOC("downloads/bridge/<desc>",
+      "Return a download status for a given bridge identity digest"),
+  ITEM("info/names", misc,
+       "List of GETINFO options, types, and documentation."),
+  ITEM("events/names", misc,
+       "Events that the controller can ask for with SETEVENTS."),
+  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
+  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
+  PREFIX("desc/id/", dir, "Router descriptors by ID."),
+  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
+  ITEM("desc/all-recent", dir,
+       "All non-expired, non-superseded router descriptors."),
+  ITEM("desc/download-enabled", dir,
+       "Do we try to download router descriptors?"),
+  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
+  ITEM("md/all", dir, "All known microdescriptors."),
+  PREFIX("md/id/", dir, "Microdescriptors by ID"),
+  PREFIX("md/name/", dir, "Microdescriptors by name"),
+  ITEM("md/download-enabled", dir,
+       "Do we try to download microdescriptors?"),
+  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
+  PREFIX("hs/client/desc/id", dir,
+         "Hidden Service descriptor in client's cache by onion."),
+  PREFIX("hs/service/desc/id/", dir,
+         "Hidden Service descriptor in services's cache by onion."),
+  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
+  ITEM("ns/all", networkstatus,
+       "Brief summary of router status (v2 directory format)"),
+  PREFIX("ns/id/", networkstatus,
+         "Brief summary of router status by ID (v2 directory format)."),
+  PREFIX("ns/name/", networkstatus,
+         "Brief summary of router status by nickname (v2 directory format)."),
+  PREFIX("ns/purpose/", networkstatus,
+         "Brief summary of router status by purpose (v2 directory format)."),
+  PREFIX("consensus/", networkstatus,
+         "Information about and from the ns consensus."),
+  ITEM("network-status", dir,
+       "Brief summary of router status (v1 directory format)"),
+  ITEM("network-liveness", liveness,
+       "Current opinion on whether the network is live"),
+  ITEM("circuit-status", events, "List of current circuits originating here."),
+  ITEM("stream-status", events,"List of current streams."),
+  ITEM("orconn-status", events, "A list of current OR connections."),
+  ITEM("dormant", misc,
+       "Is Tor dormant (not building circuits because it's idle)?"),
+  PREFIX("address-mappings/", events, NULL),
+  DOC("address-mappings/all", "Current address mappings."),
+  DOC("address-mappings/cache", "Current cached DNS replies."),
+  DOC("address-mappings/config",
+      "Current address mappings from configuration."),
+  DOC("address-mappings/control", "Current address mappings from controller."),
+  PREFIX("status/", events, NULL),
+  DOC("status/circuit-established",
+      "Whether we think client functionality is working."),
+  DOC("status/enough-dir-info",
+      "Whether we have enough up-to-date directory information to build "
+      "circuits."),
+  DOC("status/bootstrap-phase",
+      "The last bootstrap phase status event that Tor sent."),
+  DOC("status/clients-seen",
+      "Breakdown of client countries seen by a bridge."),
+  DOC("status/fresh-relay-descs",
+      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
+  DOC("status/version/recommended", "List of currently recommended versions."),
+  DOC("status/version/current", "Status of the current version."),
+  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
+  ITEM("traffic/read", misc,"Bytes read since the process was started."),
+  ITEM("traffic/written", misc,
+       "Bytes written since the process was started."),
+  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
+  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
+  ITEM("process/uid", misc, "User id running the tor process."),
+  ITEM("process/user", misc,
+       "Username under which the tor process is running."),
+  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
+  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
+  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
+  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
+  PREFIX("dir/status/", dir,
+         "v2 networkstatus docs as retrieved from a DirPort."),
+  ITEM("dir/status-vote/current/consensus", dir,
+       "v3 Networkstatus consensus as retrieved from a DirPort."),
+  ITEM("exit-policy/default", policies,
+       "The default value appended to the configured exit policy."),
+  ITEM("exit-policy/reject-private/default", policies,
+       "The default rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate."),
+  ITEM("exit-policy/reject-private/relay", policies,
+       "The relay-specific rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
+  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
+  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
+  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
+  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
+  ITEM("onions/current", onions,
+       "Onion services owned by the current control connection."),
+  ITEM("onions/detached", onions,
+       "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
+  { NULL, NULL, NULL, 0 }
+};
+
+/** Allocate and return a list of recognized GETINFO options. */
+static char *
+list_getinfo_options(void)
+{
+  int i;
+  smartlist_t *lines = smartlist_new();
+  char *ans;
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    if (!getinfo_items[i].desc)
+      continue;
+
+    smartlist_add_asprintf(lines, "%s%s -- %s\n",
+                 getinfo_items[i].varname,
+                 getinfo_items[i].is_prefix ? "*" : "",
+                 getinfo_items[i].desc);
+  }
+  smartlist_sort_strings(lines);
+
+  ans = smartlist_join_strings(lines, "", 0, NULL);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+
+  return ans;
+}
+
+/** Lookup the 'getinfo' entry <b>question</b>, and return
+ * the answer in <b>*answer</b> (or NULL if key not recognized).
+ * Return 0 if success or unrecognized, or -1 if recognized but
+ * internal error. */
+static int
+handle_getinfo_helper(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **err_out)
+{
+  int i;
+  *answer = NULL; /* unrecognized key by default */
+
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    int match;
+    if (getinfo_items[i].is_prefix)
+      match = !strcmpstart(question, getinfo_items[i].varname);
+    else
+      match = !strcmp(question, getinfo_items[i].varname);
+    if (match) {
+      tor_assert(getinfo_items[i].fn);
+      return getinfo_items[i].fn(control_conn, question, answer, err_out);
+    }
+  }
+
+  return 0; /* unrecognized */
+}
+
+const control_cmd_syntax_t getinfo_syntax = {
+  .max_args = UINT_MAX,
+};
+
+/** Called when we receive a GETINFO command.  Try to fetch all requested
+ * information, and reply with information or error message. */
+int
+handle_control_getinfo(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *ans = NULL;
+  int i;
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    const char *errmsg = NULL;
+
+    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
+      if (!errmsg)
+        errmsg = "Internal error";
+      control_write_endreply(conn, 551, errmsg);
+      goto done;
+    }
+    if (!ans) {
+      if (errmsg) /* use provided error message */
+        smartlist_add_strdup(unrecognized, errmsg);
+      else /* use default error message */
+        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
+    } else {
+      smartlist_add_strdup(answers, q);
+      smartlist_add(answers, ans);
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if (smartlist_len(unrecognized)) {
+    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
+    for (i=0; i < smartlist_len(unrecognized)-1; ++i)
+      control_write_midreply(conn, 552,
+                             (char *)smartlist_get(unrecognized, i));
+
+    control_write_endreply(conn, 552, (char *)smartlist_get(unrecognized, i));
+    goto done;
+  }
+
+  for (i = 0; i < smartlist_len(answers); i += 2) {
+    char *k = smartlist_get(answers, i);
+    char *v = smartlist_get(answers, i+1);
+    if (!strchr(v, '\n') && !strchr(v, '\r')) {
+      control_printf_midreply(conn, 250, "%s=%s", k, v);
+    } else {
+      control_printf_datareply(conn, 250, "%s=", k);
+      control_write_data(conn, v);
+    }
+  }
+  send_control_done(conn);
+
+ done:
+  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
+  smartlist_free(answers);
+  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
+  smartlist_free(unrecognized);
+
+  return 0;
+}
diff --git a/src/feature/control/control_getinfo.h b/src/feature/control/control_getinfo.h
new file mode 100644
index 0000000000..52978686d8
--- /dev/null
+++ b/src/feature/control/control_getinfo.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control.h
+ * \brief Header file for control.c.
+ **/
+
+#ifndef TOR_CONTROL_GETINFO_H
+#define TOR_CONTROL_GETINFO_H
+
+struct control_cmd_syntax_t;
+struct control_cmd_args_t;
+extern const struct control_cmd_syntax_t getinfo_syntax;
+
+int handle_control_getinfo(control_connection_t *conn,
+                           const struct control_cmd_args_t *args);
+
+#ifdef CONTROL_GETINFO_PRIVATE
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_networkstatus(
+    const char *flavor,
+    download_status_t **dl_to_emit,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_cert(
+    const char *fp_sk_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_desc(
+    const char *desc_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_bridge(
+    const char *bridge_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC int getinfo_helper_downloads(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_time(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+#endif /* defined(CONTROL_GETINFO_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_GETINFO_H) */
diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c
new file mode 100644
index 0000000000..5dec87491d
--- /dev/null
+++ b/src/feature/control/control_proto.c
@@ -0,0 +1,277 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+
+/** Append a NUL-terminated string <b>s</b> to the end of
+ * <b>conn</b>-\>outbuf.
+ */
+void
+connection_write_str_to_buf(const char *s, control_connection_t *conn)
+{
+  size_t len = strlen(s);
+  connection_buf_add(s, len, TO_CONN(conn));
+}
+
+/** Acts like sprintf, but writes its formatted string to the end of
+ * <b>conn</b>-\>outbuf. */
+void
+connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
+{
+  va_list ap;
+  char *buf = NULL;
+  int len;
+
+  va_start(ap,format);
+  len = tor_vasprintf(&buf, format, ap);
+  va_end(ap);
+
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
+
+  tor_free(buf);
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
+ * contents of <b>data</b> into *<b>out</b>, adding a period before any period
+ * that appears at the start of a line, and adding a period-CRLF line at
+ * the end. Replace all LF characters sequences with CRLF.  Return the number
+ * of bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+write_escaped_data(const char *data, size_t len, char **out)
+{
+  tor_assert(len < SIZE_MAX - 9);
+  size_t sz_out = len+8+1;
+  char *outp;
+  const char *start = data, *end;
+  size_t i;
+  int start_of_line;
+  for (i=0; i < len; ++i) {
+    if (data[i] == '\n') {
+      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
+      if (sz_out >= SIZE_T_CEILING) {
+        log_warn(LD_BUG, "Input to write_escaped_data was too long");
+        *out = tor_strdup(".\r\n");
+        return 3;
+      }
+    }
+  }
+  *out = outp = tor_malloc(sz_out);
+  end = data+len;
+  start_of_line = 1;
+  while (data < end) {
+    if (*data == '\n') {
+      if (data > start && data[-1] != '\r')
+        *outp++ = '\r';
+      start_of_line = 1;
+    } else if (*data == '.') {
+      if (start_of_line) {
+        start_of_line = 0;
+        *outp++ = '.';
+      }
+    } else {
+      start_of_line = 0;
+    }
+    *outp++ = *data++;
+  }
+  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
+    *outp++ = '\r';
+    *outp++ = '\n';
+  }
+  *outp++ = '.';
+  *outp++ = '\r';
+  *outp++ = '\n';
+  *outp = '\0'; /* NUL-terminate just in case. */
+  tor_assert(outp >= *out);
+  tor_assert((size_t)(outp - *out) <= sz_out);
+  return outp - *out;
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
+ * the contents of <b>data</b> into *<b>out</b>, removing any period
+ * that appears at the start of a line, and replacing all CRLF sequences
+ * with LF.   Return the number of
+ * bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+read_escaped_data(const char *data, size_t len, char **out)
+{
+  char *outp;
+  const char *next;
+  const char *end;
+
+  *out = outp = tor_malloc(len+1);
+
+  end = data+len;
+
+  while (data < end) {
+    /* we're at the start of a line. */
+    if (*data == '.')
+      ++data;
+    next = memchr(data, '\n', end-data);
+    if (next) {
+      size_t n_to_copy = next-data;
+      /* Don't copy a CR that precedes this LF. */
+      if (n_to_copy && *(next-1) == '\r')
+        --n_to_copy;
+      memcpy(outp, data, n_to_copy);
+      outp += n_to_copy;
+      data = next+1; /* This will point at the start of the next line,
+                      * or the end of the string, or a period. */
+    } else {
+      memcpy(outp, data, end-data);
+      outp += (end-data);
+      *outp = '\0';
+      return outp - *out;
+    }
+    *outp++ = '\n';
+  }
+
+  *outp = '\0';
+  return outp - *out;
+}
+
+/** Send a "DONE" message down the control connection <b>conn</b>. */
+void
+send_control_done(control_connection_t *conn)
+{
+  control_write_endreply(conn, 250, "OK");
+}
+
+/** Write a reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param s string reply content
+ */
+MOCK_IMPL(void,
+control_write_reply, (control_connection_t *conn, int code, int c,
+                      const char *s))
+{
+  connection_printf_to_buf(conn, "%03d%c%s\r\n", code, c, s);
+}
+
+/** Write a formatted reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param fmt format string
+ * @param ap va_list from caller
+ */
+void
+control_vprintf_reply(control_connection_t *conn, int code, int c,
+                      const char *fmt, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, fmt, ap);
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+  control_write_reply(conn, code, c, buf);
+  tor_free(buf);
+}
+
+/** Write an EndReplyLine */
+void
+control_write_endreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, ' ', s);
+}
+
+/** Write a formatted EndReplyLine */
+void
+control_printf_endreply(control_connection_t *conn, int code,
+                        const char *fmt, ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, ' ', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a MidReplyLine */
+void
+control_write_midreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '-', s);
+}
+
+/** Write a formatted MidReplyLine */
+void
+control_printf_midreply(control_connection_t *conn, int code, const char *fmt,
+                        ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '-', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a DataReplyLine */
+void
+control_write_datareply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '+', s);
+}
+
+/** Write a formatted DataReplyLine */
+void
+control_printf_datareply(control_connection_t *conn, int code, const char *fmt,
+                         ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '+', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a CmdData */
+void
+control_write_data(control_connection_t *conn, const char *data)
+{
+  char *esc = NULL;
+  size_t esc_len;
+
+  esc_len = write_escaped_data(data, strlen(data), &esc);
+  connection_buf_add(esc, esc_len, TO_CONN(conn));
+  tor_free(esc);
+}
diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h
new file mode 100644
index 0000000000..3182f3d415
--- /dev/null
+++ b/src/feature/control/control_proto.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.h
+ * \brief Header file for control_proto.c.
+ **/
+
+#ifndef TOR_CONTROL_PROTO_H
+#define TOR_CONTROL_PROTO_H
+
+void connection_write_str_to_buf(const char *s, control_connection_t *conn);
+void connection_printf_to_buf(control_connection_t *conn,
+                                     const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+size_t write_escaped_data(const char *data, size_t len, char **out);
+size_t read_escaped_data(const char *data, size_t len, char **out);
+void send_control_done(control_connection_t *conn);
+
+MOCK_DECL(void, control_write_reply, (control_connection_t *conn, int code,
+                                      int c, const char *s));
+void control_vprintf_reply(control_connection_t *conn, int code, int c,
+                           const char *fmt, va_list ap)
+  CHECK_PRINTF(4, 0);
+void control_write_endreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_endreply(control_connection_t *conn, int code,
+                             const char *fmt, ...)
+  CHECK_PRINTF(3, 4);
+void control_write_midreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_midreply(control_connection_t *conn, int code,
+                             const char *fmt,
+                             ...)
+  CHECK_PRINTF(3, 4);
+void control_write_datareply(control_connection_t *conn, int code,
+                             const char *s);
+void control_printf_datareply(control_connection_t *conn, int code,
+                              const char *fmt,
+                              ...)
+  CHECK_PRINTF(3, 4);
+void control_write_data(control_connection_t *conn, const char *data);
+
+#endif /* !defined(TOR_CONTROL_PROTO_H) */
diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c
index a1ddd2119a..33c5ba1336 100644
--- a/src/feature/control/fmt_serverstatus.c
+++ b/src/feature/control/fmt_serverstatus.c
@@ -9,8 +9,8 @@
 #include "app/config/config.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/voteflags.h"// XXXX remove
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -66,11 +66,9 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   smartlist_t *rs_entries;
   time_t now = time(NULL);
   time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  const or_options_t *options = get_options();
   /* We include v2 dir auths here too, because they need to answer
    * controllers. Eventually we'll deprecate this whole function;
    * see also networkstatus_getinfo_by_purpose(). */
-  int authdir = authdir_mode_publishes_statuses(options);
   tor_assert(router_status_out);
 
   rs_entries = smartlist_new();
@@ -78,10 +76,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
     const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
     tor_assert(node);
-    if (authdir) {
-      /* Update router status in routerinfo_t. */
-      dirserv_set_router_is_running(ri, now);
-    }
     if (for_controller) {
       char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
       char *cp = name_buf;
diff --git a/src/feature/control/fmt_serverstatus.h b/src/feature/control/fmt_serverstatus.h
index 4b95e5b59f..d9190cb7e1 100644
--- a/src/feature/control/fmt_serverstatus.h
+++ b/src/feature/control/fmt_serverstatus.h
@@ -15,4 +15,4 @@
 int list_server_status_v1(smartlist_t *routers, char **router_status_out,
                           int for_controller);
 
-#endif
+#endif /* !defined(TOR_FMT_SERVERSTATUS_H) */
diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h
index fe22137859..94759d0d18 100644
--- a/src/feature/control/getinfo_geoip.h
+++ b/src/feature/control/getinfo_geoip.h
@@ -11,4 +11,4 @@ int getinfo_helper_geoip(control_connection_t *control_conn,
                      const char *question, char **answer,
                      const char **errmsg);
 
-#endif
+#endif /* !defined(TOR_GETINFO_GEOIP_H) */
diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h
index 876a1f947b..bfd5f4dc04 100644
--- a/src/feature/dirauth/authmode.h
+++ b/src/feature/dirauth/authmode.h
@@ -29,7 +29,7 @@ authdir_mode_v3(const or_options_t *options)
 
 #define have_module_dirauth() (1)
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 #define authdir_mode(options) (((void)(options)),0)
 #define authdir_mode_handles_descs(options,purpose) \
@@ -41,6 +41,6 @@ authdir_mode_v3(const or_options_t *options)
 
 #define have_module_dirauth() (0)
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif /* TOR_MODE_H */
+#endif /* !defined(TOR_DIRAUTH_MODE_H) */
diff --git a/src/feature/dirauth/bridgeauth.c b/src/feature/dirauth/bridgeauth.c
new file mode 100644
index 0000000000..4aaefc7a6d
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.c
@@ -0,0 +1,55 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/voteflags.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Write out router status entries for all our bridge descriptors. Here, we
+ * also mark routers as running. */
+void
+bridgeauth_dump_bridge_status_to_file(time_t now)
+{
+  char *status;
+  char *fname = NULL;
+  char *thresholds = NULL;
+  char *published_thresholds_and_status = NULL;
+  char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
+
+  dirserv_set_bridges_running(now);
+  status = networkstatus_getinfo_by_purpose("bridge", now);
+
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
+  format_iso_time(published, now);
+  dirserv_compute_bridge_flag_thresholds();
+  thresholds = dirserv_get_flag_thresholds_line();
+  tor_asprintf(&published_thresholds_and_status,
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
+  if (write_str_to_file(fname,published_thresholds_and_status,0)<0) {
+    log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file.");
+  }
+  tor_free(thresholds);
+  tor_free(published_thresholds_and_status);
+  tor_free(fname);
+  tor_free(status);
+  tor_free(fingerprint_line);
+}
diff --git a/src/feature/dirauth/bridgeauth.h b/src/feature/dirauth/bridgeauth.h
new file mode 100644
index 0000000000..4905e9c3ee
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DIRAUTH_BRIDGEAUTH_H
+#define TOR_DIRAUTH_BRIDGEAUTH_H
+
+void bridgeauth_dump_bridge_status_to_file(time_t now);
+
+#endif /* !defined(TOR_DIRAUTH_BRIDGEAUTH_H) */
diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c
index 12f9399e9f..e60c8b86bd 100644
--- a/src/feature/dirauth/bwauth.c
+++ b/src/feature/dirauth/bwauth.c
@@ -20,6 +20,7 @@
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
+#include "lib/crypt_ops/crypto_format.h"
 #include "lib/encoding/keyval.h"
 
 /** Total number of routers with measured bandwidth; this is set by
@@ -198,14 +199,38 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
 }
 
 /**
- * Read the measured bandwidth list file, apply it to the list of
- * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
+ * Read the measured bandwidth list <b>from_file</b>:
+ * - store all the headers in <b>bw_file_headers</b>,
+ * - apply bandwidth lines to the list of vote_routerstatus_t in
+ *   <b>routerstatuses</b>,
+ * - cache bandwidth lines for dirserv_get_bandwidth_for_router(),
+ * - expire old entries in the measured bandwidth cache, and
+ * - store the DIGEST_SHA256 of the contents of the file in <b>digest_out</b>.
+ *
  * Returns -1 on error, 0 otherwise.
+ *
+ * If the file can't be read, or is empty:
+ * - <b>bw_file_headers</b> is empty,
+ * - <b>routerstatuses</b> is not modified,
+ * - the measured bandwidth cache is not modified, and
+ * - <b>digest_out</b> is the zero-byte digest.
+ *
+ * Otherwise, if there is an error later in the file:
+ * - <b>bw_file_headers</b> contains all the headers up to the error,
+ * - <b>routerstatuses</b> is updated with all the relay lines up to the error,
+ * - the measured bandwidth cache is updated with all the relay lines up to
+ *   the error,
+ * - if the timestamp is valid and recent, old entries in the  measured
+ *   bandwidth cache are expired, and
+ * - <b>digest_out</b> is the digest up to the first read error (if any).
+ *   The digest is taken over all the readable file contents, even if the
+ *   file is outdated or unparseable.
  */
 int
 dirserv_read_measured_bandwidths(const char *from_file,
                                  smartlist_t *routerstatuses,
-                                 smartlist_t *bw_file_headers)
+                                 smartlist_t *bw_file_headers,
+                                 uint8_t *digest_out)
 {
   FILE *fp = tor_fopen_cloexec(from_file, "r");
   int applied_lines = 0;
@@ -219,8 +244,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   int rv = -1;
   char *line = NULL;
   size_t n = 0;
-
-  /* Initialise line, so that we can't possibly run off the end. */
+  crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256);
 
   if (fp == NULL) {
     log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
@@ -228,16 +252,18 @@ dirserv_read_measured_bandwidths(const char *from_file,
     goto err;
   }
 
-  /* If fgets fails, line is either unmodified, or indeterminate. */
   if (tor_getline(&line,&n,fp) <= 0) {
     log_warn(LD_DIRSERV, "Empty bandwidth file");
     goto err;
   }
+  /* If the line could be gotten, add it to the digest */
+  crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
 
   if (!strlen(line) || line[strlen(line)-1] != '\n') {
     log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    /* Continue adding lines to the digest. */
+    goto continue_digest;
   }
 
   line[strlen(line)-1] = '\0';
@@ -245,14 +271,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
   if (!ok) {
     log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    goto continue_digest;
   }
 
-  now = time(NULL);
+  now = approx_time();
   if ((now - file_time) > MAX_MEASUREMENT_AGE) {
     log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
              (unsigned)(time(NULL) - file_time));
-    goto err;
+    goto continue_digest;
   }
 
   /* If timestamp was correct and bw_file_headers is not NULL,
@@ -267,6 +293,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   while (!feof(fp)) {
     measured_bw_line_t parsed_line;
     if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
       if (measured_bw_line_parse(&parsed_line, line,
                                  line_is_after_headers) != -1) {
         /* This condition will be true when the first complete valid bw line
@@ -305,6 +332,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
            "Applied %d measurements.", applied_lines);
   rv = 0;
 
+ continue_digest:
+  /* Continue parsing lines to return the digest of the Bandwidth File. */
+  while (!feof(fp)) {
+    if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
+    }
+  }
+
  err:
   if (line) {
     // we need to raw_free this buffer because we got it from tor_getdelim()
@@ -312,6 +347,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
   }
   if (fp)
     fclose(fp);
+  if (digest_out)
+    crypto_digest_get_digest(digest, (char *) digest_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
   return rv;
 }
 
@@ -327,6 +365,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
  * the header block yet. If we encounter an incomplete bw line, return -1 but
  * don't warn since there could be additional header lines coming. If we
  * encounter a proper bw line, return 0 (and we got past the headers).
+ *
+ * If the line contains "vote=0", stop parsing it, and return -1, so that the
+ * line is ignored during voting.
  */
 STATIC int
 measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h
index 4507728458..81c8affbd7 100644
--- a/src/feature/dirauth/bwauth.h
+++ b/src/feature/dirauth/bwauth.h
@@ -21,8 +21,8 @@
 
 int dirserv_read_measured_bandwidths(const char *from_file,
                                      smartlist_t *routerstatuses,
-                                     smartlist_t *bw_file_headers);
-
+                                     smartlist_t *bw_file_headers,
+                                     uint8_t *digest_out);
 int dirserv_query_measured_bw_cache_kb(const char *node_id,
                                        long *bw_out,
                                        time_t *as_of_out);
@@ -55,4 +55,4 @@ STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
 STATIC void dirserv_expire_measured_bw_cache(time_t now);
 #endif /* defined(BWAUTH_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_BWAUTH_H) */
diff --git a/src/feature/dirauth/dirauth_periodic.c b/src/feature/dirauth/dirauth_periodic.c
new file mode 100644
index 0000000000..02727d61b4
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.c
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "app/config/or_options_st.h"
+#include "core/mainloop/netstatus.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/mainloop/periodic.h"
+
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: if we're an authority, check on our authority
+ * certificate (the one that authenticates our authority signing key).
+ */
+static int
+check_authority_cert_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  (void)options;
+  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
+   * close to expiring and warn the admin if it is. */
+  v3_authority_check_key_expiry();
+#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
+  return CHECK_V3_CERTIFICATE_INTERVAL;
+}
+
+DECLARE_EVENT(check_authority_cert, DIRAUTH, 0);
+
+/**
+ * Scheduled callback: Run directory-authority voting functionality.
+ *
+ * The schedule is a bit complicated here, so dirvote_act() manages the
+ * schedule itself.
+ **/
+static int
+dirvote_callback(time_t now, const or_options_t *options)
+{
+  if (!authdir_mode_v3(options)) {
+    tor_assert_nonfatal_unreached();
+    return 3600;
+  }
+
+  time_t next = dirvote_act(options, now);
+  if (BUG(next == TIME_MAX)) {
+    /* This shouldn't be returned unless we called dirvote_act() without
+     * being an authority.  If it happens, maybe our configuration will
+     * fix itself in an hour or so? */
+    return 3600;
+  }
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(dirvote, DIRAUTH, FL(NEED_NET));
+
+/** Reschedule the directory-authority voting event.  Run this whenever the
+ * schedule has changed. */
+void
+reschedule_dirvote(const or_options_t *options)
+{
+  if (authdir_mode_v3(options)) {
+    periodic_event_reschedule(&dirvote_event);
+  }
+}
+
+/**
+ * Periodic callback: if we're an authority, record our measured stability
+ * information from rephist in an mtbf file.
+ */
+static int
+save_stability_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options)) {
+    if (rep_hist_record_mtbf_data(now, 1)<0) {
+      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+    }
+  }
+#define SAVE_STABILITY_INTERVAL (30*60)
+  return SAVE_STABILITY_INTERVAL;
+}
+
+DECLARE_EVENT(save_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're an authority, make sure we test
+ * the routers on the network for reachability.
+ */
+static int
+launch_reachability_tests_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options) &&
+      !net_is_disabled()) {
+    /* try to determine reachability of the other Tor relays */
+    dirserv_test_reachability(now);
+  }
+  return REACHABILITY_TEST_INTERVAL;
+}
+
+DECLARE_EVENT(launch_reachability_tests, AUTHORITIES, FL(NEED_NET));
+
+/**
+ * Periodic callback: if we're an authority, discount the stability
+ * information (and other rephist information) that's older.
+ */
+static int
+downrate_stability_callback(time_t now, const or_options_t *options)
+{
+  (void)options;
+  /* 1d. Periodically, we discount older stability information so that new
+   * stability info counts more, and save the stability information to disk as
+   * appropriate. */
+  time_t next = rep_hist_downrate_old_runs(now);
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(downrate_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're the bridge authority, write a networkstatus
+ * file to disk.
+ */
+static int
+write_bridge_ns_callback(time_t now, const or_options_t *options)
+{
+  if (options->BridgeAuthoritativeDir) {
+    bridgeauth_dump_bridge_status_to_file(now);
+#define BRIDGE_STATUSFILE_INTERVAL (30*60)
+     return BRIDGE_STATUSFILE_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(write_bridge_ns, BRIDGEAUTH, 0);
+
+void
+dirauth_register_periodic_events(void)
+{
+  periodic_events_register(&downrate_stability_event);
+  periodic_events_register(&launch_reachability_tests_event);
+  periodic_events_register(&save_stability_event);
+  periodic_events_register(&check_authority_cert_event);
+  periodic_events_register(&dirvote_event);
+  periodic_events_register(&write_bridge_ns_event);
+}
diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h
new file mode 100644
index 0000000000..866fbd35de
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIRVOTE_PERIODIC_H
+#define DIRVOTE_PERIODIC_H
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+void dirauth_register_periodic_events(void);
+void reschedule_dirvote(const or_options_t *options);
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+static inline void
+reschedule_dirvote(const or_options_t *options)
+{
+  (void)options;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(DIRVOTE_PERIODIC_H) */
diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c
new file mode 100644
index 0000000000..e38d391300
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.c
@@ -0,0 +1,40 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_dirauth_initialize(void)
+{
+  dirauth_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_dirauth_shutdown(void)
+{
+  dirserv_free_fingerprint_list();
+  dirvote_free_all();
+  dirserv_clear_measured_bw_cache();
+  keypin_close_journal();
+}
+
+const struct subsys_fns_t sys_dirauth = {
+  .name = "dirauth",
+  .supported = true,
+  .level = 70,
+  .initialize = subsys_dirauth_initialize,
+  .shutdown = subsys_dirauth_shutdown,
+};
diff --git a/src/feature/dirauth/dirauth_sys.h b/src/feature/dirauth/dirauth_sys.h
new file mode 100644
index 0000000000..4e9b6a2ab4
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef DIRAUTH_SYS_H
+#define DIRAUTH_SYS_H
+
+extern const struct subsys_fns_t sys_dirauth;
+
+#endif /* !defined(DIRAUTH_SYS_H) */
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
index af8b3dc207..043bbfc227 100644
--- a/src/feature/dirauth/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -28,6 +28,7 @@
 #include "feature/nodelist/fmt_routerstatus.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -60,6 +61,9 @@
 #include "lib/encoding/confline.h"
 #include "lib/crypt_ops/crypto_format.h"
 
+/* Algorithm to use for the bandwidth file digest. */
+#define DIGEST_ALG_BW_FILE DIGEST_SHA256
+
 /**
  * \file dirvote.c
  * \brief Functions to compute directory consensus, and schedule voting.
@@ -216,7 +220,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                           networkstatus_t *v3_ns)
 {
   smartlist_t *chunks = smartlist_new();
-  char *packages = NULL;
   char fingerprint[FINGERPRINT_LEN+1];
   char digest[DIGEST_LEN];
   uint32_t addr;
@@ -242,19 +245,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                                                 v3_ns->server_versions);
   protocols_lines = format_protocols_lines_for_vote(v3_ns);
 
-  if (v3_ns->package_lines) {
-    smartlist_t *tmp = smartlist_new();
-    SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p,
-                      if (validate_recommended_package_line(p))
-                        smartlist_add_asprintf(tmp, "package %s\n", p));
-    smartlist_sort_strings(tmp);
-    packages = smartlist_join_strings(tmp, "", 0, NULL);
-    SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp));
-    smartlist_free(tmp);
-  } else {
-    packages = tor_strdup("");
-  }
-
     /* Get shared random commitments/reveals line(s). */
   shared_random_vote_str = sr_get_string_for_vote();
 
@@ -268,6 +258,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     char *flag_thresholds = dirserv_get_flag_thresholds_line();
     char *params;
     char *bw_headers_line = NULL;
+    char *bw_file_digest = NULL;
     authority_cert_t *cert = v3_ns->cert;
     char *methods =
       make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
@@ -307,6 +298,27 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
       tor_free(bw_file_headers);
     }
 
+    /* Create bandwidth-file-digest if applicable.
+     * v3_ns->b64_digest_bw_file will contain the digest when V3BandwidthsFile
+     * is configured and the bandwidth file could be read, even if it was not
+     * parseable.
+     */
+    if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) {
+      /* Encode the digest. */
+      char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0};
+      digest256_to_base64(b64_digest_bw_file,
+                          (const char *)v3_ns->bw_file_digest256);
+      /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */
+      char *digest_algo_b64_digest_bw_file = NULL;
+      tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s",
+                   crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE),
+                   b64_digest_bw_file);
+      /* No need for tor_strdup(""), format_line_if_present does it. */
+      bw_file_digest = format_line_if_present(
+          "bandwidth-file-digest", digest_algo_b64_digest_bw_file);
+      tor_free(digest_algo_b64_digest_bw_file);
+    }
+
     smartlist_add_asprintf(chunks,
                  "network-status-version 3\n"
                  "vote-status %s\n"
@@ -318,11 +330,11 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  "voting-delay %d %d\n"
                  "%s%s" /* versions */
                  "%s" /* protocols */
-                 "%s" /* packages */
                  "known-flags %s\n"
                  "flag-thresholds %s\n"
                  "params %s\n"
                  "%s" /* bandwidth file headers */
+                 "%s" /* bandwidth file digest */
                  "dir-source %s %s %s %s %d %d\n"
                  "contact %s\n"
                  "%s" /* shared randomness information */
@@ -334,11 +346,11 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                  client_versions_line,
                  server_versions_line,
                  protocols_lines,
-                 packages,
                  flags,
                  flag_thresholds,
                  params,
                  bw_headers_line ? bw_headers_line : "",
+                 bw_file_digest ? bw_file_digest: "",
                  voter->nickname, fingerprint, voter->address,
                  fmt_addr32(addr), voter->dir_port, voter->or_port,
                  voter->contact,
@@ -351,6 +363,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     tor_free(methods);
     tor_free(shared_random_vote_str);
     tor_free(bw_headers_line);
+    tor_free(bw_file_digest);
 
     if (!tor_digest_is_zero(voter->legacy_id_digest)) {
       char fpbuf[HEX_DIGEST_LEN+1];
@@ -412,7 +425,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
 
   {
     networkstatus_t *v;
-    if (!(v = networkstatus_parse_vote_from_string(status, NULL,
+    if (!(v = networkstatus_parse_vote_from_string(status, strlen(status),
+                                                   NULL,
                                                    v3_ns->type))) {
       log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: "
               "<<%s>>",
@@ -430,7 +444,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
   tor_free(client_versions_line);
   tor_free(server_versions_line);
   tor_free(protocols_lines);
-  tor_free(packages);
 
   SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
   smartlist_free(chunks);
@@ -2409,7 +2422,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
   {
     networkstatus_t *c;
-    if (!(c = networkstatus_parse_vote_from_string(result, NULL,
+    if (!(c = networkstatus_parse_vote_from_string(result, strlen(result),
+                                                   NULL,
                                                    NS_TYPE_CONSENSUS))) {
       log_err(LD_BUG, "Generated a networkstatus consensus we couldn't "
               "parse.");
@@ -2567,7 +2581,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target,
       return -1;
     }
     for (alg = DIGEST_SHA1; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) {
-      if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
+      if (!fast_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
         if (fast_memeq(target->digests.d[alg], digests->d[alg],
                        DIGEST256_LEN)) {
           ++n_matches;
@@ -2763,7 +2777,7 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses)
       char d[HEX_DIGEST256_LEN+1];
       const char *alg_name =
         crypto_digest_algorithm_get_name(alg);
-      if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
+      if (fast_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
         continue;
       base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN);
       smartlist_add_asprintf(elements, "additional-digest %s %s %s\n",
@@ -3132,7 +3146,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
   *msg_out = NULL;
 
  again:
-  vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote,
+  vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body),
+                                              &end_of_vote,
                                               NS_TYPE_VOTE);
   if (!end_of_vote)
     end_of_vote = vote_body + strlen(vote_body);
@@ -3390,7 +3405,9 @@ dirvote_compute_consensuses(void)
                  flavor_name);
         continue;
       }
-      consensus = networkstatus_parse_vote_from_string(consensus_body, NULL,
+      consensus = networkstatus_parse_vote_from_string(consensus_body,
+                                                       strlen(consensus_body),
+                                                       NULL,
                                                        NS_TYPE_CONSENSUS);
       if (!consensus) {
         log_warn(LD_DIR, "Couldn't parse %s consensus we generated!",
@@ -3529,7 +3546,7 @@ dirvote_add_signatures_to_pending_consensus(
      * just in case we break detached signature processing at some point. */
     {
       networkstatus_t *v = networkstatus_parse_vote_from_string(
-                                             pc->body, NULL,
+                                             pc->body, strlen(pc->body), NULL,
                                              NS_TYPE_CONSENSUS);
       tor_assert(v);
       networkstatus_vote_free(v);
@@ -3654,7 +3671,9 @@ dirvote_publish_consensus(void)
       continue;
     }
 
-    if (networkstatus_set_current_consensus(pending->body, name, 0, NULL))
+    if (networkstatus_set_current_consensus(pending->body,
+                                            strlen(pending->body),
+                                            name, 0, NULL))
       log_warn(LD_DIR, "Error publishing %s consensus", name);
     else
       log_notice(LD_DIR, "Published %s consensus", name);
@@ -3791,8 +3810,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     smartlist_add_asprintf(chunks, "a %s\n",
                            fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
 
-  if (family)
-    smartlist_add_asprintf(chunks, "family %s\n", family);
+  if (family) {
+    if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) {
+      smartlist_add_asprintf(chunks, "family %s\n", family);
+    } else {
+      const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest;
+      char *canonical_family = nodefamily_canonicalize(family, id, 0);
+      smartlist_add_asprintf(chunks, "family %s\n", canonical_family);
+      tor_free(canonical_family);
+    }
+  }
 
   if (summary && strcmp(summary, "reject 1-65535"))
     smartlist_add_asprintf(chunks, "p %s\n", summary);
@@ -3869,8 +3896,7 @@ dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len,
                                ",");
   tor_assert(microdesc_consensus_methods);
 
-  if (digest256_to_base64(d64, md->digest)<0)
-    goto out;
+  digest256_to_base64(d64, md->digest);
 
   if (tor_snprintf(out_buf, out_buf_len, "m %s sha256=%s\n",
                    microdesc_consensus_methods, d64)<0)
@@ -3890,7 +3916,10 @@ static const struct consensus_method_range_t {
   int high;
 } microdesc_consensus_methods[] = {
   {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1},
-  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD},
+  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC,
+   MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1},
+  {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS,
+   MAX_SUPPORTED_CONSENSUS_METHOD},
   {-1, -1}
 };
 
@@ -4364,6 +4393,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs)
    * forget to add it to this clause. */
 }
 
+/** Space-separated list of all the flags that we will always vote on. */
+const char DIRVOTE_UNIVERSAL_FLAGS[] =
+  "Authority "
+  "Exit "
+  "Fast "
+  "Guard "
+  "HSDir "
+  "Stable "
+  "StaleDesc "
+  "V2Dir "
+  "Valid";
+/** Space-separated list of all flags that we may or may not vote on,
+ * depending on our configuration. */
+const char DIRVOTE_OPTIONAL_FLAGS[] =
+  "BadExit "
+  "Running";
+
 /** Return a new networkstatus_t* containing our current opinion. (For v3
  * authorities) */
 networkstatus_t *
@@ -4388,6 +4434,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   const int vote_on_reachability = running_long_enough_to_decide_unreachable();
   smartlist_t *microdescriptors = NULL;
   smartlist_t *bw_file_headers = NULL;
+  uint8_t bw_file_digest256[DIGEST256_LEN] = {0};
 
   tor_assert(private_key);
   tor_assert(cert);
@@ -4425,7 +4472,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
    * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
    */
   if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                     NULL);
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4479,8 +4527,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
       vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       rs = &vrs->status;
-      set_routerstatus_from_routerinfo(rs, node, ri, now,
-                                       listbadexits);
+      dirauth_set_routerstatus_from_routerinfo(rs, node, ri, now,
+                                               listbadexits);
 
       if (ri->cache_info.signing_key_cert) {
         memcpy(vrs->ed25519_id,
@@ -4530,7 +4578,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     /* Only set bw_file_headers when V3BandwidthsFile is configured */
     bw_file_headers = smartlist_new();
     dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
-                                     routerstatuses, bw_file_headers);
+                                     routerstatuses, bw_file_headers,
+                                     bw_file_digest256);
+
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4601,18 +4651,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   tor_assert_nonfatal(protover_all_supported(
                                v3_out->recommended_client_protocols, NULL));
 
-  v3_out->package_lines = smartlist_new();
-  {
-    config_line_t *cl;
-    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
-      if (validate_recommended_package_line(cl->value))
-        smartlist_add_strdup(v3_out->package_lines, cl->value);
-    }
-  }
-
   v3_out->known_flags = smartlist_new();
   smartlist_split_string(v3_out->known_flags,
-                         "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   if (vote_on_reachability)
     smartlist_add_strdup(v3_out->known_flags, "Running");
@@ -4627,6 +4668,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     smartlist_sort_strings(v3_out->net_params);
   }
   v3_out->bw_file_headers = bw_file_headers;
+  memcpy(v3_out->bw_file_digest256, bw_file_digest256, DIGEST256_LEN);
 
   voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
   voter->nickname = tor_strdup(options->Nickname);
diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h
index 02d88d19d1..b7df33a3a9 100644
--- a/src/feature/dirauth/dirvote.h
+++ b/src/feature/dirauth/dirvote.h
@@ -57,7 +57,7 @@
 #define MIN_SUPPORTED_CONSENSUS_METHOD 25
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 28
+#define MAX_SUPPORTED_CONSENSUS_METHOD 29
 
 /** Lowest consensus method where authorities vote on required/recommended
  * protocols. */
@@ -79,6 +79,12 @@
  * addresses. See #23828 and #20916. */
 #define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28
 
+/**
+ * Lowest consensus method where microdescriptor lines are put in canonical
+ * form for improved compressibility and ease of storage. See proposal 298.
+ **/
+#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29
+
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
  * get confused with the above macros.) */
@@ -92,6 +98,9 @@
 /** Maximum size of a line in a vote. */
 #define MAX_BW_FILE_HEADERS_LINE_LEN 1024
 
+extern const char DIRVOTE_UNIVERSAL_FLAGS[];
+extern const char DIRVOTE_OPTIONAL_FLAGS[];
+
 /*
  * Public API. Used outside of the dirauth subsystem.
  *
@@ -119,7 +128,7 @@ struct config_line_t;
 char *format_recommended_version_list(const struct config_line_t *line,
                                       int warn);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline time_t
 dirvote_act(const or_options_t *options, time_t now)
@@ -184,7 +193,7 @@ dirvote_add_signatures(const char *detached_signatures_body,
   return 0;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Item access */
 MOCK_DECL(const char*, dirvote_get_pending_consensus,
diff --git a/src/feature/dirauth/dsigs_parse.c b/src/feature/dirauth/dsigs_parse.c
index d88176fee9..c5c8e18866 100644
--- a/src/feature/dirauth/dsigs_parse.c
+++ b/src/feature/dirauth/dsigs_parse.c
@@ -127,7 +127,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos)
     }
     digests = detached_get_digests(sigs, flavor);
     tor_assert(digests);
-    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
+    if (!fast_mem_is_zero(digests->d[alg], digest_length)) {
       log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
                "signatures document", flavor, algname);
       continue;
diff --git a/src/feature/dirauth/dsigs_parse.h b/src/feature/dirauth/dsigs_parse.h
index fec51ba488..0cc53072f8 100644
--- a/src/feature/dirauth/dsigs_parse.h
+++ b/src/feature/dirauth/dsigs_parse.h
@@ -19,4 +19,4 @@ void ns_detached_signatures_free_(ns_detached_signatures_t *s);
 #define ns_detached_signatures_free(s) \
   FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
 
-#endif
+#endif /* !defined(TOR_DSIGS_PARSE_H) */
diff --git a/src/feature/dirauth/guardfraction.h b/src/feature/dirauth/guardfraction.h
index 72404907a4..9f01ded838 100644
--- a/src/feature/dirauth/guardfraction.h
+++ b/src/feature/dirauth/guardfraction.h
@@ -21,4 +21,4 @@ dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
 int dirserv_read_guardfraction_file(const char *fname,
                                  smartlist_t *vote_routerstatuses);
 
-#endif
+#endif /* !defined(TOR_GUARDFRACTION_H) */
diff --git a/src/feature/dirauth/keypin.c b/src/feature/dirauth/keypin.c
index 06cb9ba1ff..316b7d6c2f 100644
--- a/src/feature/dirauth/keypin.c
+++ b/src/feature/dirauth/keypin.c
@@ -438,7 +438,7 @@ keypin_load_journal_impl(const char *data, size_t size)
   tor_log(severity, LD_DIRSERV,
           "Loaded %d entries from keypin journal. "
           "Found %d corrupt lines (ignored), %d duplicates (harmless), "
-          "and %d conflicts (resolved in favor or more recent entry).",
+          "and %d conflicts (resolved in favor of more recent entry).",
           n_entries, n_corrupt_lines, n_duplicates, n_conflicts);
 
   return 0;
diff --git a/src/feature/dirauth/keypin.h b/src/feature/dirauth/keypin.h
index 722b6ca5fc..1de84f6d4a 100644
--- a/src/feature/dirauth/keypin.h
+++ b/src/feature/dirauth/keypin.h
@@ -11,10 +11,25 @@ int keypin_check_and_add(const uint8_t *rsa_id_digest,
                          const int replace_existing_entry);
 int keypin_check(const uint8_t *rsa_id_digest,
                  const uint8_t *ed25519_id_key);
+int keypin_close_journal(void);
 
+#ifdef HAVE_MODULE_DIRAUTH
 int keypin_open_journal(const char *fname);
-int keypin_close_journal(void);
 int keypin_load_journal(const char *fname);
+#else
+static inline int
+keypin_open_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+static inline int
+keypin_load_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 void keypin_clear(void);
 int keypin_check_lone_rsa(const uint8_t *rsa_id_digest);
 
@@ -44,4 +59,3 @@ MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent));
 #endif /* defined(KEYPIN_PRIVATE) */
 
 #endif /* !defined(TOR_KEYPIN_H) */
-
diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h
index 0f92be2f0d..61d20b7525 100644
--- a/src/feature/dirauth/ns_detached_signatures_st.h
+++ b/src/feature/dirauth/ns_detached_signatures_st.h
@@ -18,5 +18,5 @@ struct ns_detached_signatures_t {
                          * document_signature_t */
 };
 
-#endif
+#endif /* !defined(NS_DETACHED_SIGNATURES_ST_H) */
 
diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c
index 21b8e239ec..71e3195c01 100644
--- a/src/feature/dirauth/process_descs.c
+++ b/src/feature/dirauth/process_descs.c
@@ -216,9 +216,14 @@ dirserv_load_fingerprint_file(void)
 
 #define DISABLE_DISABLING_ED25519
 
-/** Check whether <b>router</b> has a nickname/identity key combination that
- * we recognize from the fingerprint list, or an IP we automatically act on
- * according to our configuration.  Return the appropriate router status.
+/** Check whether <b>router</b> has:
+ * - a nickname/identity key combination that we recognize from the fingerprint
+ *   list,
+ * - an IP we automatically act on according to our configuration,
+ * - an appropriate version, and
+ * - matching pinned keys.
+ *
+ * Return the appropriate router status.
  *
  * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
  * *<b>msg</b> to an explanation of why. */
@@ -236,7 +241,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
     return FP_REJECT;
   }
 
-  /* Check for the more usual versions to reject a router first. */
+  /* Check for the more common reasons to reject a router first. */
   const uint32_t r = dirserv_get_status_impl(d, router->nickname,
                                              router->addr, router->or_port,
                                              router->platform, msg, severity);
@@ -310,6 +315,47 @@ dirserv_would_reject_router(const routerstatus_t *rs)
   return (res & FP_REJECT) != 0;
 }
 
+/**
+ * Check whether the platform string in <b>platform</b> describes a platform
+ * that, as a directory authority, we want to reject.  If it does, return
+ * true, and set *<b>msg</b> (if present) to a rejection message.  Otherwise
+ * return false.
+ */
+STATIC bool
+dirserv_rejects_tor_version(const char *platform,
+                            const char **msg)
+{
+  if (!platform)
+    return false;
+
+  static const char please_upgrade_string[] =
+    "Tor version is insecure or unsupported. Please upgrade!";
+
+  /* Versions before Tor 0.2.9 are unsupported. Versions between 0.2.9.0 and
+   * 0.2.9.4 suffer from bug #20499, where relays don't keep their consensus
+   * up to date */
+  if (!tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
+    if (msg)
+      *msg = please_upgrade_string;
+    return true;
+  }
+
+  /* Series between Tor 0.3.0 and 0.3.4 inclusive are unsupported, and some
+   * have bug #27841, which makes them broken as intro points. Reject them.
+   *
+   * Also reject unstable versions of 0.3.5, since (as of this writing)
+   * they are almost none of the network. */
+  if (tor_version_as_new_as(platform,"0.3.0.0-alpha-dev") &&
+      !tor_version_as_new_as(platform,"0.3.5.7")) {
+    if (msg) {
+      *msg = please_upgrade_string;
+    }
+    return true;
+  }
+
+  return false;
+}
+
 /** Helper: As dirserv_router_get_status, but takes the router fingerprint
  * (hex, no spaces), nickname, address (used for logging only), IP address, OR
  * port and platform (logging only) as arguments.
@@ -342,22 +388,8 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
     }
   }
 
-  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
-   * missing some important security fixes too. Disable them. */
-  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
-    if (msg)
-      *msg = "Tor version is insecure or unsupported. Please upgrade!";
-    return FP_REJECT;
-  }
-
-  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
-   * keep their consensus up to date so they make bad guards.
-   * The simple fix is to just drop them from the network. */
-  if (platform &&
-      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
-      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
-    if (msg)
-      *msg = "Tor version contains bug 20499. Please upgrade!";
+  /* Check whether the version is obsolete, broken, insecure, etc... */
+  if (platform && dirserv_rejects_tor_version(platform, msg)) {
     return FP_REJECT;
   }
 
@@ -423,20 +455,32 @@ dirserv_free_fingerprint_list(void)
 
 /** Return -1 if <b>ri</b> has a private or otherwise bad address,
  * unless we're configured to not care. Return 0 if all ok. */
-static int
+STATIC int
 dirserv_router_has_valid_address(routerinfo_t *ri)
 {
   tor_addr_t addr;
+
   if (get_options()->DirAllowPrivateAddresses)
     return 0; /* whatever it is, we're fine with it */
+
   tor_addr_from_ipv4h(&addr, ri->addr);
+  if (tor_addr_is_null(&addr) || tor_addr_is_internal(&addr, 0)) {
+    log_info(LD_DIRSERV,
+             "Router %s published internal IPv4 address. Refusing.",
+             router_describe(ri));
+    return -1; /* it's a private IP, we should reject it */
+  }
 
-  if (tor_addr_is_internal(&addr, 0)) {
+  /* We only check internal v6 on non-null addresses because we do not require
+   * IPv6 and null IPv6 is normal. */
+  if (!tor_addr_is_null(&ri->ipv6_addr) &&
+      tor_addr_is_internal(&ri->ipv6_addr, 0)) {
     log_info(LD_DIRSERV,
-             "Router %s published internal IP address. Refusing.",
+             "Router %s published internal IPv6 address. Refusing.",
              router_describe(ri));
     return -1; /* it's a private IP, we should reject it */
   }
+
   return 0;
 }
 
@@ -519,7 +563,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
 /** As for dirserv_add_descriptor(), but accepts multiple documents, and
  * returns the most severe error that occurred for any one of them. */
 was_router_added_t
-dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
                                  const char *source,
                                  const char **msg)
 {
@@ -534,7 +579,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   int general = purpose == ROUTER_PURPOSE_GENERAL;
   tor_assert(msg);
 
-  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
+  r=ROUTER_ADDED_SUCCESSFULLY; /* Least severe return value. */
+
+  if (!string_is_utf8_no_bom(desc, desclen)) {
+    *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM.";
+    return ROUTER_AUTHDIR_REJECTS;
+  }
 
   format_iso_time(time_buf, now);
   if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
@@ -545,14 +595,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
                    !general ? router_purpose_to_string(purpose) : "",
                    !general ? "\n" : "")<0) {
     *msg = "Couldn't format annotations";
-    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
-     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
-    return -1;
+    return ROUTER_AUTHDIR_BUG_ANNOTATIONS;
   }
 
   s = desc;
   list = smartlist_new();
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0,
                                      annotation_buf, NULL)) {
     SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
         msg_out = NULL;
@@ -568,7 +616,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   smartlist_clear(list);
 
   s = desc;
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0,
                                      NULL, NULL)) {
     SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
         msg_out = NULL;
diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h
index ae2d6ad25d..e504daa7b7 100644
--- a/src/feature/dirauth/process_descs.h
+++ b/src/feature/dirauth/process_descs.h
@@ -12,27 +12,107 @@
 #ifndef TOR_RECV_UPLOADS_H
 #define TOR_RECV_UPLOADS_H
 
-int dirserv_load_fingerprint_file(void);
+// for was_router_added_t.
+#include "feature/nodelist/routerlist.h"
+
 void dirserv_free_fingerprint_list(void);
-int dirserv_add_own_fingerprint(crypto_pk_t *pk);
 
+#ifdef HAVE_MODULE_DIRAUTH
+int dirserv_load_fingerprint_file(void);
 enum was_router_added_t dirserv_add_multiple_descriptors(
-                                     const char *desc, uint8_t purpose,
+                                     const char *desc, size_t desclen,
+                                     uint8_t purpose,
                                      const char *source,
                                      const char **msg);
 enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
                                                const char **msg,
                                                const char *source);
 
+int dirserv_would_reject_router(const routerstatus_t *rs);
 int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
                                    int complain,
                                    int *valid_out);
+int dirserv_add_own_fingerprint(crypto_pk_t *pk);
 uint32_t dirserv_router_get_status(const routerinfo_t *router,
                                    const char **msg,
                                    int severity);
 void dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                       uint32_t authstatus);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+static inline int
+dirserv_load_fingerprint_file(void)
+{
+  return 0;
+}
+static inline enum was_router_added_t
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
+                                 const char *source,
+                                 const char **msg)
+{
+  (void)desc;
+  (void)desclen;
+  (void)purpose;
+  (void)source;
+  (void)msg;
+  return (enum was_router_added_t)0;
+}
+static inline enum was_router_added_t
+dirserv_add_descriptor(routerinfo_t *ri,
+                       const char **msg,
+                       const char *source)
+{
+  (void)ri;
+  (void)msg;
+  (void)source;
+  return (enum was_router_added_t)0;
+}
+static inline int
+dirserv_would_reject_router(const routerstatus_t *rs)
+{
+  (void)rs;
+  return 0;
+}
+static inline int
+authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                               int complain,
+                               int *valid_out)
+{
+  (void)ri;
+  (void)msg;
+  (void)complain;
+  (void)valid_out;
+  return 0;
+}
+static inline int
+dirserv_add_own_fingerprint(crypto_pk_t *pk)
+{
+  (void)pk;
+  return 0;
+}
+static inline uint32_t
+dirserv_router_get_status(const routerinfo_t *router,
+                          const char **msg,
+                          int severity)
+{
+  (void)router;
+  (void)msg;
+  (void)severity;
+  return 0;
+}
+static inline void
+dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                 uint32_t authstatus)
+{
+  (void)node;
+  (void)authstatus;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-int dirserv_would_reject_router(const routerstatus_t *rs);
+#ifdef TOR_UNIT_TESTS
+STATIC int dirserv_router_has_valid_address(routerinfo_t *ri);
+STATIC bool dirserv_rejects_tor_version(const char *platform,
+                                        const char **msg);
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_RECV_UPLOADS_H) */
diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h
index 5a938673ff..46d0e7ee2e 100644
--- a/src/feature/dirauth/reachability.h
+++ b/src/feature/dirauth/reachability.h
@@ -24,13 +24,36 @@
 #define REACHABILITY_TEST_CYCLE_PERIOD \
   (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
 
+void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
+void dirserv_test_reachability(time_t now);
+
+#ifdef HAVE_MODULE_DIRAUTH
+int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old);
 void dirserv_orconn_tls_done(const tor_addr_t *addr,
                              uint16_t or_port,
                              const char *digest_rcvd,
                              const struct ed25519_public_key_t *ed_id_rcvd);
-int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
-                                            const routerinfo_t *ri_old);
-void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
-void dirserv_test_reachability(time_t now);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+static inline int
+dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old)
+{
+  (void)ri;
+  (void)ri_old;
+  return 0;
+}
+static inline void
+dirserv_orconn_tls_done(const tor_addr_t *addr,
+                        uint16_t or_port,
+                        const char *digest_rcvd,
+                        const struct ed25519_public_key_t *ed_id_rcvd)
+{
+  (void)addr;
+  (void)or_port;
+  (void)digest_rcvd;
+  (void)ed_id_rcvd;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif
+#endif /* !defined(TOR_REACHABILITY_H) */
diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h
index 8200d78f72..af17e945e8 100644
--- a/src/feature/dirauth/recommend_pkg.h
+++ b/src/feature/dirauth/recommend_pkg.h
@@ -12,6 +12,18 @@
 #ifndef TOR_RECOMMEND_PKG_H
 #define TOR_RECOMMEND_PKG_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 int validate_recommended_package_line(const char *line);
 
-#endif
+#else
+
+static inline int
+validate_recommended_package_line(const char *line)
+{
+  (void) line;
+  return 0;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(TOR_RECOMMEND_PKG_H) */
diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c
index 34b2283250..a45f0a29c3 100644
--- a/src/feature/dirauth/shared_random.c
+++ b/src/feature/dirauth/shared_random.c
@@ -90,7 +90,7 @@
 #include "core/or/or.h"
 #include "feature/dirauth/shared_random.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
@@ -120,8 +120,8 @@ static const char sr_flag_ns_str[] = "shared-rand-participate";
 static int32_t num_srv_agreements_from_vote;
 
 /* Return a heap allocated copy of the SRV <b>orig</b>. */
-STATIC sr_srv_t *
-srv_dup(const sr_srv_t *orig)
+sr_srv_t *
+sr_srv_dup(const sr_srv_t *orig)
 {
   sr_srv_t *duplicate = NULL;
 
@@ -224,7 +224,7 @@ verify_commit_and_reveal(const sr_commit_t *commit)
 STATIC int
 commit_has_reveal_value(const sr_commit_t *commit)
 {
-  return !tor_mem_is_zero(commit->encoded_reveal,
+  return !fast_mem_is_zero(commit->encoded_reveal,
                           sizeof(commit->encoded_reveal));
 }
 
@@ -486,7 +486,7 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
   {
     /* Send a reveal value for this commit if we have one. */
     const char *reveal_str = commit->encoded_reveal;
-    if (tor_mem_is_zero(commit->encoded_reveal,
+    if (fast_mem_is_zero(commit->encoded_reveal,
                         sizeof(commit->encoded_reveal))) {
       reveal_str = "";
     }
@@ -1253,8 +1253,8 @@ sr_act_post_consensus(const networkstatus_t *consensus)
      * decided by the majority. */
     sr_state_unset_fresh_srv();
     /* Set the SR values from the given consensus. */
-    sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv));
-    sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv));
+    sr_state_set_previous_srv(sr_srv_dup(consensus->sr_info.previous_srv));
+    sr_state_set_current_srv(sr_srv_dup(consensus->sr_info.current_srv));
   }
 
   /* Prepare our state so that it's ready for the next voting period. */
diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h
index 25d95ebbc7..7ff9f15512 100644
--- a/src/feature/dirauth/shared_random.h
+++ b/src/feature/dirauth/shared_random.h
@@ -110,7 +110,7 @@ int sr_init(int save_to_disk);
 void sr_save_and_cleanup(void);
 void sr_act_post_consensus(const networkstatus_t *consensus);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline int
 sr_init(int save_to_disk)
@@ -131,7 +131,7 @@ sr_act_post_consensus(const networkstatus_t *consensus)
   (void) consensus;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Public methods used only by dirauth code. */
 
@@ -154,6 +154,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
 void sr_compute_srv(void);
 sr_commit_t *sr_generate_our_commit(time_t timestamp,
                                     const authority_cert_t *my_rsa_cert);
+sr_srv_t *sr_srv_dup(const sr_srv_t *orig);
 
 #ifdef SHARED_RANDOM_PRIVATE
 
@@ -172,7 +173,6 @@ STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes,
                                              int current);
 
 STATIC void save_commit_to_state(sr_commit_t *commit);
-STATIC sr_srv_t *srv_dup(const sr_srv_t *orig);
 STATIC int commitments_are_the_same(const sr_commit_t *commit_one,
                                     const sr_commit_t *commit_two);
 STATIC int commit_is_authoritative(const sr_commit_t *commit,
diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c
index b3e4a4ef92..4078d6a24a 100644
--- a/src/feature/dirauth/shared_random_state.c
+++ b/src/feature/dirauth/shared_random_state.c
@@ -12,7 +12,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/nodelist/networkstatus.h"
@@ -22,6 +22,7 @@
 #include "feature/dirauth/shared_random_state.h"
 #include "feature/dircommon/voting_schedule.h"
 #include "lib/encoding/confline.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
@@ -50,24 +51,21 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue";
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t);
 
-/* These next two are duplicates or near-duplicates from config.c */
-#define VAR(name, conftype, member, initvalue)                              \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member),      \
-      initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) }
-/* As VAR, but the option name and member name are the same. */
-#define V(member, conftype, initvalue) \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
+
 /* Our persistent state magic number. */
 #define SR_DISK_STATE_MAGIC 0x98AB1254
 
 static int
 disk_state_validate_cb(void *old_state, void *state, void *default_state,
                        int from_setconf, char **msg);
-static void disk_state_free_cb(void *);
 
 /* Array of variables that are saved to disk as a persistent state. */
-static config_var_t state_vars[] = {
-  V(Version,                    UINT, "0"),
+static const config_var_t state_vars[] = {
+  V(Version,                    POSINT, "0"),
   V(TorVersion,                 STRING, NULL),
   V(ValidAfter,                 ISOTIME, NULL),
   V(ValidUntil,                 ISOTIME, NULL),
@@ -82,25 +80,45 @@ static config_var_t state_vars[] = {
 
 /* "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST,
-  offsetof(sr_disk_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines)
+static const struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(sr_disk_state_t, ExtraLines),
 };
 
 /* Configuration format of sr_disk_state_t. */
 static const config_format_t state_format = {
   sizeof(sr_disk_state_t),
-  SR_DISK_STATE_MAGIC,
-  offsetof(sr_disk_state_t, magic_),
+  {
+   "sr_disk_state_t",
+   SR_DISK_STATE_MAGIC,
+   offsetof(sr_disk_state_t, magic_),
+  },
   NULL,
   NULL,
   state_vars,
   disk_state_validate_cb,
-  disk_state_free_cb,
+  NULL,
   &state_extra_var,
+  -1,
 };
 
+/* Global configuration manager for the shared-random state file */
+static config_mgr_t *shared_random_state_mgr = NULL;
+
+/** Return the configuration manager for the shared-random state file. */
+static const config_mgr_t *
+get_srs_mgr(void)
+{
+  if (PREDICT_UNLIKELY(shared_random_state_mgr == NULL)) {
+    shared_random_state_mgr = config_mgr_new(&state_format);
+    config_mgr_freeze(shared_random_state_mgr);
+  }
+  return shared_random_state_mgr;
+}
+
+static void state_query_del_(sr_state_object_t obj_type, void *data);
+
 /* Return a string representation of a protocol phase. */
 STATIC const char *
 get_phase_str(sr_phase_t phase)
@@ -260,23 +278,22 @@ disk_state_free_(sr_disk_state_t *state)
   if (state == NULL) {
     return;
   }
-  config_free(&state_format, state);
+  config_free(get_srs_mgr(), state);
 }
 
 /* Allocate a new disk state, initialize it and return it. */
 static sr_disk_state_t *
 disk_state_new(time_t now)
 {
-  sr_disk_state_t *new_state = tor_malloc_zero(sizeof(*new_state));
+  sr_disk_state_t *new_state = config_new(get_srs_mgr());
 
-  new_state->magic_ = SR_DISK_STATE_MAGIC;
   new_state->Version = SR_PROTO_VERSION;
   new_state->TorVersion = tor_strdup(get_version());
   new_state->ValidUntil = get_state_valid_until_time(now);
   new_state->ValidAfter = now;
 
   /* Init config format. */
-  config_init(&state_format, new_state);
+  config_init(get_srs_mgr(), new_state);
   return new_state;
 }
 
@@ -344,12 +361,6 @@ disk_state_validate_cb(void *old_state, void *state, void *default_state,
   return 0;
 }
 
-static void
-disk_state_free_cb(void *state)
-{
-  disk_state_free_(state);
-}
-
 /* Parse the Commit line(s) in the disk state and translate them to the
  * the memory state. Return 0 on success else -1 on error. */
 static int
@@ -535,7 +546,7 @@ disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
   tor_assert(commit);
   tor_assert(line);
 
-  if (!tor_mem_is_zero(commit->encoded_reveal,
+  if (!fast_mem_is_zero(commit->encoded_reveal,
                        sizeof(commit->encoded_reveal))) {
     /* Add extra whitespace so we can format the line correctly. */
     tor_asprintf(&reveal_str, " %s", commit->encoded_reveal);
@@ -580,11 +591,12 @@ disk_state_reset(void)
   config_free_lines(sr_disk_state->ExtraLines);
   tor_free(sr_disk_state->TorVersion);
 
-  /* Clean up the struct */
-  memset(sr_disk_state, 0, sizeof(*sr_disk_state));
+  /* Clear other fields. */
+  sr_disk_state->ValidAfter = 0;
+  sr_disk_state->ValidUntil = 0;
+  sr_disk_state->Version = 0;
 
   /* Reset it with useful data */
-  sr_disk_state->magic_ = SR_DISK_STATE_MAGIC;
   sr_disk_state->TorVersion = tor_strdup(get_version());
 }
 
@@ -679,7 +691,7 @@ disk_state_load_from_disk_impl(const char *fname)
     }
 
     disk_state = disk_state_new(time(NULL));
-    config_assign(&state_format, disk_state, lines, 0, &errmsg);
+    config_assign(get_srs_mgr(), disk_state, lines, 0, &errmsg);
     config_free_lines(lines);
     if (errmsg) {
       log_warn(LD_DIR, "SR: Reading state error: %s", errmsg);
@@ -732,7 +744,7 @@ disk_state_save_to_disk(void)
   /* Make sure that our disk state is up to date with our memory state
    * before saving it to disk. */
   disk_state_update();
-  state = config_dump(&state_format, NULL, sr_disk_state, 0, 0);
+  state = config_dump(get_srs_mgr(), NULL, sr_disk_state, 0, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&content,
                "# Tor shared random state file last generated on %s "
@@ -833,6 +845,9 @@ state_query_get_commit(const char *rsa_fpr)
 static void *
 state_query_get_(sr_state_object_t obj_type, const void *data)
 {
+  if (BUG(!sr_state))
+    return NULL;
+
   void *obj = NULL;
 
   switch (obj_type) {
@@ -861,23 +876,44 @@ state_query_get_(sr_state_object_t obj_type, const void *data)
 }
 
 /* Helper function: This handles the PUT state action using an
- * <b>obj_type</b> and <b>data</b> needed for the action. */
+ * <b>obj_type</b> and <b>data</b> needed for the action.
+ * PUT frees the previous data before replacing it, if needed. */
 static void
 state_query_put_(sr_state_object_t obj_type, void *data)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
     sr_commit_t *commit = data;
     tor_assert(commit);
+    /* commit_add_to_state() frees the old commit, if there is one */
     commit_add_to_state(commit, sr_state);
     break;
   }
   case SR_STATE_OBJ_CURSRV:
-    sr_state->current_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->current_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_CURSRV, NULL);
+      sr_state->current_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_PREVSRV:
-    sr_state->previous_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->previous_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_PREVSRV, NULL);
+      sr_state->previous_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_VALID_AFTER:
     sr_state->valid_after = *((time_t *) data);
@@ -897,6 +933,9 @@ state_query_put_(sr_state_object_t obj_type, void *data)
 static void
 state_query_del_all_(sr_state_object_t obj_type)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
@@ -907,7 +946,7 @@ state_query_del_all_(sr_state_object_t obj_type)
     } DIGESTMAP_FOREACH_END;
     break;
   }
-  /* The following object are _NOT_ suppose to be removed. */
+  /* The following objects are _NOT_ supposed to be removed. */
   case SR_STATE_OBJ_CURSRV:
   case SR_STATE_OBJ_PREVSRV:
   case SR_STATE_OBJ_PHASE:
@@ -925,6 +964,9 @@ state_query_del_(sr_state_object_t obj_type, void *data)
 {
   (void) data;
 
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_PREVSRV:
     tor_free(sr_state->previous_srv);
@@ -999,16 +1041,16 @@ state_del_previous_srv(void)
   state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL);
 }
 
-/* Rotate SRV value by freeing the previous value, assigning the current
- * value to the previous one and nullifying the current one. */
+/* Rotate SRV value by setting the previous SRV to the current SRV, and
+ * clearing the current SRV. */
 STATIC void
 state_rotate_srv(void)
 {
   /* First delete previous SRV from the state. Object will be freed. */
   state_del_previous_srv();
-  /* Set previous SRV with the current one. */
-  sr_state_set_previous_srv(sr_state_get_current_srv());
-  /* Nullify the current srv. */
+  /* Set previous SRV to a copy of the current one. */
+  sr_state_set_previous_srv(sr_srv_dup(sr_state_get_current_srv()));
+  /* Free and NULL the current srv. */
   sr_state_set_current_srv(NULL);
 }
 
@@ -1024,12 +1066,15 @@ sr_state_set_valid_after(time_t valid_after)
 sr_phase_t
 sr_state_get_phase(void)
 {
-  void *ptr;
+  void *ptr=NULL;
   state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PHASE, NULL, &ptr);
+  tor_assert(ptr);
   return *(sr_phase_t *) ptr;
 }
 
-/* Return the previous SRV value from our state. Value CAN be NULL. */
+/* Return the previous SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_previous_srv(void)
 {
@@ -1048,7 +1093,9 @@ sr_state_set_previous_srv(const sr_srv_t *srv)
               NULL);
 }
 
-/* Return the current SRV value from our state. Value CAN be NULL. */
+/* Return the current SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_current_srv(void)
 {
@@ -1240,6 +1287,7 @@ sr_state_free_all(void)
   /* Nullify our global state. */
   sr_state = NULL;
   sr_disk_state = NULL;
+  config_mgr_free(shared_random_state_mgr);
 }
 
 /* Save our current state in memory to disk. */
diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h
index 92acdf1157..7869f92b4f 100644
--- a/src/feature/dirauth/vote_microdesc_hash_st.h
+++ b/src/feature/dirauth/vote_microdesc_hash_st.h
@@ -18,5 +18,5 @@ struct vote_microdesc_hash_t {
   char *microdesc_hash_line;
 };
 
-#endif
+#endif /* !defined(VOTE_MICRODESC_HASH_ST_H) */
 
diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c
index 54c70b989a..f552af98c4 100644
--- a/src/feature/dirauth/voteflags.c
+++ b/src/feature/dirauth/voteflags.c
@@ -29,6 +29,7 @@
 
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
 #include "lib/container/order.h"
@@ -95,7 +96,7 @@ real_uptime(const routerinfo_t *router, time_t now)
  */
 static int
 dirserv_thinks_router_is_unreliable(time_t now,
-                                    routerinfo_t *router,
+                                    const routerinfo_t *router,
                                     int need_uptime, int need_capacity)
 {
   if (need_uptime) {
@@ -238,7 +239,7 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
   uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
   long *tks;
   double *mtbfs, *wfus;
-  smartlist_t *nodelist;
+  const smartlist_t *nodelist;
   time_t now = time(NULL);
   const or_options_t *options = get_options();
 
@@ -531,38 +532,45 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
   node->is_running = answer;
 }
 
-/** Extract status information from <b>ri</b> and from other authority
- * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
- * set.
- *
- * We assume that ri-\>is_running has already been set, e.g. by
- *   dirserv_set_router_is_running(ri, now);
+/* Check <b>node</b> and <b>ri</b> on whether or not we should publish a
+ * relay's IPv6 addresses. */
+static int
+should_publish_node_ipv6(const node_t *node, const routerinfo_t *ri,
+                         time_t now)
+{
+  const or_options_t *options = get_options();
+
+  return options->AuthDirHasIPv6Connectivity == 1 &&
+    !tor_addr_is_null(&ri->ipv6_addr) &&
+    ((node->last_reachable6 >= now - REACHABLE_TIMEOUT) ||
+     router_is_me(ri));
+}
+
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>, as per
+ * <b>set_routerstatus_from_routerinfo</b>.  Additionally, sets information
+ * in from the authority subsystem.
  */
 void
-set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                 node_t *node,
-                                 routerinfo_t *ri,
-                                 time_t now,
-                                 int listbadexits)
+dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                         node_t *node,
+                                         const routerinfo_t *ri,
+                                         time_t now,
+                                         int listbadexits)
 {
   const or_options_t *options = get_options();
   uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
 
-  memset(rs, 0, sizeof(routerstatus_t));
-
-  rs->is_authority =
-    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
-
-  /* Already set by compute_performance_thresholds. */
-  rs->is_exit = node->is_exit;
-  rs->is_stable = node->is_stable =
-    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
-  rs->is_fast = node->is_fast =
-    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
-  rs->is_flagged_running = node->is_running; /* computed above */
+  /* Set these flags so that set_routerstatus_from_routerinfo can copy them.
+   */
+  node->is_stable = !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
+  node->is_fast = !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
+  node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now);
 
-  rs->is_valid = node->is_valid;
+  set_routerstatus_from_routerinfo(rs, node, ri);
 
+  /* Override rs->is_possible_guard. */
   if (node->is_fast && node->is_stable &&
       ri->supports_tunnelled_dir_requests &&
       ((options->AuthDirGuardBWGuarantee &&
@@ -578,29 +586,16 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
     rs->is_possible_guard = 0;
   }
 
+  /* Override rs->is_bad_exit */
   rs->is_bad_exit = listbadexits && node->is_bad_exit;
-  rs->is_hs_dir = node->is_hs_dir =
-    dirserv_thinks_router_is_hs_dir(ri, node, now);
-
-  rs->is_named = rs->is_unnamed = 0;
-
-  rs->published_on = ri->cache_info.published_on;
-  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
-  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
-         DIGEST_LEN);
-  rs->addr = ri->addr;
-  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
-  rs->or_port = ri->or_port;
-  rs->dir_port = ri->dir_port;
-  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
-  if (options->AuthDirHasIPv6Connectivity == 1 &&
-      !tor_addr_is_null(&ri->ipv6_addr) &&
-      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
-    /* We're configured as having IPv6 connectivity. There's an IPv6
-       OR port and it's reachable so copy it to the routerstatus.  */
-    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
-    rs->ipv6_orport = ri->ipv6_orport;
-  } else {
+
+  /* Set rs->is_staledesc. */
+  rs->is_staledesc =
+    (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now;
+
+  if (! should_publish_node_ipv6(node, ri, now)) {
+    /* We're not configured as having IPv6 connectivity or the node isn't:
+     * zero its IPv6 information. */
     tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
     rs->ipv6_orport = 0;
   }
@@ -642,3 +637,20 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs)
     rs->is_hs_dir = 0;
   }
 }
+
+/** Use dirserv_set_router_is_running() to set bridges as running if they're
+ * reachable.
+ *
+ * This function is called from set_bridge_running_callback() when running as
+ * a bridge authority.
+ */
+void
+dirserv_set_bridges_running(time_t now)
+{
+  routerlist_t *rl = router_get_routerlist();
+
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+    if (ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      dirserv_set_router_is_running(ri, now);
+  } SMARTLIST_FOREACH_END(ri);
+}
diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h
index aa7b6ed082..c4f36e7817 100644
--- a/src/feature/dirauth/voteflags.h
+++ b/src/feature/dirauth/voteflags.h
@@ -12,20 +12,28 @@
 #ifndef TOR_VOTEFLAGS_H
 #define TOR_VOTEFLAGS_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
 char *dirserv_get_flag_thresholds_line(void);
 void dirserv_compute_bridge_flag_thresholds(void);
 int running_long_enough_to_decide_unreachable(void);
 
-void set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                      node_t *node,
-                                      routerinfo_t *ri, time_t now,
-                                      int listbadexits);
+void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                              node_t *node,
+                                              const routerinfo_t *ri,
+                                              time_t now,
+                                              int listbadexits);
 
 void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+void dirserv_set_bridges_running(time_t now);
 
 #ifdef VOTEFLAGS_PRIVATE
+/** Any descriptor older than this age causes the authorities to set the
+ * StaleDesc flag. */
+#define DESC_IS_STALE_INTERVAL (18*60*60)
 STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
-#endif
+#endif /* defined(VOTEFLAGS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_VOTEFLAGS_H) */
diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h
index 71dca8c3a2..a28802f905 100644
--- a/src/feature/dircache/cached_dir_st.h
+++ b/src/feature/dircache/cached_dir_st.h
@@ -21,5 +21,5 @@ struct cached_dir_t {
   int refcnt; /**< Reference count for this cached_dir_t. */
 };
 
-#endif
+#endif /* !defined(CACHED_DIR_ST_H) */
 
diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c
index cf4fe8701d..2ec9981c03 100644
--- a/src/feature/dircache/conscache.c
+++ b/src/feature/dircache/conscache.c
@@ -92,7 +92,7 @@ consensus_cache_open(const char *subdir, int max_entries)
    */
 #define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
   storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
-#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
+#else /* !defined(MUST_UNMAP_TO_UNLINK) */
   /* Otherwise, we can just tell the storagedir to use the same limits
    * as this cache. */
   storagedir_max_entries = max_entries;
diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c
index 025361fa60..397efa0341 100644
--- a/src/feature/dircache/consdiffmgr.c
+++ b/src/feature/dircache/consdiffmgr.c
@@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = {
 
 static int consdiffmgr_ensure_space_for_files(int n);
 static int consensus_queue_compression_work(const char *consensus,
+                                            size_t consensus_len,
                                             const networkstatus_t *as_parsed);
 static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
                                           consensus_cache_entry_t *diff_to);
@@ -509,8 +510,25 @@ get_max_age_to_cache(void)
                                         MAX_MAX_AGE_TO_CACHE);
 }
 
+#ifdef TOR_UNIT_TESTS
+/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For
+ * testing. */
+int
+consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                  const networkstatus_t *as_parsed)
+{
+  size_t len = strlen(consensus);
+  /* make a non-nul-terminated copy so that we can have a better chance
+   * of catching errors. */
+  char *ctmp = tor_memdup(consensus, len);
+  int r = consdiffmgr_add_consensus(ctmp, len, as_parsed);
+  tor_free(ctmp);
+  return r;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /**
- * Given a string containing a networkstatus consensus, and the results of
+ * Given a buffer containing a networkstatus consensus, and the results of
  * having parsed that consensus, add that consensus to the cache if it is not
  * already present and not too old.  Create new consensus diffs from or to
  * that consensus as appropriate.
@@ -519,6 +537,7 @@ get_max_age_to_cache(void)
  */
 int
 consdiffmgr_add_consensus(const char *consensus,
+                          size_t consensus_len,
                           const networkstatus_t *as_parsed)
 {
   if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
@@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus,
   }
 
   /* We don't have it. Add it to the cache. */
-  return consensus_queue_compression_work(consensus, as_parsed);
+  return consensus_queue_compression_work(consensus, consensus_len, as_parsed);
 }
 
 /**
@@ -1387,19 +1406,21 @@ typedef struct consensus_diff_worker_job_t {
 } consensus_diff_worker_job_t;
 
 /** Given a consensus_cache_entry_t, check whether it has a label claiming
- * that it was compressed.  If so, uncompress its contents into <b>out</b> and
- * set <b>outlen</b> to hold their size.  If not, just copy the body into
- * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
- * -1 on failure.
- *
- * In all cases, the output is nul-terminated. */
+ * that it was compressed.  If so, uncompress its contents into *<b>out</b> and
+ * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer
+ * that the caller will need to free.  If not, just set *<b>out</b> and
+ * <b>outlen</b> to its extent in memory.  Return 0 on success, -1 on failure.
+ **/
 STATIC int
-uncompress_or_copy(char **out, size_t *outlen,
-                   consensus_cache_entry_t *ent)
+uncompress_or_set_ptr(const char **out, size_t *outlen,
+                      char **owned_out,
+                      consensus_cache_entry_t *ent)
 {
   const uint8_t *body;
   size_t bodylen;
 
+  *owned_out = NULL;
+
   if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
     return -1;
 
@@ -1410,8 +1431,17 @@ uncompress_or_copy(char **out, size_t *outlen,
   if (lv_compression)
     method = compression_method_get_by_name(lv_compression);
 
-  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+  int rv;
+  if (method == NO_METHOD) {
+    *out = (const char *)body;
+    *outlen = bodylen;
+    rv = 0;
+  } else {
+    rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen,
                         method, 1, LOG_WARN);
+    *out = *owned_out;
+  }
+  return rv;
 }
 
 /**
@@ -1478,16 +1508,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
 
   char *consensus_diff;
   {
-    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    const char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    char *owned1 = NULL, *owned2 = NULL;
     size_t diff_from_nt_len, diff_to_nt_len;
 
-    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
-                           job->diff_from) < 0) {
+    if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1,
+                              job->diff_from) < 0) {
       return WQ_RPL_REPLY;
     }
-    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
-                           job->diff_to) < 0) {
-      tor_free(diff_from_nt);
+    if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2,
+                              job->diff_to) < 0) {
+      tor_free(owned1);
       return WQ_RPL_REPLY;
     }
     tor_assert(diff_from_nt);
@@ -1496,9 +1527,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
     // XXXX ugh; this is going to calculate the SHA3 of both its
     // XXXX inputs again, even though we already have that. Maybe it's time
     // XXXX to change the API here?
-    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
-    tor_free(diff_from_nt);
-    tor_free(diff_to_nt);
+    consensus_diff = consensus_diff_generate(diff_from_nt,
+                                             diff_from_nt_len,
+                                             diff_to_nt,
+                                             diff_to_nt_len);
+    tor_free(owned1);
+    tor_free(owned2);
   }
   if (!consensus_diff) {
     /* Couldn't generate consensus; we'll leave the reply blank. */
@@ -1746,8 +1780,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_)
                           (const uint8_t *)consensus, bodylen);
   {
     const char *start, *end;
-    if (router_get_networkstatus_v3_signed_boundaries(consensus,
-                                                        &start, &end) < 0) {
+    if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen,
+                                                      &start, &end) < 0) {
       start = consensus;
       end = consensus+bodylen;
     }
@@ -1811,14 +1845,15 @@ static int background_compression = 0;
  */
 static int
 consensus_queue_compression_work(const char *consensus,
+                                 size_t consensus_len,
                                  const networkstatus_t *as_parsed)
 {
   tor_assert(consensus);
   tor_assert(as_parsed);
 
   consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
-  job->consensus = tor_strdup(consensus);
-  job->consensus_len = strlen(consensus);
+  job->consensus = tor_memdup_nulterm(consensus, consensus_len);
+  job->consensus_len = strlen(job->consensus);
   job->flavor = as_parsed->flavor;
 
   char va_str[ISO_TIME_LEN+1];
diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h
index 39e8fa31cb..b1b3323b6c 100644
--- a/src/feature/dircache/consdiffmgr.h
+++ b/src/feature/dircache/consdiffmgr.h
@@ -22,6 +22,7 @@ typedef struct consdiff_cfg_t {
 struct consensus_cache_entry_t; // from conscache.h
 
 int consdiffmgr_add_consensus(const char *consensus,
+                              size_t consensus_len,
                               const networkstatus_t *as_parsed);
 
 consdiff_status_t consdiffmgr_find_consensus(
@@ -68,8 +69,14 @@ STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
 STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
                                     consensus_cache_entry_t *ent,
                                     const char *label);
-STATIC int uncompress_or_copy(char **out, size_t *outlen,
-                              consensus_cache_entry_t *ent);
+STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen,
+                                 char **owned_out,
+                                 consensus_cache_entry_t *ent);
 #endif /* defined(CONSDIFFMGR_PRIVATE) */
 
+#ifdef TOR_UNIT_TESTS
+int consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                      const networkstatus_t *as_parsed);
+#endif
+
 #endif /* !defined(TOR_CONSDIFFMGR_H) */
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
index e8cb284165..59cdcc5e02 100644
--- a/src/feature/dircache/dircache.c
+++ b/src/feature/dircache/dircache.c
@@ -49,7 +49,8 @@
 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
 #define ROBOTS_CACHE_LIFETIME (24*60*60)
 #define MICRODESC_CACHE_LIFETIME (48*60*60)
-
+/* Bandwidth files change every hour. */
+#define BANDWIDTH_CACHE_LIFETIME (30*60)
 /** Parse an HTTP request string <b>headers</b> of the form
  * \verbatim
  * "\%s [http[s]://]\%s HTTP/1..."
@@ -123,7 +124,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
                            long cache_lifetime)
 {
   char date[RFC1123_TIME_LEN+1];
-  time_t now = time(NULL);
+  time_t now = approx_time();
   buf_t *buf = buf_new_with_capacity(1024);
 
   tor_assert(conn);
@@ -166,22 +167,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
   buf_free(buf);
 }
 
-/** As write_http_response_header_impl, but sets encoding and content-typed
- * based on whether the response will be <b>compressed</b> or not. */
+/** As write_http_response_header_impl, but translates method into
+ * encoding */
 static void
 write_http_response_headers(dir_connection_t *conn, ssize_t length,
                             compress_method_t method,
                             const char *extra_headers, long cache_lifetime)
 {
-  const char *methodname = compression_method_get_name(method);
-  const char *doctype;
-  if (method == NO_METHOD)
-    doctype = "text/plain";
-  else
-    doctype = "application/octet-stream";
   write_http_response_header_impl(conn, length,
-                                  doctype,
-                                  methodname,
+                                  "text/plain",
+                                  compression_method_get_name(method),
                                   extra_headers,
                                   cache_lifetime);
 }
@@ -357,12 +352,15 @@ static int handle_get_robots(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
                                 const get_handler_args_t *args);
+static int handle_get_next_bandwidth(dir_connection_t *conn,
+                                     const get_handler_args_t *args);
 
 /** Table for handling GET requests. */
 static const url_table_ent_t url_table[] = {
   { "/tor/", 0, handle_get_frontpage },
   { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
   { "/tor/status-vote/current/", 1, handle_get_status_vote },
+  { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth },
   { "/tor/status-vote/next/", 1, handle_get_status_vote },
   { "/tor/micro/d/", 1, handle_get_microdesc },
   { "/tor/server/", 1, handle_get_descriptor },
@@ -495,28 +493,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
 }
 
 /** Warn that the cached consensus <b>consensus</b> of type
- * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
- * warning to avoid logging an entry on every request.
+ * <b>flavor</b> too new or too old, based on <b>is_too_new</b>,
+ * and will not be served to clients. Rate-limit the warning to avoid logging
+ * an entry on every request.
  */
 static void
-warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
-                          const char *flavor, time_t now)
+warn_consensus_is_not_reasonably_live(
+                          const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now, bool is_too_new)
 {
-#define TOO_OLD_WARNING_INTERVAL (60*60)
-  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60)
+  static ratelim_t warned[2] = { RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL),
+                                RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL) };
   char timestamp[ISO_TIME_LEN+1];
-  time_t valid_until;
-  char *dupes;
+  /* valid_after if is_too_new, valid_until if !is_too_new */
+  time_t valid_time = 0;
+  char *dupes = NULL;
 
-  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
-    return;
-
-  if ((dupes = rate_limit_log(&warned, now))) {
-    format_local_iso_time(timestamp, valid_until);
-    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
-             "serve it to clients. It was valid until %s local time and we "
-             "continued to serve it for up to 24 hours after it expired.%s",
-             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+  if (is_too_new) {
+    if (consensus_cache_entry_get_valid_after(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[1], now);
+  } else {
+    if (consensus_cache_entry_get_valid_until(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[0], now);
+  }
+
+  if (dupes) {
+    format_local_iso_time(timestamp, valid_time);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not "
+             "serve it to clients. It was valid %s %s local time and we "
+             "continued to serve it for up to 24 hours %s.%s",
+             flavor ? flavor : "",
+             flavor ? " " : "",
+             is_too_new ? "new" : "old",
+             is_too_new ? "after" : "until",
+             timestamp,
+             is_too_new ? "before it was valid" : "after it expired",
+             dupes);
     tor_free(dupes);
   }
 }
@@ -859,7 +876,6 @@ handle_get_current_consensus(dir_connection_t *conn,
 
   if (req.diff_only && !cached_consensus) {
     write_short_http_response(conn, 404, "No such diff available");
-    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -870,19 +886,30 @@ handle_get_current_consensus(dir_connection_t *conn,
                                            &compression_used);
   }
 
-  time_t fresh_until, valid_until;
-  int have_fresh_until = 0, have_valid_until = 0;
+  time_t valid_after, fresh_until, valid_until;
+  int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0;
   if (cached_consensus) {
+    have_valid_after =
+      !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after);
     have_fresh_until =
       !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
     have_valid_until =
       !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
   }
 
-  if (cached_consensus && have_valid_until &&
+  if (cached_consensus && have_valid_after &&
+      !networkstatus_valid_after_is_reasonably_live(valid_after, now)) {
+    write_short_http_response(conn, 404, "Consensus is too new");
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          1);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (
+      cached_consensus && have_valid_until &&
       !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
     write_short_http_response(conn, 404, "Consensus is too old");
-    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          0);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -924,7 +951,7 @@ handle_get_current_consensus(dir_connection_t *conn,
     goto done;
   }
 
-  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+  if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
     log_debug(LD_DIRSERV,
               "Client asked for network status lists, but we've been "
               "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1033,7 +1060,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
         }
       });
 
-    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), estimated_len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto vote_done;
     }
@@ -1045,13 +1072,11 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
       if (compress_method != NO_METHOD) {
         conn->compress_state = tor_compress_new(1, compress_method,
                            choose_compression_level(estimated_len));
-        SMARTLIST_FOREACH(items, const char *, c,
-                 connection_buf_add_compress(c, strlen(c), conn, 0));
-        connection_buf_add_compress("", 0, conn, 1);
-      } else {
-        SMARTLIST_FOREACH(items, const char *, c,
-                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
       }
+
+      SMARTLIST_FOREACH(items, const char *, c,
+                        connection_dir_buf_add(c, strlen(c), conn,
+                                               c_sl_idx == c_sl_len - 1));
     } else {
       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
           connection_buf_add(compress_method != NO_METHOD ?
@@ -1094,7 +1119,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
       write_short_http_response(conn, 404, "Not found");
       goto done;
     }
-    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
       log_info(LD_DIRSERV,
                "Client asked for server descriptors, but we've been "
                "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1192,7 +1217,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
         msg = "Not found";
       write_short_http_response(conn, 404, msg);
     } else {
-      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
         log_info(LD_DIRSERV,
                  "Client asked for server descriptors, but we've been "
                  "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1288,9 +1313,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
                       len += c->cache_info.signed_descriptor_len);
 
-    if (global_write_bucket_low(TO_CONN(conn),
-                                compress_method != NO_METHOD ? len/2 : len,
-                                2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto keys_done;
     }
@@ -1302,19 +1326,13 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     if (compress_method != NO_METHOD) {
       conn->compress_state = tor_compress_new(1, compress_method,
                                               choose_compression_level(len));
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add_compress(
-                c->cache_info.signed_descriptor_body,
-                c->cache_info.signed_descriptor_len,
-                conn, 0));
-      connection_buf_add_compress("", 0, conn, 1);
-    } else {
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add(c->cache_info.signed_descriptor_body,
-                                    c->cache_info.signed_descriptor_len,
-                                    TO_CONN(conn)));
     }
-  keys_done:
+
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+          connection_dir_buf_add(c->cache_info.signed_descriptor_body,
+                                 c->cache_info.signed_descriptor_len,
+                                 conn, c_sl_idx == c_sl_len - 1));
+ keys_done:
     smartlist_free(certs);
     goto done;
   }
@@ -1371,9 +1389,11 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn,
   const char *pubkey_str = NULL;
   const char *url = args->url;
 
-  /* Reject unencrypted dir connections */
-  if (!connection_dir_is_encrypted(conn)) {
-    write_short_http_response(conn, 404, "Not found");
+  /* Reject non anonymous dir connections (which also tests if encrypted). We
+   * do not allow single hop clients to query an HSDir. */
+  if (!connection_dir_is_anonymous(conn)) {
+    write_short_http_response(conn, 503,
+                              "Rejecting single hop HS v3 descriptor request");
     goto done;
   }
 
@@ -1438,6 +1458,39 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
   return 0;
 }
 
+/** Helper function for GET the bandwidth file used for the next vote */
+static int
+handle_get_next_bandwidth(dir_connection_t *conn,
+                          const get_handler_args_t *args)
+{
+  log_debug(LD_DIR, "Getting next bandwidth.");
+  const or_options_t *options = get_options();
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+
+  if (options->V3BandwidthsFile) {
+    char *bandwidth = read_file_to_str(options->V3BandwidthsFile,
+                                       RFTS_IGNORE_MISSING, NULL);
+    if (bandwidth != NULL) {
+      ssize_t len = strlen(bandwidth);
+      write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len,
+                                 compress_method, BANDWIDTH_CACHE_LIFETIME);
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(len/2));
+        log_debug(LD_DIR, "Compressing bandwidth file.");
+      } else {
+        log_debug(LD_DIR, "Not compressing bandwidth file.");
+      }
+      connection_dir_buf_add((const char*)bandwidth, len, conn, 1);
+      tor_free(bandwidth);
+      return 0;
+    }
+  }
+  write_short_http_response(conn, 404, "Not found");
+  return 0;
+}
+
 /** Helper function for GET robots.txt or /tor/robots.txt */
 static int
 handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
@@ -1580,10 +1633,15 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     goto done;
   }
 
-  /* Handle HS descriptor publish request. */
-  /* XXX: This should be disabled with a consensus param until we want to
-   * the prop224 be deployed and thus use. */
-  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+  /* Handle HS descriptor publish request. We force an anonymous connection
+   * (which also tests for encrypted). We do not allow single-hop client to
+   * post a descriptor onto an HSDir. */
+  if (!strcmpstart(url, "/tor/hs/")) {
+    if (!connection_dir_is_anonymous(conn)) {
+      write_short_http_response(conn, 503,
+                                "Rejecting single hop HS descriptor post");
+      goto done;
+    }
     const char *msg = "HS descriptor stored successfully.";
 
     /* We most probably have a publish request for an HS descriptor. */
@@ -1608,8 +1666,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     const char *msg = "[None]";
     uint8_t purpose = authdir_mode_bridge(options) ?
                       ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
-                                             conn->base_.address, &msg);
+    was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len,
+                                           purpose, conn->base_.address, &msg);
     tor_assert(msg);
 
     if (r == ROUTER_ADDED_SUCCESSFULLY) {
diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h
index 236ea649ef..de0d205f6a 100644
--- a/src/feature/dircache/dircache.h
+++ b/src/feature/dircache/dircache.h
@@ -38,6 +38,6 @@ STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
                                       const char **end_pos);
 
 STATIC unsigned parse_accept_encoding_header(const char *h);
-#endif
+#endif /* defined(DIRCACHE_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCACHE_H) */
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
index 213c490314..79400bf15f 100644
--- a/src/feature/dircache/dirserv.c
+++ b/src/feature/dircache/dirserv.c
@@ -234,6 +234,7 @@ free_cached_dir_(void *_d)
  * validation is performed. */
 void
 dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
+                                           size_t networkstatus_len,
                                            const char *flavor_name,
                                            const common_digests_t *digests,
                                            const uint8_t *sha3_as_signed,
@@ -244,7 +245,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
   if (!cached_consensuses)
     cached_consensuses = strmap_new();
 
-  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
+  new_networkstatus =
+    new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len),
+                   published);
   memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
   memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
          DIGEST256_LEN);
@@ -580,11 +583,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
       /* Absent objects count as "done". */
       return SRFS_DONE;
     }
-    if (conn->compress_state) {
-      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
-    } else {
-      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add((const char*)body, bodylen, conn, 0);
+
     return SRFS_DONE;
   } else {
     cached_dir_t *cached = spooled->cached_dir_ref;
@@ -619,14 +620,10 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
     if (BUG(remaining < 0))
       return SRFS_ERR;
     ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
-    if (conn->compress_state) {
-      connection_buf_add_compress(
-              ptr + spooled->cached_dir_offset,
-              bytes, conn, 0);
-    } else {
-      connection_buf_add(ptr + spooled->cached_dir_offset,
-                              bytes, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add(ptr + spooled->cached_dir_offset,
+                           bytes, conn, 0);
+
     spooled->cached_dir_offset += bytes;
     if (spooled->cached_dir_offset >= (off_t)total_len) {
       return SRFS_DONE;
diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h
index 890b10fd80..7f944459da 100644
--- a/src/feature/dircache/dirserv.h
+++ b/src/feature/dircache/dirserv.h
@@ -84,6 +84,7 @@ int directory_too_idle_to_fetch_descriptors(const or_options_t *options,
 
 cached_dir_t *dirserv_get_consensus(const char *flavor_name);
 void dirserv_set_cached_consensus_networkstatus(const char *consensus,
+                                              size_t consensus_len,
                                               const char *flavor_name,
                                               const common_digests_t *digests,
                                               const uint8_t *sha3_as_signed,
diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h
index 2f5706cdd9..8e35532435 100644
--- a/src/feature/dirclient/dir_server_st.h
+++ b/src/feature/dirclient/dir_server_st.h
@@ -51,4 +51,4 @@ struct dir_server_t {
                                **/
 };
 
-#endif
+#endif /* !defined(DIR_SERVER_ST_H) */
diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c
index 6725fc3369..fa82bdc1e7 100644
--- a/src/feature/dirclient/dirclient.c
+++ b/src/feature/dirclient/dirclient.c
@@ -14,7 +14,7 @@
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/shared_random.h"
@@ -2205,24 +2205,31 @@ handle_response_fetch_consensus(dir_connection_t *conn,
   if (looks_like_a_consensus_diff(body, body_len)) {
     /* First find our previous consensus. Maybe it's in ram, maybe not. */
     cached_dir_t *cd = dirserv_get_consensus(flavname);
-    const char *consensus_body;
-    char *owned_consensus = NULL;
+    const char *consensus_body = NULL;
+    size_t consensus_body_len;
+    tor_mmap_t *mapped_consensus = NULL;
     if (cd) {
       consensus_body = cd->dir;
+      consensus_body_len = cd->dir_len;
     } else {
-      owned_consensus = networkstatus_read_cached_consensus(flavname);
-      consensus_body = owned_consensus;
+      mapped_consensus = networkstatus_map_cached_consensus(flavname);
+      if (mapped_consensus) {
+        consensus_body = mapped_consensus->data;
+        consensus_body_len = mapped_consensus->size;
+      }
     }
     if (!consensus_body) {
       log_warn(LD_DIR, "Received a consensus diff, but we can't find "
                "any %s-flavored consensus in our current cache.",flavname);
+      tor_munmap_file(mapped_consensus);
       networkstatus_consensus_download_failed(0, flavname);
       // XXXX if this happens too much, see below
       return -1;
     }
 
-    new_consensus = consensus_diff_apply(consensus_body, body);
-    tor_free(owned_consensus);
+    new_consensus = consensus_diff_apply(consensus_body, consensus_body_len,
+                                         body, body_len);
+    tor_munmap_file(mapped_consensus);
     if (new_consensus == NULL) {
       log_warn(LD_DIR, "Could not apply consensus diff received from server "
                "'%s:%d'", conn->base_.address, conn->base_.port);
@@ -2244,7 +2251,9 @@ handle_response_fetch_consensus(dir_connection_t *conn,
     sourcename = "downloaded";
   }
 
-  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+  if ((r=networkstatus_set_current_consensus(consensus,
+                                             strlen(consensus),
+                                             flavname, 0,
                                              conn->identity_digest))<0) {
     log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
            "Unable to load %s consensus directory %s from "
@@ -2522,7 +2531,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn,
            conn->base_.port);
   tor_assert(conn->requested_resource &&
              !strcmpstart(conn->requested_resource, "d/"));
-  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  tor_assert_nonfatal(!fast_mem_is_zero(conn->identity_digest, DIGEST_LEN));
   which = smartlist_new();
   dir_split_resource_into_fingerprints(conn->requested_resource+2,
                                        which, NULL,
diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h
index 1a93265dc3..be4374c7cf 100644
--- a/src/feature/dirclient/dirclient.h
+++ b/src/feature/dirclient/dirclient.h
@@ -167,6 +167,6 @@ STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
 
 STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
                                      const char *resource);
-#endif
+#endif /* defined(DIRCLIENT_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCLIENT_H) */
diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h
index 99e0d0225b..681712b059 100644
--- a/src/feature/dirclient/dlstatus.h
+++ b/src/feature/dirclient/dlstatus.h
@@ -53,6 +53,6 @@ STATIC void next_random_exponential_delay_range(int *low_bound_out,
 /* no more than triple the previous delay */
 #define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
 
-#endif
+#endif /* defined(DLSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_DLSTATUS_H) */
diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h
index 11555a1dcc..39a5ad2860 100644
--- a/src/feature/dirclient/download_status_st.h
+++ b/src/feature/dirclient/download_status_st.h
@@ -61,5 +61,5 @@ struct download_status_t {
                         * only updated if backoff == 1 */
 };
 
-#endif
+#endif /* !defined(DOWNLOAD_STATUS_ST_H) */
 
diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c
index d0f7594ce3..8e93953f73 100644
--- a/src/feature/dircommon/consdiff.c
+++ b/src/feature/dircommon/consdiff.c
@@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest,(const char *cons,
+consensus_compute_digest,(const char *cons, size_t len,
                           consensus_digest_t *digest_out))
 {
   int r = crypto_digest256((char*)digest_out->sha3_256,
-                           cons, strlen(cons), DIGEST_SHA3_256);
+                           cons, len, DIGEST_SHA3_256);
   return r;
 }
 
@@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons,
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest_as_signed,(const char *cons,
+consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out))
 {
   return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
-                                                    cons);
+                                                    cons, len);
 }
 
 /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
@@ -1229,7 +1229,8 @@ consdiff_apply_diff(const smartlist_t *cons1,
   cons2_str = consensus_join_lines(cons2);
 
   consensus_digest_t cons2_digests;
-  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+  if (consensus_compute_digest(cons2_str, strlen(cons2_str),
+                               &cons2_digests) < 0) {
     /* LCOV_EXCL_START -- digest can't fail */
     log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
         "resulting from applying a consensus diff.");
@@ -1283,12 +1284,13 @@ consdiff_apply_diff(const smartlist_t *cons1,
  * generated cdlines will become invalid.
  */
 STATIC int
-consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+consensus_split_lines(smartlist_t *out,
+                      const char *s, size_t len,
+                      memarea_t *area)
 {
-  const char *end_of_str = s + strlen(s);
-  tor_assert(*end_of_str == '\0');
+  const char *end_of_str = s + len;
 
-  while (*s) {
+  while (s < end_of_str) {
     const char *eol = memchr(s, '\n', end_of_str - s);
     if (!eol) {
       /* File doesn't end with newline. */
@@ -1334,25 +1336,25 @@ consensus_join_lines(const smartlist_t *inp)
  * success, retun a newly allocated string containing that diff.  On failure,
  * return NULL. */
 char *
-consensus_diff_generate(const char *cons1,
-                        const char *cons2)
+consensus_diff_generate(const char *cons1, size_t cons1len,
+                        const char *cons2, size_t cons2len)
 {
   consensus_digest_t d1, d2;
   smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
   int r1, r2;
   char *result = NULL;
 
-  r1 = consensus_compute_digest_as_signed(cons1, &d1);
-  r2 = consensus_compute_digest(cons2, &d2);
+  r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1);
+  r2 = consensus_compute_digest(cons2, cons2len, &d2);
   if (BUG(r1 < 0 || r2 < 0))
     return NULL; // LCOV_EXCL_LINE
 
   memarea_t *area = memarea_new();
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, cons1, area) < 0)
+  if (consensus_split_lines(lines1, cons1, cons1len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, cons2, area) < 0)
+  if (consensus_split_lines(lines2, cons2, cons2len, area) < 0)
     goto done;
 
   result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
@@ -1375,7 +1377,9 @@ consensus_diff_generate(const char *cons1,
  * consensus.  On failure, return NULL. */
 char *
 consensus_diff_apply(const char *consensus,
-                     const char *diff)
+                     size_t consensus_len,
+                     const char *diff,
+                     size_t diff_len)
 {
   consensus_digest_t d1;
   smartlist_t *lines1 = NULL, *lines2 = NULL;
@@ -1383,15 +1387,15 @@ consensus_diff_apply(const char *consensus,
   char *result = NULL;
   memarea_t *area = memarea_new();
 
-  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1);
   if (BUG(r1 < 0))
     goto done;
 
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, consensus, area) < 0)
+  if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, diff, area) < 0)
+  if (consensus_split_lines(lines2, diff, diff_len, area) < 0)
     goto done;
 
   result = consdiff_apply_diff(lines1, lines2, &d1);
diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h
index 98217e6d46..b63fcb2cc6 100644
--- a/src/feature/dircommon/consdiff.h
+++ b/src/feature/dircommon/consdiff.h
@@ -7,10 +7,10 @@
 
 #include "core/or/or.h"
 
-char *consensus_diff_generate(const char *cons1,
-                              const char *cons2);
-char *consensus_diff_apply(const char *consensus,
-                           const char *diff);
+char *consensus_diff_generate(const char *cons1, size_t cons1len,
+                              const char *cons2, size_t cons2len);
+char *consensus_diff_apply(const char *consensus, size_t consensus_len,
+                           const char *diff, size_t diff_len);
 
 int looks_like_a_consensus_diff(const char *document, size_t len);
 
@@ -78,7 +78,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
 STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
                         const smartlist_slice_t *slice1,
                         const smartlist_slice_t *slice2);
-STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+STATIC int consensus_split_lines(smartlist_t *out,
+                                 const char *s, size_t len,
                                  struct memarea_t *area);
 STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
                                   const char *s);
@@ -86,10 +87,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
 STATIC int line_str_eq(const cdline_t *a, const char *b);
 
 MOCK_DECL(STATIC int,
-          consensus_compute_digest,(const char *cons,
+          consensus_compute_digest,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
-          consensus_compute_digest_as_signed,(const char *cons,
+          consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                               consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
           consensus_digest_eq,(const uint8_t *d1,
diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h
index 8c59cc7a46..a858560c29 100644
--- a/src/feature/dircommon/dir_connection_st.h
+++ b/src/feature/dircommon/dir_connection_st.h
@@ -64,4 +64,4 @@ struct dir_connection_t {
 #endif /* defined(MEASUREMENTS_21206) */
 };
 
-#endif
+#endif /* !defined(DIR_CONNECTION_ST_H) */
diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c
index 9e6f72e9ac..8e5b413326 100644
--- a/src/feature/dircommon/directory.c
+++ b/src/feature/dircommon/directory.c
@@ -7,6 +7,10 @@
 
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/channeltls.h"
 #include "feature/dircache/dircache.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -15,6 +19,10 @@
 #include "feature/stats/geoip_stats.h"
 #include "lib/compress/compress.h"
 
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 
@@ -167,6 +175,82 @@ connection_dir_is_encrypted(const dir_connection_t *conn)
   return TO_CONN(conn)->linked;
 }
 
+/** Return true iff the given directory connection <b>dir_conn</b> is
+ * anonymous, that is, it is on a circuit via a public relay and not directly
+ * from a client or bridge.
+ *
+ * For client circuits via relays: true for 2-hop+ paths.
+ * For client circuits via bridges: true for 3-hop+ paths.
+ *
+ * This first test if the connection is encrypted since it is a strong
+ * requirement for anonymity. */
+bool
+connection_dir_is_anonymous(const dir_connection_t *dir_conn)
+{
+  const connection_t *conn, *linked_conn;
+  const edge_connection_t *edge_conn;
+  const circuit_t *circ;
+
+  tor_assert(dir_conn);
+
+  if (!connection_dir_is_encrypted(dir_conn)) {
+    return false;
+  }
+
+  /*
+   * Buckle up, we'll do a deep dive into the connection in order to get the
+   * final connection channel of that connection in order to figure out if
+   * this is a client or relay link.
+   *
+   * We go: dir_conn -> linked_conn -> edge_conn -> on_circuit -> p_chan.
+   */
+
+  conn = TO_CONN(dir_conn);
+  linked_conn = conn->linked_conn;
+
+  /* The dir connection should be connected to an edge connection. It can not
+   * be closed or marked for close. */
+  if (linked_conn == NULL || linked_conn->magic != EDGE_CONNECTION_MAGIC ||
+      conn->linked_conn_is_closed || conn->linked_conn->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not linked to edge");
+    return false;
+  }
+
+  edge_conn = TO_EDGE_CONN((connection_t *) linked_conn);
+  circ = edge_conn->on_circuit;
+
+  /* Can't be a circuit we initiated and without a circuit, no channel. */
+  if (circ == NULL || CIRCUIT_IS_ORIGIN(circ)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not on OR circuit");
+    return false;
+  }
+
+  /* It is possible that the circuit was closed because one of the channel was
+   * closed or a DESTROY cell was received. Either way, this connection can
+   * not continue so return that it is not anonymous since we can not know for
+   * sure if it is. */
+  if (circ->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "circuit marked for close");
+    return false;
+  }
+
+  /* Get the previous channel to learn if it is a client or relay link. We
+   * BUG() because if the circuit is not mark for close, we ought to have a
+   * p_chan else we have a code flow issue. */
+  if (BUG(CONST_TO_OR_CIRCUIT(circ)->p_chan == NULL)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "no p_chan on circuit");
+    return false;
+  }
+
+  /* Will be true if the channel is an unauthenticated peer which is only true
+   * for clients and bridges. */
+  return !channel_is_client(CONST_TO_OR_CIRCUIT(circ)->p_chan);
+}
+
 /** Parse an HTTP request line at the start of a headers string.  On failure,
  * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
  * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h
index ba3f8c1b0e..4fc743ad3d 100644
--- a/src/feature/dircommon/directory.h
+++ b/src/feature/dircommon/directory.h
@@ -94,6 +94,7 @@ int parse_http_command(const char *headers,
 char *http_get_header(const char *headers, const char *which);
 
 int connection_dir_is_encrypted(const dir_connection_t *conn);
+bool connection_dir_is_anonymous(const dir_connection_t *conn);
 int connection_dir_reached_eof(dir_connection_t *conn);
 int connection_dir_process_inbuf(dir_connection_t *conn);
 int connection_dir_finished_flushing(dir_connection_t *conn);
diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h
index 47b90ab009..814a325314 100644
--- a/src/feature/dircommon/vote_timing_st.h
+++ b/src/feature/dircommon/vote_timing_st.h
@@ -20,5 +20,5 @@ struct vote_timing_t {
   int dist_delay;
 };
 
-#endif
+#endif /* !defined(VOTE_TIMING_ST_H) */
 
diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dircommon/voting_schedule.c
index 0a7476eda7..5576ec69f7 100644
--- a/src/feature/dircommon/voting_schedule.c
+++ b/src/feature/dircommon/voting_schedule.c
@@ -150,7 +150,7 @@ voting_schedule_get_next_valid_after_time(void)
   /* This is a safe guard in order to make sure that the voting schedule
    * static object is at least initialized. Using this function with a zeroed
    * voting schedule can lead to bugs. */
-  if (tor_mem_is_zero((const char *) &voting_schedule,
+  if (fast_mem_is_zero((const char *) &voting_schedule,
                       sizeof(voting_schedule))) {
     need_to_recalculate_voting_schedule = true;
     goto done; /* no need for next check if we have to recalculate anyway */
diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dircommon/voting_schedule.h
index bafd81184e..d78c7ee2da 100644
--- a/src/feature/dircommon/voting_schedule.h
+++ b/src/feature/dircommon/voting_schedule.h
@@ -61,5 +61,5 @@ time_t voting_schedule_get_start_of_next_interval(time_t now,
                                                   int offset);
 time_t voting_schedule_get_next_valid_after_time(void);
 
-#endif /* TOR_VOTING_SCHEDULE_H */
+#endif /* !defined(TOR_VOTING_SCHEDULE_H) */
 
diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c
index 1680bdbf30..8ba5a53981 100644
--- a/src/feature/dirparse/authcert_parse.c
+++ b/src/feature/dirparse/authcert_parse.c
@@ -24,7 +24,8 @@ static token_rule_t dir_key_certificate_table[] = {
 /** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
  * the first character after the certificate. */
 authority_cert_t *
-authority_cert_parse_from_string(const char *s, const char **end_of_string)
+authority_cert_parse_from_string(const char *s, size_t maxlen,
+                                 const char **end_of_string)
 {
   /** Reject any certificate at least this big; it is probably an overflow, an
    * attack, a bug, or some other nonsense. */
@@ -35,24 +36,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
   char digest[DIGEST_LEN];
   directory_token_t *tok;
   char fp_declared[DIGEST_LEN];
-  char *eos;
+  const char *eos;
   size_t len;
   int found;
   memarea_t *area = NULL;
+  const char *end_of_s = s + maxlen;
   const char *s_dup = s;
 
-  s = eat_whitespace(s);
-  eos = strstr(s, "\ndir-key-certification");
+  s = eat_whitespace_eos(s, end_of_s);
+  eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification");
   if (! eos) {
     log_warn(LD_DIR, "No signature found on key certificate");
     return NULL;
   }
-  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
+  eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n");
   if (! eos) {
     log_warn(LD_DIR, "No end-of-signature found on key certificate");
     return NULL;
   }
-  eos = strchr(eos+2, '\n');
+  eos = memchr(eos+2, '\n', end_of_s - (eos+2));
   tor_assert(eos);
   ++eos;
   len = eos - s;
@@ -69,7 +71,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
     log_warn(LD_DIR, "Error tokenizing key certificate");
     goto err;
   }
-  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
+  if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version",
                            "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
     goto err;
   tok = smartlist_get(tokens, 0);
diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h
index ca475ad0e3..800631c3de 100644
--- a/src/feature/dirparse/authcert_parse.h
+++ b/src/feature/dirparse/authcert_parse.h
@@ -13,6 +13,7 @@
 #define TOR_AUTHCERT_PARSE_H
 
 authority_cert_t *authority_cert_parse_from_string(const char *s,
+                                                   size_t maxlen,
                                                    const char **end_of_string);
 
 #endif /* !defined(TOR_AUTHCERT_PARSE_H) */
diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c
index 5a75af3994..4bb4db7821 100644
--- a/src/feature/dirparse/microdesc_parse.c
+++ b/src/feature/dirparse/microdesc_parse.c
@@ -18,6 +18,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/relay/router.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -32,7 +33,7 @@ static token_rule_t microdesc_token_table[] = {
   T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
   T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
   T0N("a",                     K_A,                GE(1),       NO_OBJ ),
-  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("family",                K_FAMILY,           CONCAT_ARGS, NO_OBJ ),
   T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
   T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
   A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
@@ -91,6 +92,190 @@ find_start_of_next_microdesc(const char *s, const char *eos)
 #undef NEXT_LINE
 }
 
+static inline int
+policy_is_reject_star_or_null(struct short_policy_t *policy)
+{
+  return !policy || short_policy_is_reject_star(policy);
+}
+
+/**
+ * Return a human-readable description of a given saved_location_t.
+ * Never returns NULL.
+ **/
+static const char *
+saved_location_to_string(saved_location_t where)
+{
+  const char *location;
+  switch (where) {
+    case SAVED_NOWHERE:
+      location = "download or generated string";
+      break;
+    case SAVED_IN_CACHE:
+      location = "cache";
+      break;
+    case SAVED_IN_JOURNAL:
+      location = "journal";
+      break;
+    default:
+      location = "unknown location";
+      break;
+  }
+  return location;
+}
+
+/**
+ * Given a microdescriptor stored in <b>where</b> which starts at <b>s</b>,
+ * which ends at <b>start_of_next_microdescriptor</b>, and which is located
+ * within a larger document beginning at <b>start</b>: Fill in the body,
+ * bodylen, bodylen, saved_location, off, and digest fields of <b>md</b> as
+ * appropriate.
+ *
+ * The body field will be an alias within <b>s</b> if <b>saved_location</b>
+ * is SAVED_IN_CACHE, and will be copied into body and nul-terminated
+ * otherwise.
+ **/
+static int
+microdesc_extract_body(microdesc_t *md,
+                       const char *start,
+                       const char *s, const char *start_of_next_microdesc,
+                       saved_location_t where)
+{
+  const bool copy_body = (where != SAVED_IN_CACHE);
+
+  const char *cp = tor_memstr(s, start_of_next_microdesc-s, "onion-key");
+
+  const bool no_onion_key = (cp == NULL);
+  if (no_onion_key) {
+    cp = s; /* So that we have *some* junk to put in the body */
+  }
+
+  md->bodylen = start_of_next_microdesc - cp;
+  md->saved_location = where;
+  if (copy_body)
+    md->body = tor_memdup_nulterm(cp, md->bodylen);
+  else
+    md->body = (char*)cp;
+  md->off = cp - start;
+
+  crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+
+  return no_onion_key ? -1 : 0;
+}
+
+/**
+ * Parse a microdescriptor which begins at <b>s</b> and ends at
+ * <b>start_of_next_microdesc.  Store its fields into <b>md</b>.  Use
+ * <b>where</b> for generating log information.  If <b>allow_annotations</b>
+ * is true, then one or more annotations may precede the microdescriptor body
+ * proper.  Use <b>area</b> for memory management, clearing it when done.
+ *
+ * On success, return 0; otherwise return -1.
+ **/
+static int
+microdesc_parse_fields(microdesc_t *md,
+                       memarea_t *area,
+                       const char *s, const char *start_of_next_microdesc,
+                       int allow_annotations,
+                       saved_location_t where)
+{
+  smartlist_t *tokens = smartlist_new();
+  int rv = -1;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+  directory_token_t *tok;
+
+  if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                      microdesc_token_table, flags)) {
+    log_warn(LD_DIR, "Unparseable microdescriptor found in %s",
+             saved_location_to_string(where));
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+    if (parse_iso_time(tok->args[0], &md->last_listed)) {
+      log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+      goto err;
+    }
+  }
+
+  tok = find_by_keyword(tokens, K_ONION_KEY);
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_DIR,
+             "Relay's onion key had invalid exponent.");
+    goto err;
+  }
+  md->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  md->onion_pkey_len = tok->object_size;
+  crypto_pk_free(tok->key);
+
+  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+    curve25519_public_key_t k;
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
+      goto err;
+    }
+    md->onion_curve25519_pkey =
+      tor_memdup(&k, sizeof(curve25519_public_key_t));
+  }
+
+  smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
+  if (id_lines) {
+    SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
+      tor_assert(t->n_args >= 2);
+      if (!strcmp(t->args[0], "ed25519")) {
+        if (md->ed25519_identity_pkey) {
+          log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        ed25519_public_key_t k;
+        if (ed25519_public_from_base64(&k, t->args[1])<0) {
+          log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
+      }
+    } SMARTLIST_FOREACH_END(t);
+    smartlist_free(id_lines);
+  }
+
+  {
+    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+    if (a_lines) {
+      find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
+      smartlist_free(a_lines);
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+    md->family = nodefamily_parse(tok->args[0],
+                                  NULL,
+                                  NF_WARN_MALFORMED);
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_P))) {
+    md->exit_policy = parse_short_policy(tok->args[0]);
+  }
+  if ((tok = find_opt_by_keyword(tokens, K_P6))) {
+    md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+  }
+
+  if (policy_is_reject_star_or_null(md->exit_policy) &&
+      policy_is_reject_star_or_null(md->ipv6_exit_policy)) {
+    md->policy_is_reject_star = 1;
+  }
+
+  rv = 0;
+ err:
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  memarea_clear(area);
+  smartlist_free(tokens);
+
+  return rv;
+}
+
 /** Parse as many microdescriptors as are found from the string starting at
  * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
  * annotations we recognize and ignore ones we don't.
@@ -108,16 +293,11 @@ microdescs_parse_from_string(const char *s, const char *eos,
                              saved_location_t where,
                              smartlist_t *invalid_digests_out)
 {
-  smartlist_t *tokens;
   smartlist_t *result;
   microdesc_t *md = NULL;
   memarea_t *area;
   const char *start = s;
   const char *start_of_next_microdesc;
-  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
-  const int copy_body = (where != SAVED_IN_CACHE);
-
-  directory_token_t *tok;
 
   if (!eos)
     eos = s + strlen(s);
@@ -125,143 +305,47 @@ microdescs_parse_from_string(const char *s, const char *eos,
   s = eat_whitespace_eos(s, eos);
   area = memarea_new();
   result = smartlist_new();
-  tokens = smartlist_new();
 
   while (s < eos) {
-    int okay = 0;
+   bool okay = false;
 
     start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
     if (!start_of_next_microdesc)
       start_of_next_microdesc = eos;
 
     md = tor_malloc_zero(sizeof(microdesc_t));
+    uint8_t md_digest[DIGEST256_LEN];
     {
-      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
-                                  "onion-key");
-      const int no_onion_key = (cp == NULL);
-      if (no_onion_key) {
-        cp = s; /* So that we have *some* junk to put in the body */
-      }
+      const bool body_not_found =
+        microdesc_extract_body(md, start, s,
+                               start_of_next_microdesc,
+                               where) < 0;
 
-      md->bodylen = start_of_next_microdesc - cp;
-      md->saved_location = where;
-      if (copy_body)
-        md->body = tor_memdup_nulterm(cp, md->bodylen);
-      else
-        md->body = (char*)cp;
-      md->off = cp - start;
-      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
-      if (no_onion_key) {
+      memcpy(md_digest, md->digest, DIGEST256_LEN);
+      if (body_not_found) {
         log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
         goto next;
       }
     }
 
-    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
-                        microdesc_token_table, flags)) {
-      log_warn(LD_DIR, "Unparseable microdescriptor");
-      goto next;
+    if (microdesc_parse_fields(md, area, s, start_of_next_microdesc,
+                               allow_annotations, where) == 0) {
+      smartlist_add(result, md);
+      md = NULL; // prevent free
+      okay = true;
     }
 
-    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
-      if (parse_iso_time(tok->args[0], &md->last_listed)) {
-        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
-        goto next;
-      }
-    }
-
-    tok = find_by_keyword(tokens, K_ONION_KEY);
-    if (!crypto_pk_public_exponent_ok(tok->key)) {
-      log_warn(LD_DIR,
-               "Relay's onion key had invalid exponent.");
-      goto next;
-    }
-    router_set_rsa_onion_pkey(tok->key, &md->onion_pkey,
-                              &md->onion_pkey_len);
-    crypto_pk_free(tok->key);
-
-    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
-      curve25519_public_key_t k;
-      tor_assert(tok->n_args >= 1);
-      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
-        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
-        goto next;
-      }
-      md->onion_curve25519_pkey =
-        tor_memdup(&k, sizeof(curve25519_public_key_t));
-    }
-
-    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
-    if (id_lines) {
-      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
-        tor_assert(t->n_args >= 2);
-        if (!strcmp(t->args[0], "ed25519")) {
-          if (md->ed25519_identity_pkey) {
-            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          ed25519_public_key_t k;
-          if (ed25519_public_from_base64(&k, t->args[1])<0) {
-            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
-        }
-      } SMARTLIST_FOREACH_END(t);
-      smartlist_free(id_lines);
-    }
-
-    {
-      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
-      if (a_lines) {
-        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
-        smartlist_free(a_lines);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
-      int i;
-      md->family = smartlist_new();
-      for (i=0;i<tok->n_args;++i) {
-        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
-          log_warn(LD_DIR, "Illegal nickname %s in family line",
-                   escaped(tok->args[i]));
-          goto next;
-        }
-        smartlist_add_strdup(md->family, tok->args[i]);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_P))) {
-      md->exit_policy = parse_short_policy(tok->args[0]);
-    }
-    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
-      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
-    }
-
-    smartlist_add(result, md);
-    okay = 1;
-
-    md = NULL;
   next:
     if (! okay && invalid_digests_out) {
       smartlist_add(invalid_digests_out,
-                    tor_memdup(md->digest, DIGEST256_LEN));
+                    tor_memdup(md_digest, DIGEST256_LEN));
     }
     microdesc_free(md);
     md = NULL;
-
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    memarea_clear(area);
-    smartlist_clear(tokens);
     s = start_of_next_microdesc;
   }
 
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
   memarea_drop_all(area);
-  smartlist_free(tokens);
 
   return result;
 }
diff --git a/src/feature/dirparse/microdesc_parse.h b/src/feature/dirparse/microdesc_parse.h
index 23a90084b1..95af85544a 100644
--- a/src/feature/dirparse/microdesc_parse.h
+++ b/src/feature/dirparse/microdesc_parse.h
@@ -17,4 +17,4 @@ smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
                                           saved_location_t where,
                                           smartlist_t *invalid_digests_out);
 
-#endif
+#endif /* !defined(TOR_MICRODESC_PARSE_H) */
diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c
index 109eebeb66..d5405e6464 100644
--- a/src/feature/dirparse/ns_parse.c
+++ b/src/feature/dirparse/ns_parse.c
@@ -151,10 +151,11 @@ static token_rule_t networkstatus_vote_footer_token_table[] = {
  * -1. */
 int
 router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              size_t len,
                                               const char **start_out,
                                               const char **end_out)
 {
-  return router_get_hash_impl_helper(s, strlen(s),
+  return router_get_hash_impl_helper(s, len,
                                      "network-status-version",
                                      "\ndirectory-signature",
                                      ' ', LOG_INFO,
@@ -166,12 +167,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s,
  * signed portion can be identified.  Return 0 on success, -1 on failure. */
 int
 router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                           const char *s)
+                                           const char *s, size_t len)
 {
   const char *start, *end;
-  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+  if (router_get_networkstatus_v3_signed_boundaries(s, len,
+                                                    &start, &end) < 0) {
     start = s;
-    end = s + strlen(s);
+    end = s + len;
   }
   tor_assert(start);
   tor_assert(end);
@@ -182,9 +184,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
 /** Set <b>digests</b> to all the digests of the consensus document in
  * <b>s</b> */
 int
-router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
+router_get_networkstatus_v3_hashes(const char *s, size_t len,
+                                   common_digests_t *digests)
 {
-  return router_get_hashes_impl(s,strlen(s),digests,
+  return router_get_hashes_impl(s, len, digests,
                                 "network-status-version",
                                 "\ndirectory-signature",
                                 ' ');
@@ -195,13 +198,13 @@ router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
  * return the start of the directory footer, or the next directory signature.
  * If none is found, return the end of the string. */
 static inline const char *
-find_start_of_next_routerstatus(const char *s)
+find_start_of_next_routerstatus(const char *s, const char *s_eos)
 {
   const char *eos, *footer, *sig;
-  if ((eos = strstr(s, "\nr ")))
+  if ((eos = tor_memstr(s, s_eos - s, "\nr ")))
     ++eos;
   else
-    eos = s + strlen(s);
+    eos = s_eos;
 
   footer = tor_memstr(s, eos-s, "\ndirectory-footer");
   sig = tor_memstr(s, eos-s, "\ndirectory-signature");
@@ -289,7 +292,8 @@ routerstatus_parse_guardfraction(const char *guardfraction_str,
  **/
 STATIC routerstatus_t *
 routerstatus_parse_entry_from_string(memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *s_eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
@@ -308,7 +312,7 @@ routerstatus_parse_entry_from_string(memarea_t *area,
     flav = FLAV_NS;
   tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
 
-  eos = find_start_of_next_routerstatus(*s);
+  eos = find_start_of_next_routerstatus(*s, s_eos);
 
   if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing router status");
@@ -430,6 +434,8 @@ routerstatus_parse_entry_from_string(memarea_t *area,
         rs->is_hs_dir = 1;
       } else if (!strcmp(tok->args[i], "V2Dir")) {
         rs->is_v2_dir = 1;
+      } else if (!strcmp(tok->args[i], "StaleDesc")) {
+        rs->is_staledesc = 1;
       }
     }
     /* These are implied true by having been included in a consensus made
@@ -1051,7 +1057,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
 /** Parse a v3 networkstatus vote, opinion, or consensus (depending on
  * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
 networkstatus_t *
-networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
+networkstatus_parse_vote_from_string(const char *s,
+                                     size_t s_len,
+                                     const char **eos_out,
                                      networkstatus_type_t ns_type)
 {
   smartlist_t *tokens = smartlist_new();
@@ -1067,20 +1075,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   memarea_t *area = NULL, *rs_area = NULL;
   consensus_flavor_t flav = FLAV_NS;
   char *last_kwd=NULL;
+  const char *eos = s + s_len;
 
   tor_assert(s);
 
   if (eos_out)
     *eos_out = NULL;
 
-  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
-      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
+  if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed,
+                                                 s, s_len)<0) {
     log_warn(LD_DIR, "Unable to compute digest of network-status");
     goto err;
   }
 
   area = memarea_new();
-  end_of_header = find_start_of_next_routerstatus(s);
+  end_of_header = find_start_of_next_routerstatus(s, eos);
   if (tokenize_string(area, s, end_of_header, tokens,
                       (ns_type == NS_TYPE_CONSENSUS) ?
                       networkstatus_consensus_token_table :
@@ -1111,10 +1121,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   if (ns_type != NS_TYPE_CONSENSUS) {
     const char *end_of_cert = NULL;
-    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
+    if (!(cert = tor_memstr(s, end_of_header - s,
+                            "\ndir-key-certificate-version")))
       goto err;
     ++cert;
-    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
+    ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert,
+                                                &end_of_cert);
     if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
       goto err;
   }
@@ -1424,10 +1436,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   s = end_of_header;
   ns->routerstatus_list = smartlist_new();
 
-  while (!strcmpstart(s, "r ")) {
+  while (eos - s >= 2 && fast_memeq(s, "r ", 2)) {
     if (ns->type != NS_TYPE_CONSENSUS) {
       vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
+      if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns,
                                                rs, 0, 0)) {
         smartlist_add(ns->routerstatus_list, rs);
       } else {
@@ -1435,7 +1447,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
       }
     } else {
       routerstatus_t *rs;
-      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
+      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos,
+                                                     rs_tokens,
                                                      NULL, NULL,
                                                      ns->consensus_method,
                                                      flav))) {
@@ -1465,7 +1478,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
                             vrs) {
       if (! vrs->has_ed25519_listing ||
-          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
+          fast_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
         continue;
       if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
         log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
@@ -1480,10 +1493,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   /* Parse footer; check signature. */
   footer_tokens = smartlist_new();
-  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
+  if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version ")))
     ++end_of_footer;
   else
-    end_of_footer = s + strlen(s);
+    end_of_footer = eos;
   if (tokenize_string(area,s, end_of_footer, footer_tokens,
                       networkstatus_vote_footer_token_table, 0)) {
     log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h
index 10a6f9cefc..0cf2cc88d0 100644
--- a/src/feature/dirparse/ns_parse.h
+++ b/src/feature/dirparse/ns_parse.h
@@ -12,18 +12,19 @@
 #ifndef TOR_NS_PARSE_H
 #define TOR_NS_PARSE_H
 
-int router_get_networkstatus_v3_hashes(const char *s,
+int router_get_networkstatus_v3_hashes(const char *s, size_t len,
                                        common_digests_t *digests);
-int router_get_networkstatus_v3_signed_boundaries(const char *s,
+int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len,
                                                   const char **start_out,
                                                   const char **end_out);
 int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                               const char *s);
+                                               const char *s, size_t len);
 int compare_vote_routerstatus_entries(const void **_a, const void **_b);
 
 int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
 enum networkstatus_type_t;
 networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
+                                           size_t len,
                                            const char **eos_out,
                                            enum networkstatus_type_t ns_type);
 
@@ -35,11 +36,12 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
 struct memarea_t;
 STATIC routerstatus_t *routerstatus_parse_entry_from_string(
                                      struct memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
                                      consensus_flavor_t flav);
-#endif
+#endif /* defined(NS_PARSE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NS_PARSE_H) */
diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c
index 1664a77bbe..c22ed186b8 100644
--- a/src/feature/dirparse/parsecommon.c
+++ b/src/feature/dirparse/parsecommon.c
@@ -15,6 +15,7 @@
 #include "lib/string/printf.h"
 #include "lib/memarea/memarea.h"
 #include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/ctime/di_ops.h"
 
 #include <string.h>
 
@@ -169,7 +170,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok,
   char *cp = mem;
   int j = 0;
   char *args[MAX_ARGS];
-  memset(args, 0, sizeof(args));
   while (*cp) {
     if (j == MAX_ARGS)
       return -1;
@@ -251,6 +251,16 @@ token_check_object(memarea_t *area, const char *kwd,
   return tok;
 }
 
+/** Return true iff the <b>memlen</b>-byte chunk of memory at
+ * <b>memlen</b> is the same length as <b>token</b>, and their
+ * contents are equal. */
+static bool
+mem_eq_token(const void *mem, size_t memlen, const char *token)
+{
+  size_t len = strlen(token);
+  return memlen == len && fast_memeq(mem, token, len);
+}
+
 /** Helper function: read the next token from *s, advance *s to the end of the
  * token, and return the parsed token.  Parse *<b>s</b> according to the list
  * of tokens in <b>table</b>.
@@ -266,7 +276,7 @@ get_next_token(memarea_t *area,
    * attack, a bug, or some other nonsense. */
 #define MAX_LINE_LENGTH (128*1024)
 
-  const char *next, *eol, *obstart;
+  const char *next, *eol;
   size_t obname_len;
   int i;
   directory_token_t *tok;
@@ -290,7 +300,7 @@ get_next_token(memarea_t *area,
 
   next = find_whitespace_eos(*s, eol);
 
-  if (!strcmp_len(*s, "opt", next-*s)) {
+  if (mem_eq_token(*s, next-*s, "opt")) {
     /* Skip past an "opt" at the start of the line. */
     *s = eat_whitespace_eos_no_nl(next, eol);
     next = find_whitespace_eos(*s, eol);
@@ -301,7 +311,7 @@ get_next_token(memarea_t *area,
   /* Search the table for the appropriate entry.  (I tried a binary search
    * instead, but it wasn't any faster.) */
   for (i = 0; table[i].t ; ++i) {
-    if (!strcmp_len(*s, table[i].t, next-*s)) {
+    if (mem_eq_token(*s, next-*s, table[i].t)) {
       /* We've found the keyword. */
       kwd = table[i].t;
       tok->tp = table[i].v;
@@ -352,9 +362,8 @@ get_next_token(memarea_t *area,
   if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
     goto check_object;
 
-  obstart = *s; /* Set obstart to start of object spec */
   if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
-      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      !mem_eq_token(eol-5, 5, "-----") ||   /* nuls or invalid endings */
       (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
     RET_ERR("Malformed object: bad begin line");
   }
@@ -373,8 +382,8 @@ get_next_token(memarea_t *area,
     eol = eos;
   /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
   if ((size_t)(eol-next) != 9+obname_len+5 ||
-      strcmp_len(next+9, tok->object_type, obname_len) ||
-      strcmp_len(eol-5, "-----", 5)) {
+      !mem_eq_token(next+9, obname_len, tok->object_type) ||
+      !mem_eq_token(eol-5, 5, "-----")) {
     tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
              tok->object_type);
     ebuf[sizeof(ebuf)-1] = '\0';
@@ -383,28 +392,32 @@ get_next_token(memarea_t *area,
   if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
     RET_ERR("Couldn't parse object: missing footer or object much too big.");
 
+  {
+    int r;
+    size_t maxsize = base64_decode_maxsize(next-*s);
+    tok->object_body = ALLOC(maxsize);
+    r = base64_decode(tok->object_body, maxsize, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+
   if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
     if (o_syn != NEED_KEY && o_syn != NEED_KEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected public key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size);
+    if (! tok->key)
       RET_ERR("Couldn't parse public key.");
   } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
     if (o_syn != NEED_SKEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected private key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_private_key1024_from_string(tok->key,
-                                                   obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode_private(tok->object_body,
+                                             tok->object_size,
+                                             1024);
+    if (! tok->key)
       RET_ERR("Couldn't parse private key.");
-  } else { /* If it's something else, try to base64-decode it */
-    int r;
-    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
-    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
-    if (r<0)
-      RET_ERR("Malformed object: bad base64-encoded data");
-    tok->object_size = r;
   }
   *s = eol;
 
diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c
index e44fbf77f9..f78c46f186 100644
--- a/src/feature/dirparse/routerparse.c
+++ b/src/feature/dirparse/routerparse.c
@@ -591,8 +591,8 @@ router_parse_entry_from_string(const char *s, const char *end,
              "Relay's onion key had invalid exponent.");
     goto err;
   }
-  router_set_rsa_onion_pkey(tok->key, &router->onion_pkey,
-                            &router->onion_pkey_len);
+  router->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  router->onion_pkey_len = tok->object_size;
   crypto_pk_free(tok->key);
 
   if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
diff --git a/src/feature/dirparse/sigcommon.h b/src/feature/dirparse/sigcommon.h
index fdd8e839a9..b6b34e8f62 100644
--- a/src/feature/dirparse/sigcommon.h
+++ b/src/feature/dirparse/sigcommon.h
@@ -43,6 +43,6 @@ MOCK_DECL(STATIC int, signed_digest_equals,
 MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
                            const char *start, size_t len,
                            digest_algorithm_t alg));
-#endif
+#endif /* defined(SIGCOMMON_PRIVATE) */
 
 #endif /* !defined(TOR_SIGCOMMON_H) */
diff --git a/src/feature/dirparse/signing.h b/src/feature/dirparse/signing.h
index 2e3699baf8..8b119b4eb2 100644
--- a/src/feature/dirparse/signing.h
+++ b/src/feature/dirparse/signing.h
@@ -20,4 +20,4 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
                                    const char *digest,
                                    size_t digest_len,
                                    crypto_pk_t *private_key);
-#endif
+#endif /* !defined(TOR_SIGNING_H) */
diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h
index 853fe8cb0f..36c6b5a1ec 100644
--- a/src/feature/dirparse/unparseable.h
+++ b/src/feature/dirparse/unparseable.h
@@ -26,7 +26,7 @@ void dump_desc_init(void);
   log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
             name, (unsigned long)alloc, (unsigned long)used);     \
   STMT_END
-#else /* !(defined(DEBUG_AREA_ALLOC)) */
+#else /* !defined(DEBUG_AREA_ALLOC) */
 #define DUMP_AREA(a,name) STMT_NIL
 #endif /* defined(DEBUG_AREA_ALLOC) */
 
@@ -51,6 +51,6 @@ EXTERN(struct smartlist_t *, descs_dumped)
 MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
     (const char *dirname, const char *f));
 STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-#endif
+#endif /* defined(UNPARSEABLE_PRIVATE) */
 
 #endif /* !defined(TOR_UNPARSEABLE_H) */
diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c
index 09932c97ac..f7847d9a16 100644
--- a/src/feature/hibernate/hibernate.c
+++ b/src/feature/hibernate/hibernate.c
@@ -35,8 +35,9 @@ hibernating, phase 2:
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "feature/hibernate/hibernate.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/relay/router.h"
@@ -56,7 +57,7 @@ hibernating, phase 2:
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-# endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
@@ -66,8 +67,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL;
 /** If are hibernating, when do we plan to wake up? Set to 0 if we
  * aren't hibernating. */
 static time_t hibernate_end_time = 0;
-/** If we are shutting down, when do we plan finally exit? Set to 0 if
- * we aren't shutting down. */
+/** If we are shutting down, when do we plan to finally exit? Set to 0 if we
+ * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed
+ * to happen from mainloop_schedule_shutdown() now.) */
 static time_t shutdown_time = 0;
 
 /** A timed event that we'll use when it's time to wake up from
@@ -560,7 +562,7 @@ time_to_record_bandwidth_usage(time_t now)
   /* Note every 600 sec */
 #define NOTE_INTERVAL (600)
   /* Or every 20 megabytes */
-#define NOTE_BYTES 20*(1024*1024)
+#define NOTE_BYTES (20*1024*1024)
   static uint64_t last_read_bytes_noted = 0;
   static uint64_t last_written_bytes_noted = 0;
   static time_t last_time_noted = 0;
@@ -813,7 +815,7 @@ hibernate_soft_limit_reached(void)
    * We want to stop accepting connections when ALL of the following are true:
    *   - We expect to use up the remaining bytes in under 3 hours
    *   - We have used up 95% of our bytes.
-   *   - We have less than 500MB of bytes left.
+   *   - We have less than 500MBytes left.
    */
   uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT);
   if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) {
@@ -831,8 +833,6 @@ hibernate_soft_limit_reached(void)
   return get_accounting_bytes() >= soft_limit;
 }
 
-#define TOR_USEC_PER_SEC (1000000)
-
 /** Called when we get a SIGINT, or when bandwidth soft limit is
  * reached. Puts us into "loose hibernation": we don't accept new
  * connections, but we continue handling old ones. */
@@ -867,7 +867,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
     log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new "
                "connections, and will shut down in %d seconds. Interrupt "
                "again to exit now.", options->ShutdownWaitLength);
-    shutdown_time = time(NULL) + options->ShutdownWaitLength;
+    /* We add an arbitrary delay here so that even if something goes wrong
+     * with the mainloop shutdown code, we can still shutdown from
+     * consider_hibernation() if we call it... but so that the
+     * mainloop_schedule_shutdown() mechanism will be the first one called.
+     */
+    shutdown_time = time(NULL) + options->ShutdownWaitLength + 5;
+    mainloop_schedule_shutdown(options->ShutdownWaitLength);
 #ifdef HAVE_SYSTEMD
     /* tell systemd that we may need more than the default 90 seconds to shut
      * down so they don't kill us. add some extra time to actually finish
@@ -887,7 +893,7 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
      */
     sd_notifyf(0, "EXTEND_TIMEOUT_USEC=%" PRIu64,
             ((uint64_t)(options->ShutdownWaitLength) + 30) * TOR_USEC_PER_SEC);
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
   } else { /* soft limit reached */
     hibernate_end_time = interval_end_time;
   }
@@ -1096,11 +1102,12 @@ consider_hibernation(time_t now)
   hibernate_state_t prev_state = hibernate_state;
 
   /* If we're in 'exiting' mode, then we just shut down after the interval
-   * elapses. */
+   * elapses.  The mainloop was supposed to catch this via
+   * mainloop_schedule_shutdown(), but apparently it didn't. */
   if (hibernate_state == HIBERNATE_STATE_EXITING) {
     tor_assert(shutdown_time);
     if (shutdown_time <= now) {
-      log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+      log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting.");
       tor_shutdown_event_loop_and_exit(0);
     }
     return; /* if exiting soon, don't worry about bandwidth limits */
@@ -1112,7 +1119,7 @@ consider_hibernation(time_t now)
     if (hibernate_end_time > now && accounting_enabled) {
       /* If we're hibernating, don't wake up until it's time, regardless of
        * whether we're in a new interval. */
-      return ;
+      return;
     } else {
       hibernate_end_time_elapsed(now);
     }
@@ -1240,8 +1247,6 @@ on_hibernate_state_change(hibernate_state_t prev_state)
   if (prev_state != HIBERNATE_STATE_INITIAL) {
     rescan_periodic_events(get_options());
   }
-
-  reschedule_per_second_timer();
 }
 
 /** Free all resources held by the accounting module */
diff --git a/src/feature/hibernate/hibernate.h b/src/feature/hibernate/hibernate.h
index 3309ef0ce3..2e245f6ab1 100644
--- a/src/feature/hibernate/hibernate.h
+++ b/src/feature/hibernate/hibernate.h
@@ -32,6 +32,7 @@ int getinfo_helper_accounting(control_connection_t *conn,
                               const char **errmsg);
 uint64_t get_accounting_max_total(void);
 void accounting_free_all(void);
+bool accounting_tor_is_dormant(void);
 
 #ifdef HIBERNATE_PRIVATE
 /** Possible values of hibernate_state */
diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c
index 05f9940ae6..9817113b23 100644
--- a/src/feature/hs/hs_cache.c
+++ b/src/feature/hs/hs_cache.c
@@ -710,6 +710,11 @@ cache_clean_v3_as_client(time_t now)
     MAP_DEL_CURRENT(key);
     entry_size = cache_get_client_entry_size(entry);
     bytes_removed += entry_size;
+    /* We just removed an old descriptor. We need to close all intro circuits
+     * so we don't have leftovers that can be selected while lacking a
+     * descriptor. We leave the rendezvous circuits opened because they could
+     * be in use. */
+    hs_client_close_intro_circuits_from_desc(entry->desc);
     /* Entry is not in the cache anymore, destroy it. */
     cache_client_desc_free(entry);
     /* Update our OOM. We didn't use the remove() function because we are in
diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c
index 613ffe7260..df59f73c1b 100644
--- a/src/feature/hs/hs_cell.c
+++ b/src/feature/hs/hs_cell.c
@@ -473,10 +473,132 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
   }
 }
 
+/* Build and add to the given DoS cell extension the given parameter type and
+ * value. */
+static void
+build_establish_intro_dos_param(trn_cell_extension_dos_t *dos_ext,
+                                uint8_t param_type, uint64_t param_value)
+{
+  trn_cell_extension_dos_param_t *dos_param =
+    trn_cell_extension_dos_param_new();
+
+  /* Extra safety. We should never send an unknown parameter type. */
+  tor_assert(param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC ||
+             param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+
+  trn_cell_extension_dos_param_set_type(dos_param, param_type);
+  trn_cell_extension_dos_param_set_value(dos_param, param_value);
+  trn_cell_extension_dos_add_params(dos_ext, dos_param);
+
+  /* Not freeing the trunnel object because it is now owned by dos_ext. */
+}
+
+/* Build the DoS defense cell extension and put it in the given extensions
+ * object. Return 0 on success, -1 on failure.  (Right now, failure is only
+ * possible if there is a bug.) */
+static int
+build_establish_intro_dos_extension(const hs_service_config_t *service_config,
+                                    trn_cell_extension_t *extensions)
+{
+  ssize_t ret;
+  size_t dos_ext_encoded_len;
+  uint8_t *field_array;
+  trn_cell_extension_field_t *field = NULL;
+  trn_cell_extension_dos_t *dos_ext = NULL;
+
+  tor_assert(service_config);
+  tor_assert(extensions);
+
+  /* We are creating a cell extension field of the type DoS. */
+  field = trn_cell_extension_field_new();
+  trn_cell_extension_field_set_field_type(field,
+                                          TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+
+  /* Build DoS extension field. We will put in two parameters. */
+  dos_ext = trn_cell_extension_dos_new();
+  trn_cell_extension_dos_set_n_params(dos_ext, 2);
+
+  /* Build DoS parameter INTRO2 rate per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC,
+                                  service_config->intro_dos_rate_per_sec);
+  /* Build DoS parameter INTRO2 burst per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC,
+                                  service_config->intro_dos_burst_per_sec);
+
+  /* Set the field with the encoded DoS extension. */
+  ret = trn_cell_extension_dos_encoded_len(dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  dos_ext_encoded_len = ret;
+  /* Set length field and the field array size length. */
+  trn_cell_extension_field_set_field_len(field, dos_ext_encoded_len);
+  trn_cell_extension_field_setlen_field(field, dos_ext_encoded_len);
+  /* Encode the DoS extension into the cell extension field. */
+  field_array = trn_cell_extension_field_getarray_field(field);
+  ret = trn_cell_extension_dos_encode(field_array,
+                 trn_cell_extension_field_getlen_field(field), dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  tor_assert(ret == (ssize_t) dos_ext_encoded_len);
+
+  /* Finally, encode field into the cell extension. */
+  trn_cell_extension_add_fields(extensions, field);
+
+  /* We've just add an extension field to the cell extensions so increment the
+   * total number. */
+  trn_cell_extension_set_num(extensions,
+                             trn_cell_extension_get_num(extensions) + 1);
+
+  /* Cleanup. DoS extension has been encoded at this point. */
+  trn_cell_extension_dos_free(dos_ext);
+
+  return 0;
+
+ err:
+  trn_cell_extension_field_free(field);
+  trn_cell_extension_dos_free(dos_ext);
+  return -1;
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
+/* Allocate and build all the ESTABLISH_INTRO cell extension. The given
+ * extensions pointer is always set to a valid cell extension object. */
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip)
+{
+  int ret;
+  trn_cell_extension_t *extensions;
+
+  tor_assert(service_config);
+  tor_assert(ip);
+
+  extensions = trn_cell_extension_new();
+  trn_cell_extension_set_num(extensions, 0);
+
+  /* If the defense has been enabled service side (by the operator with a
+   * torrc option) and the intro point does support it. */
+  if (service_config->has_dos_defense_enabled &&
+      ip->support_intro2_dos_defense) {
+    /* This function takes care to increment the number of extensions. */
+    ret = build_establish_intro_dos_extension(service_config, extensions);
+    if (ret < 0) {
+      /* Return no extensions on error. */
+      goto end;
+    }
+  }
+
+ end:
+  return extensions;
+}
+
 /* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
  * object. The encoded cell is put in cell_out that MUST at least be of the
  * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else
@@ -484,15 +606,17 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
  * legacy cell creation. */
 ssize_t
 hs_cell_build_establish_intro(const char *circ_nonce,
+                              const hs_service_config_t *service_config,
                               const hs_service_intro_point_t *ip,
                               uint8_t *cell_out)
 {
   ssize_t cell_len = -1;
   uint16_t sig_len = ED25519_SIG_LEN;
-  trn_cell_extension_t *ext;
   trn_cell_establish_intro_t *cell = NULL;
+  trn_cell_extension_t *extensions;
 
   tor_assert(circ_nonce);
+  tor_assert(service_config);
   tor_assert(ip);
 
   /* Quickly handle the legacy IP. */
@@ -505,11 +629,12 @@ hs_cell_build_establish_intro(const char *circ_nonce,
     goto done;
   }
 
+  /* Build the extensions, if any. */
+  extensions = build_establish_intro_extensions(service_config, ip);
+
   /* Set extension data. None used here. */
-  ext = trn_cell_extension_new();
-  trn_cell_extension_set_num(ext, 0);
   cell = trn_cell_establish_intro_new();
-  trn_cell_establish_intro_set_extensions(cell, ext);
+  trn_cell_establish_intro_set_extensions(cell, extensions);
   /* Set signature size. Array is then allocated in the cell. We need to do
    * this early so we can use trunnel API to get the signature length. */
   trn_cell_establish_intro_set_sig_len(cell, sig_len);
@@ -758,7 +883,14 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
        idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) {
     link_specifier_t *lspec =
       trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx);
-    smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec));
+    if (BUG(!lspec)) {
+      goto done;
+    }
+    link_specifier_t *lspec_dup = link_specifier_dup(lspec);
+    if (BUG(!lspec_dup)) {
+      goto done;
+    }
+    smartlist_add(data->link_specifiers, lspec_dup);
   }
 
   /* Success. */
@@ -949,4 +1081,3 @@ hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
   /* The data object has no ownership of any members. */
   memwipe(data, 0, sizeof(hs_cell_introduce1_data_t));
 }
-
diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h
index 9569de535e..864b6fda5f 100644
--- a/src/feature/hs/hs_cell.h
+++ b/src/feature/hs/hs_cell.h
@@ -79,6 +79,7 @@ typedef struct hs_cell_introduce2_data_t {
 
 /* Build cell API. */
 ssize_t hs_cell_build_establish_intro(const char *circ_nonce,
+                                      const hs_service_config_t *config,
                                       const hs_service_intro_point_t *ip,
                                       uint8_t *cell_out);
 ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
@@ -105,5 +106,15 @@ int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
 /* Util API. */
 void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data);
 
+#ifdef TOR_UNIT_TESTS
+
+#include "trunnel/hs/cell_common.h"
+
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
 #endif /* !defined(TOR_HS_CELL_H) */
 
diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c
index 8acfcbd65b..5e213b5aba 100644
--- a/src/feature/hs/hs_circuit.c
+++ b/src/feature/hs/hs_circuit.c
@@ -15,6 +15,7 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuit.h"
@@ -89,7 +90,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
   cpath = tor_malloc_zero(sizeof(crypt_path_t));
   cpath->magic = CRYPT_PATH_MAGIC;
 
-  if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys),
+  if (cpath_init_circuit_crypto(cpath, (char*)keys, sizeof(keys),
                                 is_service_side, 1) < 0) {
     tor_free(cpath);
     goto err;
@@ -126,7 +127,7 @@ create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
     goto err;
   }
   /* ... and set up cpath. */
-  if (circuit_init_cpath_crypto(hop,
+  if (cpath_init_circuit_crypto(hop,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 0, 0) < 0)
     goto err;
@@ -177,7 +178,7 @@ finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
   circ->hs_circ_has_timed_out = 0;
 
   /* Append the hop to the cpath of this circuit */
-  onion_append_to_cpath(&circ->cpath, hop);
+  cpath_extend_linked_list(&circ->cpath, hop);
 
   /* In legacy code, 'pending_final_cpath' points to the final hop we just
    * appended to the cpath. We set the original pointer to NULL so that we
@@ -258,8 +259,7 @@ create_rp_circuit_identifier(const hs_service_t *service,
   tor_assert(server_pk);
   tor_assert(keys);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_RENDEZVOUS);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */
   memcpy(ident->rendezvous_cookie, rendezvous_cookie,
          sizeof(ident->rendezvous_cookie));
@@ -293,8 +293,7 @@ create_intro_circuit_identifier(const hs_service_t *service,
   tor_assert(service);
   tor_assert(ip);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_INTRO);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey);
 
   return ident;
@@ -318,7 +317,7 @@ send_establish_intro(const hs_service_t *service,
 
   /* Encode establish intro cell. */
   cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce,
-                                           ip, payload);
+                                           &service->config, ip, payload);
   if (cell_len < 0) {
     log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s "
                       "on circuit %u. Closing circuit.",
@@ -388,10 +387,7 @@ launch_rendezvous_point_circuit(const hs_service_t *service,
                                         &data->onion_pk,
                                         service->config.is_single_onion);
   if (info == NULL) {
-    /* We are done here, we can't extend to the rendezvous point.
-     * If you're running an IPv6-only v3 single onion service on 0.3.2 or with
-     * 0.3.2 clients, and somehow disable the option check, it will fail here.
-     */
+    /* We are done here, we can't extend to the rendezvous point. */
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Not enough info to open a circuit to a rendezvous point for "
            "%s service %s.",
@@ -569,81 +565,6 @@ retry_service_rendezvous_point(const origin_circuit_t *circ)
   return;
 }
 
-/* Add all possible link specifiers in node to lspecs:
- *  - legacy ID is mandatory thus MUST be present in node;
- *  - include ed25519 link specifier if present in the node, and the node
- *    supports ed25519 link authentication, even if its link versions are not
- *    compatible with us;
- *  - include IPv4 link specifier, if the primary address is not IPv4, log a
- *    BUG() warning, and return an empty smartlist;
- *  - include IPv6 link specifier if present in the node. */
-static void
-get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
-{
-  link_specifier_t *ls;
-  tor_addr_port_t ap;
-
-  tor_assert(node);
-  tor_assert(lspecs);
-
-  /* Get the relay's IPv4 address. */
-  node_get_prim_orport(node, &ap);
-
-  /* We expect the node's primary address to be a valid IPv4 address.
-   * This conforms to the protocol, which requires either an IPv4 or IPv6
-   * address (or both). */
-  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
-      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
-    return;
-  }
-
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_IPV4);
-  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
-  link_specifier_set_un_ipv4_port(ls, ap.port);
-  /* Four bytes IPv4 and two bytes port. */
-  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
-                            sizeof(ap.port));
-  smartlist_add(lspecs, ls);
-
-  /* Legacy ID is mandatory and will always be present in node. */
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
-  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
-         link_specifier_getlen_un_legacy_id(ls));
-  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
-  smartlist_add(lspecs, ls);
-
-  /* ed25519 ID is only included if the node has it, and the node declares a
-     protocol version that supports ed25519 link authentication, even if that
-     link version is not compatible with us. (We are sending the ed25519 key
-     to another tor, which may support different link versions.) */
-  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
-      node_supports_ed25519_link_authentication(node, 0)) {
-    ls = link_specifier_new();
-    link_specifier_set_ls_type(ls, LS_ED25519_ID);
-    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
-           link_specifier_getlen_un_ed25519_id(ls));
-    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
-    smartlist_add(lspecs, ls);
-  }
-
-  /* Check for IPv6. If so, include it as well. */
-  if (node_has_ipv6_orport(node)) {
-    ls = link_specifier_new();
-    node_get_pref_ipv6_orport(node, &ap);
-    link_specifier_set_ls_type(ls, LS_IPV6);
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
-    smartlist_add(lspecs, ls);
-  }
-}
-
 /* Using the given descriptor intro point ip, the node of the
  * rendezvous point rp_node and the service's subcredential, populate the
  * already allocated intro1_data object with the needed key material and link
@@ -666,10 +587,9 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip,
   tor_assert(subcredential);
   tor_assert(intro1_data);
 
-  /* Build the link specifiers from the extend information of the rendezvous
-   * circuit that we've picked previously. */
-  rp_lspecs = smartlist_new();
-  get_lspecs_from_node(rp_node, rp_lspecs);
+  /* Build the link specifiers from the node at the end of the rendezvous
+   * circuit that we opened for this introduction. */
+  rp_lspecs = node_get_link_specifier_smartlist(rp_node, 0);
   if (smartlist_len(rp_lspecs) == 0) {
     /* We can't rendezvous without link specifiers. */
     smartlist_free(rp_lspecs);
@@ -1060,9 +980,7 @@ hs_circ_handle_introduce2(const hs_service_t *service,
   ret = 0;
 
  done:
-  SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec,
-                    link_specifier_free(lspec));
-  smartlist_free(data.link_specifiers);
+  link_specifier_smartlist_free(data.link_specifiers);
   memwipe(&data, 0, sizeof(data));
   return ret;
 }
diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c
index 5480d5eb84..e34f564fb4 100644
--- a/src/feature/hs/hs_circuitmap.c
+++ b/src/feature/hs/hs_circuitmap.c
@@ -272,6 +272,33 @@ hs_circuitmap_get_or_circuit(hs_token_type_t type,
 
 /**** Public relay-side getters: */
 
+/* Public function: Return v2 and v3 introduction circuit to this relay.
+ * Always return a newly allocated list for which it is the caller's
+ * responsability to free it. */
+smartlist_t *
+hs_circuitmap_get_all_intro_circ_relay_side(void)
+{
+  circuit_t **iter;
+  smartlist_t *circuit_list = smartlist_new();
+
+  HT_FOREACH(iter, hs_circuitmap_ht, the_hs_circuitmap) {
+    circuit_t *circ = *iter;
+
+    /* An origin circuit or purpose is wrong or the hs token is not set to be
+     * a v2 or v3 intro relay side type, we ignore the circuit. Else, we have
+     * a match so add it to our list. */
+    if (CIRCUIT_IS_ORIGIN(circ) ||
+        circ->purpose != CIRCUIT_PURPOSE_INTRO_POINT ||
+        (circ->hs_token->type != HS_TOKEN_INTRO_V3_RELAY_SIDE &&
+         circ->hs_token->type != HS_TOKEN_INTRO_V2_RELAY_SIDE)) {
+      continue;
+    }
+    smartlist_add(circuit_list, circ);
+  }
+
+  return circuit_list;
+}
+
 /* Public function: Return a v3 introduction circuit to this relay with
  * <b>auth_key</b>. Return NULL if no such circuit is found in the
  * circuitmap. */
diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h
index c1bbb1ff1c..eac8230bbf 100644
--- a/src/feature/hs/hs_circuitmap.h
+++ b/src/feature/hs/hs_circuitmap.h
@@ -34,6 +34,8 @@ void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
 void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
                                          const ed25519_public_key_t *auth_key);
 
+smartlist_t *hs_circuitmap_get_all_intro_circ_relay_side(void);
+
 /** Public service-side API: */
 
 struct origin_circuit_t *
diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c
index c65f857419..8b63375939 100644
--- a/src/feature/hs/hs_client.c
+++ b/src/feature/hs/hs_client.c
@@ -167,9 +167,7 @@ purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
    * some point and we don't care about those anymore. */
   hs_build_blinded_pubkey(identity_pk, NULL, 0,
                           hs_get_time_period_num(0), &blinded_pk);
-  if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   /* Purge last hidden service request from cache for this blinded key. */
   hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk);
 }
@@ -356,7 +354,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   ed25519_public_key_t blinded_pubkey;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   hs_ident_dir_conn_t hs_conn_dir_ident;
-  int retval;
 
   tor_assert(hsdir);
   tor_assert(onion_identity_pk);
@@ -365,10 +362,7 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return HS_CLIENT_FETCH_ERROR;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Copy onion pk to a dir_ident so that we attach it to the dir conn */
   hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey,
@@ -407,7 +401,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
 STATIC routerstatus_t *
 pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 {
-  int retval;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   uint64_t current_time_period = hs_get_time_period_num(0);
   smartlist_t *responsible_hsdirs = NULL;
@@ -420,10 +413,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return NULL;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Get responsible hsdirs of service for this time period */
   responsible_hsdirs = smartlist_new();
@@ -436,7 +426,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 
   /* Pick an HSDir from the responsible ones. The ownership of
    * responsible_hsdirs is given to this function so no need to free it. */
-  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey);
+  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey, NULL);
 
   return hsdir_rs;
 }
@@ -461,6 +451,24 @@ fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk))
   return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs);
 }
 
+/* With a given <b>onion_identity_pk</b>, fetch its descriptor. If
+ * <b>hsdirs</b> is specified, use the directory servers specified in the list.
+ * Else, use a random server. */
+void
+hs_client_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  if (hsdirs != NULL) {
+    SMARTLIST_FOREACH_BEGIN(hsdirs, const routerstatus_t *, hsdir) {
+      directory_launch_v3_desc_fetch(onion_identity_pk, hsdir);
+    } SMARTLIST_FOREACH_END(hsdir);
+  } else {
+    fetch_v3_desc(onion_identity_pk);
+  }
+}
+
 /* Make sure that the given v3 origin circuit circ is a valid correct
  * introduction circuit. This will BUG() on any problems and hard assert if
  * the anonymity of the circuit is not ok. Return 0 on success else -1 where
@@ -530,13 +538,15 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id,
   SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                           hs_desc_intro_point_t *, ip) {
     SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                            const hs_desc_link_specifier_t *, lspec) {
+                            const link_specifier_t *, lspec) {
       /* Not all tor node have an ed25519 identity key so we still rely on the
        * legacy identity digest. */
-      if (lspec->type != LS_LEGACY_ID) {
+      if (link_specifier_get_ls_type(lspec) != LS_LEGACY_ID) {
         continue;
       }
-      if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) {
+      if (fast_memneq(legacy_id,
+                      link_specifier_getconstarray_un_legacy_id(lspec),
+                      DIGEST_LEN)) {
         break;
       }
       /* Found it. */
@@ -698,7 +708,7 @@ setup_intro_circ_auth_key(origin_circuit_t *circ)
   }
 
   /* Reaching this point means we didn't find any intro point for this circuit
-   * which is not suppose to happen. */
+   * which is not supposed to happen. */
   tor_assert_nonfatal_unreached();
 
  end:
@@ -764,24 +774,13 @@ STATIC extend_info_t *
 desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip)
 {
   extend_info_t *ei;
-  smartlist_t *lspecs = smartlist_new();
 
   tor_assert(ip);
 
-  /* We first encode the descriptor link specifiers into the binary
-   * representation which is a trunnel object. */
-  SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                          const hs_desc_link_specifier_t *, desc_lspec) {
-    link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec);
-    smartlist_add(lspecs, lspec);
-  } SMARTLIST_FOREACH_END(desc_lspec);
-
   /* Explicitly put the direct connection option to 0 because this is client
    * side and there is no such thing as a non anonymous client. */
-  ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0);
+  ei = hs_get_extend_info_from_lspecs(ip->link_specifiers, &ip->onion_key, 0);
 
-  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls));
-  smartlist_free(lspecs);
   return ei;
 }
 
@@ -1552,7 +1551,10 @@ parse_auth_file_content(const char *client_key_str)
   auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
   if (base32_decode((char *) auth->enc_seckey.secret_key,
                     sizeof(auth->enc_seckey.secret_key),
-                    seckey_b32, strlen(seckey_b32)) < 0) {
+                    seckey_b32, strlen(seckey_b32)) !=
+      sizeof(auth->enc_seckey.secret_key)) {
+    log_warn(LD_REND, "Client authorization encoded base32 private key "
+                      "can't be decoded: %s", seckey_b32);
     goto err;
   }
   strncpy(auth->onion_address, onion_address, HS_SERVICE_ADDR_LEN_BASE32);
diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h
index dadfa024b8..96a96755fd 100644
--- a/src/feature/hs/hs_client.h
+++ b/src/feature/hs/hs_client.h
@@ -44,6 +44,10 @@ typedef struct hs_client_service_authorization_t {
 void hs_client_note_connection_attempt_succeeded(
                                        const edge_connection_t *conn);
 
+void hs_client_launch_v3_desc_fetch(
+                               const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs);
+
 int hs_client_decode_descriptor(
                      const char *desc_str,
                      const ed25519_public_key_t *service_identity_pk,
diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c
index ebe49f09a5..036d23a6b0 100644
--- a/src/feature/hs/hs_common.c
+++ b/src/feature/hs/hs_common.c
@@ -21,6 +21,7 @@
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_service.h"
 #include "feature/hs_common/shared_random_client.h"
@@ -30,6 +31,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendservice.h"
+#include "feature/relay/routermode.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
@@ -84,7 +86,7 @@ set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
   return 0;
 }
 
-#else /* !(defined(HAVE_SYS_UN_H)) */
+#else /* !defined(HAVE_SYS_UN_H) */
 
 static int
 set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
@@ -926,7 +928,8 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out,
   }
 
   /* Decode address so we can extract needed fields. */
-  if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) {
+  if (base32_decode(decoded, sizeof(decoded), address, strlen(address))
+      != sizeof(decoded)) {
     log_warn(LD_REND, "Service address %s can't be decoded.",
              escaped_safe_str(address));
     goto invalid;
@@ -940,7 +943,7 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out,
   return -1;
 }
 
-/* Validate a given onion address. The length, the base32 decoding and
+/* Validate a given onion address. The length, the base32 decoding, and
  * checksum are validated. Return 1 if valid else 0. */
 int
 hs_address_is_valid(const char *address)
@@ -955,7 +958,7 @@ hs_address_is_valid(const char *address)
     goto invalid;
   }
 
-  /* Get the checksum it's suppose to be and compare it with what we have
+  /* Get the checksum it's supposed to be and compare it with what we have
    * encoded in the address. */
   build_hs_checksum(&service_pubkey, version, target_checksum);
   if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) {
@@ -983,7 +986,7 @@ hs_address_is_valid(const char *address)
  * The returned address is base32 encoded and put in addr_out. The caller MUST
  * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long.
  *
- * Format is as follow:
+ * Format is as follows:
  *     base32(PUBKEY || CHECKSUM || VERSION)
  *     CHECKSUM = H(".onion checksum" || PUBKEY || VERSION)
  * */
@@ -1009,24 +1012,6 @@ hs_build_address(const ed25519_public_key_t *key, uint8_t version,
   tor_assert(hs_address_is_valid(addr_out));
 }
 
-/* Return a newly allocated copy of lspec. */
-link_specifier_t *
-hs_link_specifier_dup(const link_specifier_t *lspec)
-{
-  link_specifier_t *result = link_specifier_new();
-  memcpy(result, lspec, sizeof(*result));
-  /* The unrecognized field is a dynamic array so make sure to copy its
-   * content and not the pointer. */
-  link_specifier_setlen_un_unrecognized(
-                  result, link_specifier_getlen_un_unrecognized(lspec));
-  if (link_specifier_getlen_un_unrecognized(result)) {
-    memcpy(link_specifier_getarray_un_unrecognized(result),
-           link_specifier_getconstarray_un_unrecognized(lspec),
-           link_specifier_getlen_un_unrecognized(result));
-  }
-  return result;
-}
-
 /* From a given ed25519 public key pk and an optional secret, compute a
  * blinded public key and put it in blinded_pk_out. This is only useful to
  * the client side because the client only has access to the identity public
@@ -1042,7 +1027,7 @@ hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
 
   tor_assert(pk);
   tor_assert(blinded_pk_out);
-  tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
 
   build_blinded_key_param(pk, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1067,8 +1052,8 @@ hs_build_blinded_keypair(const ed25519_keypair_t *kp,
   tor_assert(kp);
   tor_assert(blinded_kp_out);
   /* Extra safety. A zeroed key is bad. */
-  tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
-  tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
 
   build_blinded_key_param(&kp->pubkey, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1300,15 +1285,15 @@ node_has_hsdir_index(const node_t *node)
 
   /* At this point, since the node has a desc, this node must also have an
    * hsdir index. If not, something went wrong, so BUG out. */
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.fetch,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_first,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_second,
                           DIGEST256_LEN))) {
     return 0;
   }
@@ -1606,20 +1591,25 @@ hs_purge_last_hid_serv_requests(void)
 /** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the
  *  one that we should use to fetch a descriptor right now. Take into account
  *  previous failed attempts at fetching this descriptor from HSDirs using the
- *  string identifier <b>req_key_str</b>.
+ *  string identifier <b>req_key_str</b>. We return whether we are rate limited
+ *  into *<b>is_rate_limited_out</b> if it is not NULL.
  *
  *  Steals ownership of <b>responsible_dirs</b>.
  *
  *  Return the routerstatus of the chosen HSDir if successful, otherwise return
  *  NULL if no HSDirs are worth trying right now. */
 routerstatus_t *
-hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
+hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str,
+              bool *is_rate_limited_out)
 {
   smartlist_t *usable_responsible_dirs = smartlist_new();
   const or_options_t *options = get_options();
   routerstatus_t *hs_dir;
   time_t now = time(NULL);
   int excluded_some;
+  bool rate_limited = false;
+  int rate_limited_count = 0;
+  int responsible_dirs_count = smartlist_len(responsible_dirs);
 
   tor_assert(req_key_str);
 
@@ -1639,6 +1629,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     if (last + hs_hsdir_requery_period(options) >= now ||
         !node || !node_has_preferred_descriptor(node, 0)) {
       SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
+      rate_limited_count++;
       continue;
     }
     if (!routerset_contains_node(options->ExcludeNodes, node)) {
@@ -1646,6 +1637,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     }
   } SMARTLIST_FOREACH_END(dir);
 
+  if (rate_limited_count > 0 || responsible_dirs_count > 0) {
+    rate_limited = rate_limited_count == responsible_dirs_count;
+  }
+
   excluded_some =
     smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
 
@@ -1657,9 +1652,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
   smartlist_free(responsible_dirs);
   smartlist_free(usable_responsible_dirs);
   if (!hs_dir) {
+    const char *warn_str = (rate_limited) ? "we are rate limited." :
+                              "we requested them all recently without success";
     log_info(LD_REND, "Could not pick one of the responsible hidden "
-                      "service directories, because we requested them all "
-                      "recently without success.");
+                      "service directories, because %s.", warn_str);
     if (options->StrictNodes && excluded_some) {
       log_warn(LD_REND, "Could not pick a hidden service directory for the "
                "requested hidden service: they are all either down or "
@@ -1671,17 +1667,23 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1);
   }
 
+  if (is_rate_limited_out != NULL) {
+    *is_rate_limited_out = rate_limited;
+  }
+
   return hs_dir;
 }
 
-/* From a list of link specifier, an onion key and if we are requesting a
- * direct connection (ex: single onion service), return a newly allocated
- * extend_info_t object. This function always returns an extend info with
- * an IPv4 address, or NULL.
+/* Given a list of link specifiers lspecs, a curve 25519 onion_key, and
+ * a direct connection boolean direct_conn (true for single onion services),
+ * return a newly allocated extend_info_t object.
+ *
+ * This function always returns an extend info with a valid IP address and
+ * ORPort, or NULL. If direct_conn is false, the IP address is always IPv4.
  *
  * It performs the following checks:
- *  if either IPv4 or legacy ID is missing, return NULL.
- *  if direct_conn, and we can't reach the IPv4 address, return NULL.
+ *  if there is no usable IP address, or legacy ID is missing, return NULL.
+ *  if direct_conn, and we can't reach any IP address, return NULL.
  */
 extend_info_t *
 hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
@@ -1690,21 +1692,40 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
 {
   int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0;
   char legacy_id[DIGEST_LEN] = {0};
-  uint16_t port_v4 = 0;
-  tor_addr_t addr_v4;
   ed25519_public_key_t ed25519_pk;
   extend_info_t *info = NULL;
+  tor_addr_port_t ap;
 
-  tor_assert(lspecs);
+  tor_addr_make_null(&ap.addr, AF_UNSPEC);
+  ap.port = 0;
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Specified link specifiers is null");
+    goto done;
+  }
+
+  if (onion_key == NULL) {
+    log_warn(LD_BUG, "Specified onion key is null");
+    goto done;
+  }
+
+  if (smartlist_len(lspecs) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND, "Empty link specifier list.");
+    /* Return NULL. */
+    goto done;
+  }
 
   SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
     switch (link_specifier_get_ls_type(ls)) {
     case LS_IPV4:
-      /* Skip if we already seen a v4. */
-      if (have_v4) continue;
-      tor_addr_from_ipv4h(&addr_v4,
+      /* Skip if we already seen a v4. If direct_conn is true, we skip this
+       * block because fascist_firewall_choose_address_ls() will set ap. If
+       * direct_conn is false, set ap to the first IPv4 address and port in
+       * the link specifiers.*/
+      if (have_v4 || direct_conn) continue;
+      tor_addr_from_ipv4h(&ap.addr,
                           link_specifier_get_un_ipv4_addr(ls));
-      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      ap.port = link_specifier_get_un_ipv4_port(ls);
       have_v4 = 1;
       break;
     case LS_LEGACY_ID:
@@ -1728,45 +1749,38 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
     }
   } SMARTLIST_FOREACH_END(ls);
 
-  /* Legacy ID is mandatory, and we require IPv4. */
-  if (!have_v4 || !have_legacy_id) {
+  /* Choose a preferred address first, but fall back to an allowed address. */
+  if (direct_conn)
+    fascist_firewall_choose_address_ls(lspecs, 0, &ap);
+
+  /* Legacy ID is mandatory, and we require an IP address. */
+  if (!tor_addr_port_is_valid_ap(&ap, 0)) {
+    /* If we're missing the IP address, log a warning and return NULL. */
+    log_info(LD_NET, "Unreachable or invalid IP address in link state");
     goto done;
   }
-
-  /* We know we have IPv4, because we just checked. */
-  if (!direct_conn) {
-    /* All clients can extend to any IPv4 via a 3-hop path. */
-    goto validate;
-  } else if (direct_conn &&
-             fascist_firewall_allows_address_addr(&addr_v4, port_v4,
-                                                  FIREWALL_OR_CONNECTION,
-                                                  0, 0)) {
-    /* Direct connection and we can reach it in IPv4 so go for it. */
-    goto validate;
-
-    /* We will add support for falling back to a 3-hop path in a later
-     * release. */
-  } else {
-    /* If we can't reach IPv4, return NULL. */
+  if (!have_legacy_id) {
+    /* If we're missing the legacy ID, log a warning and return NULL. */
+    log_warn(LD_PROTOCOL, "Missing Legacy ID in link state");
     goto done;
   }
 
-  /* We will add support for IPv6 in a later release. */
+  /* We will add support for falling back to a 3-hop path in a later
+   * release. */
 
- validate:
   /* We'll validate now that the address we've picked isn't a private one. If
-   * it is, are we allowing to extend to private address? */
-  if (!extend_info_addr_is_allowed(&addr_v4)) {
+   * it is, are we allowed to extend to private addresses? */
+  if (!extend_info_addr_is_allowed(&ap.addr)) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Requested address is private and we are not allowed to extend to "
-           "it: %s:%u", fmt_addr(&addr_v4), port_v4);
+           "it: %s:%u", fmt_addr(&ap.addr), ap.port);
     goto done;
   }
 
   /* We do have everything for which we think we can connect successfully. */
   info = extend_info_new(NULL, legacy_id,
                          (have_ed25519_id) ? &ed25519_pk : NULL, NULL,
-                         onion_key, &addr_v4, port_v4);
+                         onion_key, &ap.addr, ap.port);
  done:
   return info;
 }
@@ -1827,3 +1841,42 @@ hs_inc_rdv_stream_counter(origin_circuit_t *circ)
     tor_assert_nonfatal_unreached();
   }
 }
+
+/* Return a newly allocated link specifier object that is a copy of dst. */
+link_specifier_t *
+link_specifier_dup(const link_specifier_t *src)
+{
+  link_specifier_t *dup = NULL;
+  uint8_t *buf = NULL;
+
+  if (BUG(!src)) {
+    goto err;
+  }
+
+  ssize_t encoded_len_alloc = link_specifier_encoded_len(src);
+  if (BUG(encoded_len_alloc < 0)) {
+    goto err;
+  }
+
+  buf = tor_malloc_zero(encoded_len_alloc);
+  ssize_t encoded_len_data = link_specifier_encode(buf,
+                                                   encoded_len_alloc,
+                                                   src);
+  if (BUG(encoded_len_data < 0)) {
+    goto err;
+  }
+
+  ssize_t parsed_len = link_specifier_parse(&dup, buf, encoded_len_alloc);
+  if (BUG(parsed_len < 0)) {
+    goto err;
+  }
+
+  goto done;
+
+ err:
+  dup = NULL;
+
+ done:
+  tor_free(buf);
+  return dup;
+}
diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h
index a44505930a..3009780d90 100644
--- a/src/feature/hs/hs_common.h
+++ b/src/feature/hs/hs_common.h
@@ -217,8 +217,6 @@ uint64_t hs_get_time_period_num(time_t now);
 uint64_t hs_get_next_time_period_num(time_t now);
 time_t hs_get_start_time_of_next_time_period(time_t now);
 
-link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec);
-
 MOCK_DECL(int, hs_in_period_between_tp_and_srv,
           (const networkstatus_t *consensus, time_t now));
 
@@ -243,7 +241,8 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk,
                               int use_second_hsdir_index,
                               int for_fetching, smartlist_t *responsible_dirs);
 routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs,
-                              const char *req_key_str);
+                              const char *req_key_str,
+                              bool *is_rate_limited_out);
 
 time_t hs_hsdir_requery_period(const or_options_t *options);
 time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
@@ -262,6 +261,8 @@ extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
                           const struct curve25519_public_key_t *onion_key,
                           int direct_conn);
 
+link_specifier_t *link_specifier_dup(const link_specifier_t *src);
+
 #ifdef HS_COMMON_PRIVATE
 
 STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out);
diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c
index ee4499ef5b..3b6caaec6a 100644
--- a/src/feature/hs/hs_config.c
+++ b/src/feature/hs/hs_config.c
@@ -218,6 +218,9 @@ config_has_invalid_options(const config_line_t *line_,
 
   const char *opts_exclude_v2[] = {
     "HiddenServiceExportCircuitID",
+    "HiddenServiceEnableIntroDoSDefense",
+    "HiddenServiceEnableIntroDoSRatePerSec",
+    "HiddenServiceEnableIntroDoSBurstPerSec",
     NULL /* End marker. */
   };
 
@@ -250,6 +253,16 @@ config_has_invalid_options(const config_line_t *line_,
                             "version %" PRIu32 " of service in %s",
                  opt, service->config.version,
                  service->config.directory_path);
+
+        if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
+          /* Special case this v2 option so that we can offer alternatives.
+           * If more such special cases appear, it would be good to
+           * generalize the exception mechanism here. */
+          log_warn(LD_CONFIG, "For v3 onion service client authorization, "
+                   "please read the 'CLIENT AUTHORIZATION' section in the "
+                   "manual.");
+        }
+
         ret = 1;
         /* Continue the loop so we can find all possible options. */
         continue;
@@ -276,6 +289,15 @@ config_validate_service(const hs_service_config_t *config)
     goto invalid;
   }
 
+  /* DoS validation values. */
+  if (config->has_dos_defense_enabled &&
+      (config->intro_dos_burst_per_sec < config->intro_dos_rate_per_sec)) {
+    log_warn(LD_CONFIG, "Hidden service DoS defenses burst (%" PRIu32 ") can "
+                        "not be smaller than the rate value (%" PRIu32 ").",
+             config->intro_dos_burst_per_sec, config->intro_dos_rate_per_sec);
+    goto invalid;
+  }
+
   /* Valid. */
   return 0;
  invalid:
@@ -296,6 +318,8 @@ config_service_v3(const config_line_t *line_,
 {
   int have_num_ip = 0;
   bool export_circuit_id = false; /* just to detect duplicate options */
+  bool dos_enabled = false, dos_rate_per_sec = false;
+  bool dos_burst_per_sec = false;
   const char *dup_opt_seen = NULL;
   const config_line_t *line;
 
@@ -334,6 +358,52 @@ config_service_v3(const config_line_t *line_,
       export_circuit_id = true;
       continue;
     }
+    if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSDefense")) {
+      config->has_dos_defense_enabled =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                                           HS_CONFIG_V3_DOS_DEFENSE_DEFAULT,
+                                           1, &ok);
+      if (!ok || dos_enabled) {
+        if (dos_enabled) {
+          dup_opt_seen = line->key;
+        }
+        goto err;
+      }
+      dos_enabled = true;
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSRatePerSec")) {
+      config->intro_dos_rate_per_sec =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                              HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN,
+                              HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX, &ok);
+      if (!ok || dos_rate_per_sec) {
+        if (dos_rate_per_sec) {
+          dup_opt_seen = line->key;
+        }
+        goto err;
+      }
+      dos_rate_per_sec = true;
+      log_info(LD_REND, "Service INTRO2 DoS defenses rate set to: %" PRIu32,
+               config->intro_dos_rate_per_sec);
+      continue;
+    }
+    if (!strcasecmp(line->key, "HiddenServiceEnableIntroDoSBurstPerSec")) {
+      config->intro_dos_burst_per_sec =
+        (unsigned int) helper_parse_uint64(line->key, line->value,
+                              HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN,
+                              HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX, &ok);
+      if (!ok || dos_burst_per_sec) {
+        if (dos_burst_per_sec) {
+          dup_opt_seen = line->key;
+        }
+        goto err;
+      }
+      dos_burst_per_sec = true;
+      log_info(LD_REND, "Service INTRO2 DoS defenses burst set to: %" PRIu32,
+               config->intro_dos_burst_per_sec);
+      continue;
+    }
   }
 
   /* We do not load the key material for the service at this stage. This is
@@ -496,15 +566,6 @@ config_generic_service(const config_line_t *line_,
    * becomes a single onion service. */
   if (rend_service_non_anonymous_mode_enabled(options)) {
     config->is_single_onion = 1;
-    /* We will add support for IPv6-only v3 single onion services in a future
-     * Tor version. This won't catch "ReachableAddresses reject *4", but that
-     * option doesn't work anyway. */
-    if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) {
-      log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not "
-               "supported. Set HiddenServiceSingleHopMode 0 and "
-               "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1.");
-      goto err;
-    }
   }
 
   /* Success */
diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h
index 040e451f13..beefc7a613 100644
--- a/src/feature/hs/hs_config.h
+++ b/src/feature/hs/hs_config.h
@@ -15,6 +15,15 @@
 #define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535
 /* Maximum number of intro points per version 3 services. */
 #define HS_CONFIG_V3_MAX_INTRO_POINTS 20
+/* Default value for the introduction DoS defenses. The MIN/MAX are inclusive
+ * meaning they can be used as valid values. */
+#define HS_CONFIG_V3_DOS_DEFENSE_DEFAULT 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT 25
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX INT32_MAX
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT 200
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX INT32_MAX
 
 /* API */
 
diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c
index 9970fdd123..abb421345c 100644
--- a/src/feature/hs/hs_control.c
+++ b/src/feature/hs/hs_control.c
@@ -7,9 +7,10 @@
  **/
 
 #include "core/or/or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_control.h"
 #include "feature/hs/hs_descriptor.h"
@@ -73,10 +74,7 @@ hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
   tor_assert(reason);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk,
@@ -98,10 +96,7 @@ hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk,
@@ -122,9 +117,7 @@ hs_control_desc_event_created(const char *onion_address,
   tor_assert(blinded_pk);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   /* Version 3 doesn't use the replica number in its descriptor ID computation
    * so we pass negative value so the control port subsystem can ignore it. */
@@ -150,9 +143,7 @@ hs_control_desc_event_upload(const char *onion_address,
   tor_assert(hsdir_index);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   control_event_hs_descriptor_upload(onion_address, hsdir_id_digest,
                                      base64_blinded_pk,
@@ -195,10 +186,7 @@ hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hs_descriptor_content(onion_address, base64_blinded_pk,
@@ -259,3 +247,16 @@ hs_control_hspost_command(const char *body, const char *onion_address,
   smartlist_free(hsdirs);
   return ret;
 }
+
+/* With a given <b>onion_identity_pk</b>, fetch its descriptor, optionally
+ * using the list of directory servers given in <b>hsdirs</b>, or a random
+ * server if it is NULL. This function calls hs_client_launch_v3_desc_fetch().
+ */
+void
+hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                           const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  hs_client_launch_v3_desc_fetch(onion_identity_pk, hsdirs);
+}
diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h
index f7ab642652..b55e4c53c9 100644
--- a/src/feature/hs/hs_control.h
+++ b/src/feature/hs/hs_control.h
@@ -48,5 +48,9 @@ void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
 int hs_control_hspost_command(const char *body, const char *onion_address,
                               const smartlist_t *hsdirs_rs);
 
+/* Command "HSFETCH [...]" */
+void hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                                const smartlist_t *hsdirs);
+
 #endif /* !defined(TOR_HS_CONTROL_H) */
 
diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c
index b6abf14a11..924ab3115e 100644
--- a/src/feature/hs/hs_descriptor.c
+++ b/src/feature/hs/hs_descriptor.c
@@ -324,12 +324,11 @@ encode_link_specifiers(const smartlist_t *specs)
 
   link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
 
-  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+  SMARTLIST_FOREACH_BEGIN(specs, const link_specifier_t *,
                           spec) {
-    link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec);
-    if (ls) {
-      link_specifier_list_add_spec(lslist, ls);
-    }
+    link_specifier_t *ls = link_specifier_dup(spec);
+    tor_assert(ls);
+    link_specifier_list_add_spec(lslist, ls);
   } SMARTLIST_FOREACH_END(spec);
 
   {
@@ -404,9 +403,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "enc-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->enc_key);
   if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
     goto done;
   }
@@ -422,7 +419,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
 }
 
 /* Encode an introduction point onion key. Return a newly allocated string
- * with it. On failure, return NULL. */
+ * with it. Can not fail. */
 static char *
 encode_onion_key(const hs_desc_intro_point_t *ip)
 {
@@ -432,12 +429,9 @@ encode_onion_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "onion-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->onion_key);
   tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64);
 
- done:
   return encoded;
 }
 
@@ -684,7 +678,7 @@ get_auth_client_str(const hs_desc_authorized_client_t *client)
   char encrypted_cookie_b64[HS_DESC_ENCRYPED_COOKIE_LEN * 2];
 
 #define ASSERT_AND_BASE64(field) STMT_BEGIN                        \
-  tor_assert(!tor_mem_is_zero((char *) client->field,              \
+  tor_assert(!fast_mem_is_zero((char *) client->field,              \
                               sizeof(client->field)));             \
   ret = base64_encode_nopad(field##_b64, sizeof(field##_b64),      \
                             client->field, sizeof(client->field)); \
@@ -798,8 +792,8 @@ get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
 /* Create the middle layer of the descriptor, which includes the client auth
  * data and the encrypted inner layer (provided as a base64 string at
  * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
- * layer plaintext, or NULL if an error occurred. It's the responsibility of
- * the caller to free the returned string. */
+ * layer plaintext. It's the responsibility of the caller to free the returned
+ * string. Can not fail. */
 static char *
 get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
                                     const char *layer2_b64_ciphertext)
@@ -815,13 +809,10 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
     const curve25519_public_key_t *ephemeral_pubkey;
 
     ephemeral_pubkey = &desc->superencrypted_data.auth_ephemeral_pubkey;
-    tor_assert(!tor_mem_is_zero((char *) ephemeral_pubkey->public_key,
+    tor_assert(!fast_mem_is_zero((char *) ephemeral_pubkey->public_key,
                                 CURVE25519_PUBKEY_LEN));
 
-    if (curve25519_public_to_base64(ephemeral_key_base64,
-                                    ephemeral_pubkey) < 0) {
-      goto done;
-    }
+    curve25519_public_to_base64(ephemeral_key_base64, ephemeral_pubkey);
     smartlist_add_asprintf(lines, "%s %s\n",
                            str_desc_auth_key, ephemeral_key_base64);
 
@@ -846,7 +837,6 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
 
   layer1_str = smartlist_join_strings(lines, "", 0, NULL);
 
- done:
   /* We need to memwipe all lines because it contains the ephemeral key */
   SMARTLIST_FOREACH(lines, char *, a, memwipe(a, 0, strlen(a)));
   SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
@@ -1092,11 +1082,7 @@ desc_encode_v3(const hs_descriptor_t *desc,
       tor_free(encoded_str);
       goto err;
     }
-    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
-      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
-      tor_free(encoded_str);
-      goto err;
-    }
+    ed25519_signature_to_base64(ed_sig_b64, &sig);
     /* Create the signature line. */
     smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
   }
@@ -1190,52 +1176,22 @@ decode_link_specifiers(const char *encoded)
   results = smartlist_new();
 
   for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
-    hs_desc_link_specifier_t *hs_spec;
     link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
-    tor_assert(ls);
-
-    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
-    hs_spec->type = link_specifier_get_ls_type(ls);
-    switch (hs_spec->type) {
-    case LS_IPV4:
-      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
-                          link_specifier_get_un_ipv4_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
-      break;
-    case LS_IPV6:
-      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
-                               link_specifier_getarray_un_ipv6_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
-      break;
-    case LS_LEGACY_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
-                 sizeof(hs_spec->u.legacy_id));
-      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
-             sizeof(hs_spec->u.legacy_id));
-      break;
-    case LS_ED25519_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_ed25519_id(ls) ==
-                 sizeof(hs_spec->u.ed25519_id));
-      memcpy(hs_spec->u.ed25519_id,
-             link_specifier_getconstarray_un_ed25519_id(ls),
-             sizeof(hs_spec->u.ed25519_id));
-      break;
-    default:
-      tor_free(hs_spec);
+    if (BUG(!ls)) {
       goto err;
     }
-
-    smartlist_add(results, hs_spec);
+    link_specifier_t *ls_dup = link_specifier_dup(ls);
+    if (BUG(!ls_dup)) {
+      goto err;
+    }
+    smartlist_add(results, ls_dup);
   }
 
   goto done;
  err:
   if (results) {
-    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    SMARTLIST_FOREACH(results, link_specifier_t *, s,
+                      link_specifier_free(s));
     smartlist_free(results);
     results = NULL;
   }
@@ -1400,6 +1356,50 @@ encrypted_data_length_is_valid(size_t len)
   return 0;
 }
 
+/* Build the KEYS component for the authorized client computation. The format
+ * of the construction is:
+ *
+ *    SECRET_SEED = x25519(sk, pk)
+ *    KEYS = KDF(subcredential | SECRET_SEED, 40)
+ *
+ * Set the <b>keys_out</b> argument to point to the buffer containing the KEYS,
+ * and return the buffer's length. The caller should wipe and free its content
+ * once done with it. This function can't fail. */
+static size_t
+build_descriptor_cookie_keys(const uint8_t *subcredential,
+                             size_t subcredential_len,
+                             const curve25519_secret_key_t *sk,
+                             const curve25519_public_key_t *pk,
+                             uint8_t **keys_out)
+{
+  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
+  uint8_t *keystream;
+  size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN;
+  crypto_xof_t *xof;
+
+  tor_assert(subcredential);
+  tor_assert(sk);
+  tor_assert(pk);
+  tor_assert(keys_out);
+
+  keystream = tor_malloc_zero(keystream_len);
+
+  /* Calculate x25519(sk, pk) to get the secret seed. */
+  curve25519_handshake(secret_seed, sk, pk);
+
+  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, subcredential, subcredential_len);
+  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
+  crypto_xof_squeeze_bytes(xof, keystream, keystream_len);
+  crypto_xof_free(xof);
+
+  memwipe(secret_seed, 0, sizeof(secret_seed));
+
+  *keys_out = keystream;
+  return keystream_len;
+}
+
 /* Decrypt the descriptor cookie given the descriptor, the auth client,
  * and the client secret key. On sucess, return 0 and a newly allocated
  * descriptor cookie descriptor_cookie_out. On error or if the client id
@@ -1412,33 +1412,29 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
                           uint8_t **descriptor_cookie_out)
 {
   int ret = -1;
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key = NULL;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
   uint8_t *descriptor_cookie = NULL;
+  const uint8_t *cookie_key = NULL;
   crypto_cipher_t *cipher = NULL;
-  crypto_xof_t *xof = NULL;
 
   tor_assert(desc);
   tor_assert(client);
   tor_assert(client_auth_sk);
-  tor_assert(!tor_mem_is_zero(
+  tor_assert(!fast_mem_is_zero(
         (char *) &desc->superencrypted_data.auth_ephemeral_pubkey,
         sizeof(desc->superencrypted_data.auth_ephemeral_pubkey)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_sk,
+  tor_assert(!fast_mem_is_zero((char *) client_auth_sk,
                               sizeof(*client_auth_sk)));
-  tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN));
+  tor_assert(!fast_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN));
 
-  /* Calculate x25519(client_x, hs_Y) */
-  curve25519_handshake(secret_seed, client_auth_sk,
-                       &desc->superencrypted_data.auth_ephemeral_pubkey);
-
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(desc->subcredential, DIGEST256_LEN,
+                             client_auth_sk,
+                             &desc->superencrypted_data.auth_ephemeral_pubkey,
+                             &keystream);
+  tor_assert(keystream_length > 0);
 
   /* If the client id of auth client is not the same as the calculcated
    * client id, it means that this auth client is invaild according to the
@@ -1464,8 +1460,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
   if (cipher) {
     crypto_cipher_free(cipher);
   }
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
   return ret;
 }
 
@@ -1481,10 +1477,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
  */
 MOCK_IMPL(STATIC size_t,
 decrypt_desc_layer,(const hs_descriptor_t *desc,
-                    const uint8_t *encrypted_blob,
-                    size_t encrypted_blob_size,
                     const uint8_t *descriptor_cookie,
-                    int is_superencrypted_layer,
+                    bool is_superencrypted_layer,
                     char **decrypted_out))
 {
   uint8_t *decrypted = NULL;
@@ -1494,6 +1488,12 @@ decrypt_desc_layer,(const hs_descriptor_t *desc,
   uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
   const uint8_t *salt, *encrypted, *desc_mac;
   size_t encrypted_len, result_len = 0;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
 
   tor_assert(decrypted_out);
   tor_assert(desc);
@@ -1607,9 +1607,8 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out)
   tor_assert(decrypted_out);
 
   superencrypted_len = decrypt_desc_layer(desc,
-                                 desc->plaintext_data.superencrypted_blob,
-                                 desc->plaintext_data.superencrypted_blob_size,
-                                 NULL, 1, &superencrypted_plaintext);
+                                          NULL,
+                                          true, &superencrypted_plaintext);
 
   if (!superencrypted_len) {
     log_warn(LD_REND, "Decrypting superencrypted desc failed.");
@@ -1658,9 +1657,9 @@ desc_decrypt_encrypted(const hs_descriptor_t *desc,
   }
 
   encrypted_len = decrypt_desc_layer(desc,
-                                 desc->superencrypted_data.encrypted_blob,
-                                 desc->superencrypted_data.encrypted_blob_size,
-                                 descriptor_cookie, 0, &encrypted_plaintext);
+                                     descriptor_cookie,
+                                     false, &encrypted_plaintext);
+
   if (!encrypted_len) {
     goto err;
   }
@@ -1972,6 +1971,7 @@ decode_intro_points(const hs_descriptor_t *desc,
   SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
   smartlist_free(intro_points);
 }
+
 /* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
  * descriptor in encoded_desc validates the descriptor content. */
 STATIC int
@@ -2575,7 +2575,7 @@ hs_desc_decode_descriptor(const char *encoded,
 
   /* Subcredentials are not optional. */
   if (BUG(!subcredential ||
-          tor_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
+          fast_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
     log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!");
     goto err;
   }
@@ -2838,8 +2838,8 @@ hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
     return;
   }
   if (ip->link_specifiers) {
-    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
-                      ls, hs_desc_link_specifier_free(ls));
+    SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *,
+                      ls, link_specifier_free(ls));
     smartlist_free(ip->link_specifiers);
   }
   tor_cert_free(ip->auth_key_cert);
@@ -2878,38 +2878,33 @@ hs_desc_build_authorized_client(const uint8_t *subcredential,
                                 const uint8_t *descriptor_cookie,
                                 hs_desc_authorized_client_t *client_out)
 {
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
+  const uint8_t *cookie_key;
   crypto_cipher_t *cipher;
-  crypto_xof_t *xof;
 
   tor_assert(client_auth_pk);
   tor_assert(auth_ephemeral_sk);
   tor_assert(descriptor_cookie);
   tor_assert(client_out);
   tor_assert(subcredential);
-  tor_assert(!tor_mem_is_zero((char *) auth_ephemeral_sk,
+  tor_assert(!fast_mem_is_zero((char *) auth_ephemeral_sk,
                               sizeof(*auth_ephemeral_sk)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_pk,
+  tor_assert(!fast_mem_is_zero((char *) client_auth_pk,
                               sizeof(*client_auth_pk)));
-  tor_assert(!tor_mem_is_zero((char *) descriptor_cookie,
+  tor_assert(!fast_mem_is_zero((char *) descriptor_cookie,
                               HS_DESC_DESCRIPTOR_COOKIE_LEN));
-  tor_assert(!tor_mem_is_zero((char *) subcredential,
+  tor_assert(!fast_mem_is_zero((char *) subcredential,
                               DIGEST256_LEN));
 
-  /* Calculate x25519(hs_y, client_X) */
-  curve25519_handshake(secret_seed,
-                       auth_ephemeral_sk,
-                       client_auth_pk);
-
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(subcredential, DIGEST256_LEN,
+                                 auth_ephemeral_sk, client_auth_pk,
+                                 &keystream);
+  tor_assert(keystream_length > 0);
 
+  /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */
   memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN);
   cookie_key = keystream + HS_DESC_CLIENT_ID_LEN;
 
@@ -2924,8 +2919,8 @@ hs_desc_build_authorized_client(const uint8_t *subcredential,
                         (const char *) descriptor_cookie,
                         HS_DESC_DESCRIPTOR_COOKIE_LEN);
 
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
 
   crypto_cipher_free(cipher);
 }
@@ -2937,69 +2932,6 @@ hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client)
   tor_free(client);
 }
 
-/* Free the given descriptor link specifier. */
-void
-hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls)
-{
-  if (ls == NULL) {
-    return;
-  }
-  tor_free(ls);
-}
-
-/* Return a newly allocated descriptor link specifier using the given extend
- * info and requested type. Return NULL on error. */
-hs_desc_link_specifier_t *
-hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type)
-{
-  hs_desc_link_specifier_t *ls = NULL;
-
-  tor_assert(info);
-
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = type;
-  switch (ls->type) {
-  case LS_IPV4:
-    if (info->addr.family != AF_INET) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_IPV6:
-    if (info->addr.family != AF_INET6) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_LEGACY_ID:
-    /* Bug out if the identity digest is not set */
-    if (BUG(tor_mem_is_zero(info->identity_digest,
-                            sizeof(info->identity_digest)))) {
-      goto err;
-    }
-    memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id));
-    break;
-  case LS_ED25519_ID:
-    /* ed25519 keys are optional for intro points */
-    if (ed25519_public_key_is_zero(&info->ed_identity)) {
-      goto err;
-    }
-    memcpy(ls->u.ed25519_id, info->ed_identity.pubkey,
-           sizeof(ls->u.ed25519_id));
-    break;
-  default:
-    /* Unknown type is code flow error. */
-    tor_assert(0);
-  }
-
-  return ls;
- err:
-  tor_free(ls);
-  return NULL;
-}
-
 /* From the given descriptor, remove and free every introduction point. */
 void
 hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
@@ -3015,59 +2947,3 @@ hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
     smartlist_clear(ips);
   }
 }
-
-/* From a descriptor link specifier object spec, returned a newly allocated
- * link specifier object that is the encoded representation of spec. Return
- * NULL on error. */
-link_specifier_t *
-hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec)
-{
-  tor_assert(spec);
-
-  link_specifier_t *ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, spec->type);
-
-  switch (spec->type) {
-  case LS_IPV4:
-    link_specifier_set_un_ipv4_addr(ls,
-                                    tor_addr_to_ipv4h(&spec->u.ap.addr));
-    link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
-    /* Four bytes IPv4 and two bytes port. */
-    link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
-                                  sizeof(spec->u.ap.port));
-    break;
-  case LS_IPV6:
-  {
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
-    break;
-  }
-  case LS_LEGACY_ID:
-  {
-    size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
-    uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
-    memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
-    link_specifier_set_ls_len(ls, legacy_id_len);
-    break;
-  }
-  case LS_ED25519_ID:
-  {
-    size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls);
-    uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls);
-    memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len);
-    link_specifier_set_ls_len(ls, ed25519_id_len);
-    break;
-  }
-  default:
-    tor_assert_nonfatal_unreached();
-    link_specifier_free(ls);
-    ls = NULL;
-  }
-
-  return ls;
-}
diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h
index 04a8e16d63..0a843f4f3c 100644
--- a/src/feature/hs/hs_descriptor.h
+++ b/src/feature/hs/hs_descriptor.h
@@ -69,28 +69,10 @@ typedef enum {
   HS_DESC_AUTH_ED25519 = 1
 } hs_desc_auth_type_t;
 
-/* Link specifier object that contains information on how to extend to the
- * relay that is the address, port and handshake type. */
-typedef struct hs_desc_link_specifier_t {
-  /* Indicate the type of link specifier. See trunnel ed25519_cert
-   * specification. */
-  uint8_t type;
-
-  /* It must be one of these types, can't be more than one. */
-  union {
-    /* IP address and port of the relay use to extend. */
-    tor_addr_port_t ap;
-    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
-    uint8_t legacy_id[DIGEST_LEN];
-    /* ed25519 identity. A 32-byte key. */
-    uint8_t ed25519_id[ED25519_PUBKEY_LEN];
-  } u;
-} hs_desc_link_specifier_t;
-
 /* Introduction point information located in a descriptor. */
 typedef struct hs_desc_intro_point_t {
   /* Link specifier(s) which details how to extend to the relay. This list
-   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+   * contains link_specifier_t objects. It MUST have at least one. */
   smartlist_t *link_specifiers;
 
   /* Onion key of the introduction point used to extend to it for the ntor
@@ -261,12 +243,6 @@ void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc);
 #define hs_desc_encrypted_data_free(desc) \
   FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc))
 
-void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls);
-#define hs_desc_link_specifier_free(ls) \
-  FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls))
-
-hs_desc_link_specifier_t *hs_desc_link_specifier_new(
-                                  const extend_info_t *info, uint8_t type);
 void hs_descriptor_clear_intro_points(hs_descriptor_t *desc);
 
 MOCK_DECL(int,
@@ -299,10 +275,8 @@ void hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client);
   FREE_AND_NULL(hs_desc_authorized_client_t, \
                 hs_desc_authorized_client_free_, (client))
 
-link_specifier_t *hs_desc_lspec_to_trunnel(
-                                   const hs_desc_link_specifier_t *spec);
-
 hs_desc_authorized_client_t *hs_desc_build_fake_authorized_client(void);
+
 void hs_desc_build_authorized_client(const uint8_t *subcredential,
                                      const curve25519_public_key_t *
                                      client_auth_pk,
@@ -335,10 +309,8 @@ STATIC int desc_sig_is_valid(const char *b64_sig,
                              const char *encoded_desc, size_t encoded_len);
 
 MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc,
-                                             const uint8_t *encrypted_blob,
-                                             size_t encrypted_blob_size,
                                              const uint8_t *descriptor_cookie,
-                                             int is_superencrypted_layer,
+                                             bool is_superencrypted_layer,
                                              char **decrypted_out));
 
 #endif /* defined(HS_DESCRIPTOR_PRIVATE) */
diff --git a/src/feature/hs/hs_dos.c b/src/feature/hs/hs_dos.c
new file mode 100644
index 0000000000..19794e09d3
--- /dev/null
+++ b/src/feature/hs/hs_dos.c
@@ -0,0 +1,200 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.c
+ * \brief Implement denial of service mitigation for the onion service
+ *        subsystem.
+ *
+ * This module defenses:
+ *
+ * - Introduction Rate Limiting: If enabled by the consensus, an introduction
+ *   point will rate limit client introduction towards the service (INTRODUCE2
+ *   cells). It uses a token bucket model with a rate and burst per second.
+ *
+ *   Proposal 305 will expand this module by allowing an operator to define
+ *   these values into the ESTABLISH_INTRO cell. Not yet implemented.
+ **/
+
+#define HS_DOS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/routermode.h"
+
+#include "lib/evloop/token_bucket.h"
+
+#include "feature/hs/hs_dos.h"
+
+/* Default value of the allowed INTRODUCE2 cell rate per second. Above that
+ * value per second, the introduction is denied. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC 25
+
+/* Default value of the allowed INTRODUCE2 cell burst per second. This is the
+ * maximum value a token bucket has per second. We thus allow up to this value
+ * of INTRODUCE2 cell per second but the bucket is refilled by the rate value
+ * but never goes above that burst value. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC 200
+
+/* Default value of the consensus parameter enabling or disabling the
+ * introduction DoS defense. Disabled by default. */
+#define HS_DOS_INTRODUCE_ENABLED_DEFAULT 0
+
+/* Consensus parameters. The ESTABLISH_INTRO DoS cell extension have higher
+ * priority than these values. If no extension is sent, these are used only by
+ * the introduction point. */
+static uint32_t consensus_param_introduce_rate_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC;
+static uint32_t consensus_param_introduce_burst_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC;
+static uint32_t consensus_param_introduce_defense_enabled =
+  HS_DOS_INTRODUCE_ENABLED_DEFAULT;
+
+STATIC uint32_t
+get_intro2_enable_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSDefense",
+                                 HS_DOS_INTRODUCE_ENABLED_DEFAULT, 0, 1);
+}
+
+/* Return the parameter for the introduction rate per sec. */
+STATIC uint32_t
+get_intro2_rate_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSRatePerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/* Return the parameter for the introduction burst per sec. */
+STATIC uint32_t
+get_intro2_burst_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSBurstPerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/* Go over all introduction circuit relay side and adjust their rate/burst
+ * values using the global parameters. This is called right after the
+ * consensus parameters might have changed. */
+static void
+update_intro_circuits(void)
+{
+  /* Returns all HS version intro circuits. */
+  smartlist_t *intro_circs = hs_circuitmap_get_all_intro_circ_relay_side();
+
+  SMARTLIST_FOREACH_BEGIN(intro_circs, circuit_t *, circ) {
+    /* Defenses might have been enabled or disabled. */
+    TO_OR_CIRCUIT(circ)->introduce2_dos_defense_enabled =
+      consensus_param_introduce_defense_enabled;
+    /* Adjust the rate/burst value that might have changed. */
+    token_bucket_ctr_adjust(&TO_OR_CIRCUIT(circ)->introduce2_bucket,
+                            consensus_param_introduce_rate_per_sec,
+                            consensus_param_introduce_burst_per_sec);
+  } SMARTLIST_FOREACH_END(circ);
+
+  smartlist_free(intro_circs);
+}
+
+/* Set consensus parameters. */
+static void
+set_consensus_parameters(const networkstatus_t *ns)
+{
+  consensus_param_introduce_rate_per_sec =
+    get_intro2_rate_consensus_param(ns);
+  consensus_param_introduce_burst_per_sec =
+    get_intro2_burst_consensus_param(ns);
+  consensus_param_introduce_defense_enabled =
+    get_intro2_enable_consensus_param(ns);
+
+  /* The above might have changed which means we need to go through all
+   * introduction circuits (relay side) and update the token buckets. */
+  update_intro_circuits();
+}
+
+/*
+ * Public API.
+ */
+
+/* Initialize the INTRODUCE2 token bucket for the DoS defenses using the
+ * consensus/default values. We might get a cell extension that changes those
+ * later but if we don't, the default or consensus parameters are used. */
+void
+hs_dos_setup_default_intro2_defenses(or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  circ->introduce2_dos_defense_enabled =
+    consensus_param_introduce_defense_enabled;
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        consensus_param_introduce_rate_per_sec,
+                        consensus_param_introduce_burst_per_sec,
+                        (uint32_t) approx_time());
+}
+
+/* Called when the consensus has changed. We might have new consensus
+ * parameters to look at. */
+void
+hs_dos_consensus_has_changed(const networkstatus_t *ns)
+{
+  /* No point on updating these values if we are not a public relay that can
+   * be picked to be an introduction point. */
+  if (!public_server_mode(get_options())) {
+    return;
+  }
+
+  set_consensus_parameters(ns);
+}
+
+/* Return true iff an INTRODUCE2 cell can be sent on the given service
+ * introduction circuit. */
+bool
+hs_dos_can_send_intro2(or_circuit_t *s_intro_circ)
+{
+  tor_assert(s_intro_circ);
+
+  /* Allow to send the cell if the DoS defenses are disabled on the circuit.
+   * This can be set by the consensus, the ESTABLISH_INTRO cell extension or
+   * the hardcoded values in tor code. */
+  if (!s_intro_circ->introduce2_dos_defense_enabled) {
+    return true;
+  }
+
+  /* Should not happen but if so, scream loudly. */
+  if (BUG(TO_CIRCUIT(s_intro_circ)->purpose != CIRCUIT_PURPOSE_INTRO_POINT)) {
+    return false;
+  }
+
+  /* This is called just after we got a valid and parsed INTRODUCE1 cell. The
+   * service has been found and we have its introduction circuit.
+   *
+   * First, the INTRODUCE2 bucket will be refilled (if any). Then, decremented
+   * because we are about to send or not the cell we just got. Finally,
+   * evaluate if we can send it based on our token bucket state. */
+
+  /* Refill INTRODUCE2 bucket. */
+  token_bucket_ctr_refill(&s_intro_circ->introduce2_bucket,
+                          (uint32_t) approx_time());
+
+  /* Decrement the bucket for this valid INTRODUCE1 cell we just got. Don't
+   * underflow else we end up with a too big of a bucket. */
+  if (token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0) {
+    token_bucket_ctr_dec(&s_intro_circ->introduce2_bucket, 1);
+  }
+
+  /* Finally, we can send a new INTRODUCE2 if there are still tokens. */
+  return token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0;
+}
+
+/* Initialize the onion service Denial of Service subsystem. */
+void
+hs_dos_init(void)
+{
+  set_consensus_parameters(NULL);
+}
diff --git a/src/feature/hs/hs_dos.h b/src/feature/hs/hs_dos.h
new file mode 100644
index 0000000000..ccf4e27179
--- /dev/null
+++ b/src/feature/hs/hs_dos.h
@@ -0,0 +1,39 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.h
+ * \brief Header file containing denial of service defenses for the HS
+ *        subsystem for all versions.
+ **/
+
+#ifndef TOR_HS_DOS_H
+#define TOR_HS_DOS_H
+
+#include "core/or/or_circuit_st.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+/* Init */
+void hs_dos_init(void);
+
+/* Consensus. */
+void hs_dos_consensus_has_changed(const networkstatus_t *ns);
+
+/* Introduction Point. */
+bool hs_dos_can_send_intro2(or_circuit_t *s_intro_circ);
+void hs_dos_setup_default_intro2_defenses(or_circuit_t *circ);
+
+#ifdef HS_DOS_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC uint32_t get_intro2_enable_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_rate_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_burst_consensus_param(const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_DOS_PRIVATE) */
+
+#endif /* !defined(TOR_HS_DOS_H) */
diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c
index 8fd0013941..a00e55ec23 100644
--- a/src/feature/hs/hs_ident.c
+++ b/src/feature/hs/hs_ident.c
@@ -13,14 +13,10 @@
 /* Return a newly allocated circuit identifier. The given public key is copied
  * identity_pk into the identifier. */
 hs_ident_circuit_t *
-hs_ident_circuit_new(const ed25519_public_key_t *identity_pk,
-                     hs_ident_circuit_type_t circuit_type)
+hs_ident_circuit_new(const ed25519_public_key_t *identity_pk)
 {
-  tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO ||
-             circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS);
   hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
   ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
-  ident->circuit_type = circuit_type;
   return ident;
 }
 
diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h
index 8c46936a1e..82ca50f6b5 100644
--- a/src/feature/hs/hs_ident.h
+++ b/src/feature/hs/hs_ident.h
@@ -44,13 +44,6 @@ typedef struct hs_ident_circuit_t {
    * the one found in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* (All circuit) The type of circuit this identifier is attached to.
-   * Accessors of the fields in this object assert non fatal on this circuit
-   * type. In other words, if a rendezvous field is being accessed, the
-   * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is
-   * set when an object is initialized in its constructor. */
-  hs_ident_circuit_type_t circuit_type;
-
   /* (All circuit) Introduction point authentication key. It's also needed on
    * the rendezvous circuit for the ntor handshake. It's used as the unique key
    * of the introduction point so it should not be shared between multiple
@@ -120,8 +113,7 @@ typedef struct hs_ident_edge_conn_t {
 
 /* Circuit identifier API. */
 hs_ident_circuit_t *hs_ident_circuit_new(
-                             const ed25519_public_key_t *identity_pk,
-                             hs_ident_circuit_type_t circuit_type);
+                             const ed25519_public_key_t *identity_pk);
 void hs_ident_circuit_free_(hs_ident_circuit_t *ident);
 #define hs_ident_circuit_free(id) \
   FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id))
diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c
index 7717ed53d4..fe8486b1a6 100644
--- a/src/feature/hs/hs_intropoint.c
+++ b/src/feature/hs/hs_intropoint.c
@@ -10,6 +10,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/relay.h"
@@ -24,9 +25,11 @@
 #include "trunnel/hs/cell_introduce1.h"
 
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
 #include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
-#include "feature/hs/hs_common.h"
 
 #include "core/or/or_circuit_st.h"
 
@@ -179,6 +182,185 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ))
   return ret;
 }
 
+/* Validate the cell DoS extension parameters. Return true iff they've been
+ * bound check and can be used. Else return false. See proposal 305 for
+ * details and reasons about this validation. */
+STATIC bool
+cell_dos_extension_parameters_are_valid(uint64_t intro2_rate_per_sec,
+                                        uint64_t intro2_burst_per_sec)
+{
+  bool ret = false;
+
+  /* Check that received value is not below the minimum. Don't check if minimum
+     is set to 0, since the param is a positive value and gcc will complain. */
+#if HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0
+  if (intro2_rate_per_sec < HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too small. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_rate_per_sec > HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too big. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+
+  /* Check that received value is not below the minimum */
+#if HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0
+  if (intro2_burst_per_sec < HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too small. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_burst_per_sec > HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too big. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* In a rate limiting scenario, burst can never be smaller than the rate. At
+   * best it can be equal. */
+  if (intro2_burst_per_sec < intro2_rate_per_sec) {
+    log_info(LD_REND, "Intro point DoS defenses burst is smaller than rate. "
+                      "Rate: %" PRIu64 " vs Burst: %" PRIu64,
+             intro2_rate_per_sec, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* Passing validation. */
+  ret = true;
+
+ end:
+  return ret;
+}
+
+/* Parse the cell DoS extension and apply defenses on the given circuit if
+ * validation passes. If the cell extension is malformed or contains unusable
+ * values, the DoS defenses is disabled on the circuit. */
+static void
+handle_establish_intro_cell_dos_extension(
+                                const trn_cell_extension_field_t *field,
+                                or_circuit_t *circ)
+{
+  ssize_t ret;
+  uint64_t intro2_rate_per_sec = 0, intro2_burst_per_sec = 0;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  tor_assert(field);
+  tor_assert(circ);
+
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  if (ret < 0) {
+    goto end;
+  }
+
+  for (size_t i = 0; i < trn_cell_extension_dos_get_n_params(dos); i++) {
+    const trn_cell_extension_dos_param_t *param =
+      trn_cell_extension_dos_getconst_params(dos, i);
+    if (BUG(param == NULL)) {
+      goto end;
+    }
+
+    switch (trn_cell_extension_dos_param_get_type(param)) {
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC:
+      intro2_rate_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC:
+      intro2_burst_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    default:
+      goto end;
+    }
+  }
+
+  /* A value of 0 is valid in the sense that we accept it but we still disable
+   * the defenses so return false. */
+  if (intro2_rate_per_sec == 0 || intro2_burst_per_sec == 0) {
+    log_info(LD_REND, "Intro point DoS defenses parameter set to 0. "
+                      "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    circ->introduce2_dos_defense_enabled = 0;
+    goto end;
+  }
+
+  /* If invalid, we disable the defense on the circuit. */
+  if (!cell_dos_extension_parameters_are_valid(intro2_rate_per_sec,
+                                               intro2_burst_per_sec)) {
+    circ->introduce2_dos_defense_enabled = 0;
+    log_info(LD_REND, "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    goto end;
+  }
+
+  /* We passed validation, enable defenses and apply rate/burst. */
+  circ->introduce2_dos_defense_enabled = 1;
+
+  /* Initialize the INTRODUCE2 token bucket for the rate limiting. */
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        (uint32_t) intro2_rate_per_sec,
+                        (uint32_t) intro2_burst_per_sec,
+                        (uint32_t) approx_time());
+  log_info(LD_REND, "Intro point DoS defenses enabled. Rate is %" PRIu64
+                    " and Burst is %" PRIu64,
+           intro2_rate_per_sec, intro2_burst_per_sec);
+
+ end:
+  trn_cell_extension_dos_free(dos);
+  return;
+}
+
+/* Parse every cell extension in the given ESTABLISH_INTRO cell. */
+static void
+handle_establish_intro_cell_extensions(
+                            const trn_cell_establish_intro_t *parsed_cell,
+                            or_circuit_t *circ)
+{
+  const trn_cell_extension_t *extensions;
+
+  tor_assert(parsed_cell);
+  tor_assert(circ);
+
+  extensions = trn_cell_establish_intro_getconst_extensions(parsed_cell);
+  if (extensions == NULL) {
+    goto end;
+  }
+
+  /* Go over all extensions. */
+  for (size_t idx = 0; idx < trn_cell_extension_get_num(extensions); idx++) {
+    const trn_cell_extension_field_t *field =
+      trn_cell_extension_getconst_fields(extensions, idx);
+    if (BUG(field == NULL)) {
+      /* The number of extensions should match the number of fields. */
+      break;
+    }
+
+    switch (trn_cell_extension_field_get_field_type(field)) {
+    case TRUNNEL_CELL_EXTENSION_TYPE_DOS:
+      /* After this, the circuit should be set for DoS defenses. */
+      handle_establish_intro_cell_dos_extension(field, circ);
+      break;
+    default:
+      /* Unknown extension. Skip over. */
+      break;
+    }
+  }
+
+ end:
+  return;
+}
+
 /** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
  *  well-formed and passed our verifications. Perform appropriate actions to
  *  establish an intro point. */
@@ -191,6 +373,13 @@ handle_verified_establish_intro_cell(or_circuit_t *circ,
   get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
                          parsed_cell);
 
+  /* Setup INTRODUCE2 defenses on the circuit. Must be done before parsing the
+   * cell extension that can possibly change the defenses' values. */
+  hs_dos_setup_default_intro2_defenses(circ);
+
+  /* Handle cell extension if any. */
+  handle_establish_intro_cell_extensions(parsed_cell, circ);
+
   /* Then notify the hidden service that the intro point is established by
      sending an INTRO_ESTABLISHED cell */
   if (hs_intro_send_intro_established_cell(circ)) {
@@ -392,7 +581,7 @@ validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
    * safety net here. The legacy ID must be zeroes in this case. */
   legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
   legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
-  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+  if (BUG(!fast_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
     goto invalid;
   }
 
@@ -480,6 +669,20 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
     }
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(service_circ)) {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(5 * 60);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v3 cell due to DoS "
+                            "limitations. Sending NACK to client.");
+      tor_free(msg);
+    }
+    status = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+    goto send_ack;
+  }
+
   /* Relay the cell to the service on its intro circuit with an INTRODUCE2
    * cell which is the same exact payload. */
   if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
@@ -518,7 +721,7 @@ introduce1_cell_is_legacy(const uint8_t *request)
 
   /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
    * indicates a legacy cell (v2). */
-  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+  if (!fast_mem_is_zero((const char *) request, DIGEST_LEN)) {
     /* Legacy cell. */
     return 1;
   }
@@ -546,6 +749,14 @@ circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
     return 0;
   }
 
+  /* Disallow single hop client circuit. */
+  if (circ->p_chan && channel_is_client(circ->p_chan)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Single hop client was rejected while trying to introduce. "
+           "Closing circuit.");
+    return 0;
+  }
+
   return 1;
 }
 
@@ -602,8 +813,8 @@ hs_intropoint_clear(hs_intropoint_t *ip)
     return;
   }
   tor_cert_free(ip->auth_key_cert);
-  SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls,
-                    hs_desc_link_specifier_free(ls));
+  SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *, ls,
+                    link_specifier_free(ls));
   smartlist_free(ip->link_specifiers);
   memset(ip, 0, sizeof(hs_intropoint_t));
 }
diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h
index e82575f052..94ebf021e4 100644
--- a/src/feature/hs/hs_intropoint.h
+++ b/src/feature/hs/hs_intropoint.h
@@ -57,6 +57,9 @@ STATIC int handle_introduce1(or_circuit_t *client_circ,
                              const uint8_t *request, size_t request_len);
 STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
 STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+STATIC bool cell_dos_extension_parameters_are_valid(
+                                          uint64_t intro2_rate_per_sec,
+                                          uint64_t intro2_burst_per_sec);
 
 #endif /* defined(HS_INTROPOINT_PRIVATE) */
 
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
index 6d32cae86c..17ac4fa4a9 100644
--- a/src/feature/hs/hs_service.c
+++ b/src/feature/hs/hs_service.c
@@ -242,6 +242,9 @@ set_service_default_config(hs_service_config_t *c,
   c->is_single_onion = 0;
   c->dir_group_readable = 0;
   c->is_ephemeral = 0;
+  c->has_dos_defense_enabled = HS_CONFIG_V3_DOS_DEFENSE_DEFAULT;
+  c->intro_dos_rate_per_sec = HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT;
+  c->intro_dos_burst_per_sec = HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT;
 }
 
 /* From a service configuration object config, clear everything from it
@@ -280,9 +283,10 @@ describe_intro_point(const hs_service_intro_point_t *ip)
   const char *legacy_id = NULL;
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, lspec) {
-    if (lspec->type == LS_LEGACY_ID) {
-      legacy_id = (const char *) lspec->u.legacy_id;
+                          const link_specifier_t *, lspec) {
+    if (link_specifier_get_ls_type(lspec) == LS_LEGACY_ID) {
+      legacy_id = (const char *)
+        link_specifier_getconstarray_un_legacy_id(lspec);
       break;
     }
   } SMARTLIST_FOREACH_END(lspec);
@@ -426,23 +430,16 @@ service_intro_point_free_void(void *obj)
 }
 
 /* Return a newly allocated service intro point and fully initialized from the
- * given extend_info_t ei if non NULL.
- * If is_legacy is true, we also generate the legacy key.
- * If supports_ed25519_link_handshake_any is true, we add the relay's ed25519
- * key to the link specifiers.
+ * given node_t node, if non NULL.
  *
- * If ei is NULL, returns a hs_service_intro_point_t with an empty link
+ * If node is NULL, returns a hs_service_intro_point_t with an empty link
  * specifier list and no onion key. (This is used for testing.)
  * On any other error, NULL is returned.
  *
- * ei must be an extend_info_t containing an IPv4 address. (We will add supoort
- * for IPv6 in a later release.) When calling extend_info_from_node(), pass
- * 0 in for_direct_connection to make sure ei always has an IPv4 address. */
+ * node must be an node_t with an IPv4 address. */
 STATIC hs_service_intro_point_t *
-service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
-                        unsigned int supports_ed25519_link_handshake_any)
+service_intro_point_new(const node_t *node)
 {
-  hs_desc_link_specifier_t *ls;
   hs_service_intro_point_t *ip;
 
   ip = tor_malloc_zero(sizeof(*ip));
@@ -472,12 +469,17 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
   ip->replay_cache = replaycache_new(0, 0);
 
   /* Initialize the base object. We don't need the certificate object. */
-  ip->base.link_specifiers = smartlist_new();
+  ip->base.link_specifiers = node_get_link_specifier_smartlist(node, 0);
+
+  if (node == NULL) {
+    goto done;
+  }
 
   /* Generate the encryption key for this intro point. */
   curve25519_keypair_generate(&ip->enc_key_kp, 0);
-  /* Figure out if this chosen node supports v3 or is legacy only. */
-  if (is_legacy) {
+  /* Figure out if this chosen node supports v3 or is legacy only.
+   * NULL nodes are used in the unit tests. */
+  if (!node_supports_ed25519_hs_intro(node)) {
     ip->base.is_only_legacy = 1;
     /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */
     ip->legacy_key = crypto_pk_new();
@@ -490,40 +492,13 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
     }
   }
 
-  if (ei == NULL) {
-    goto done;
-  }
+  /* Flag if this intro point supports the INTRO2 dos defenses. */
+  ip->support_intro2_dos_defense =
+    node_supports_establish_intro_dos_extension(node);
 
-  /* We'll try to add all link specifiers. Legacy is mandatory.
-   * IPv4 or IPv6 is required, and we always send IPv4. */
-  ls = hs_desc_link_specifier_new(ei, LS_IPV4);
-  /* It is impossible to have an extend info object without a v4. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID);
-  /* It is impossible to have an extend info object without an identity
-   * digest. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  /* ed25519 identity key is optional for intro points. If the node supports
-   * ed25519 link authentication, we include it. */
-  if (supports_ed25519_link_handshake_any) {
-    ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID);
-    if (ls) {
-      smartlist_add(ip->base.link_specifiers, ls);
-    }
-  }
-
-  /* IPv6 is not supported in this release. */
-
-  /* Finally, copy onion key from the extend_info_t object. */
-  memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key));
+  /* Finally, copy onion key from the node. */
+  memcpy(&ip->onion_key, node_get_curve25519_onion_key(node),
+         sizeof(ip->onion_key));
 
  done:
   return ip;
@@ -656,16 +631,16 @@ get_objects_from_ident(const hs_ident_circuit_t *ident,
  * encountered in the link specifier list. Return NULL if it can't be found.
  *
  * The caller does NOT have ownership of the object, the intro point does. */
-static hs_desc_link_specifier_t *
+static link_specifier_t *
 get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
 {
-  hs_desc_link_specifier_t *lnk_spec = NULL;
+  link_specifier_t *lnk_spec = NULL;
 
   tor_assert(ip);
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          hs_desc_link_specifier_t *, ls) {
-    if (ls->type == type) {
+                          link_specifier_t *, ls) {
+    if (link_specifier_get_ls_type(ls) == type) {
       lnk_spec = ls;
       goto end;
     }
@@ -681,7 +656,7 @@ get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
 STATIC const node_t *
 get_node_from_intro_point(const hs_service_intro_point_t *ip)
 {
-  const hs_desc_link_specifier_t *ls;
+  const link_specifier_t *ls;
 
   tor_assert(ip);
 
@@ -690,7 +665,8 @@ get_node_from_intro_point(const hs_service_intro_point_t *ip)
     return NULL;
   }
   /* XXX In the future, we want to only use the ed25519 ID (#22173). */
-  return node_get_by_id((const char *) ls->u.legacy_id);
+  return node_get_by_id(
+    (const char *) link_specifier_getconstarray_un_legacy_id(ls));
 }
 
 /* Given a service intro point, return the extend_info_t for it. This can
@@ -1179,7 +1155,8 @@ parse_authorized_client(const char *client_key_str)
   client = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
   if (base32_decode((char *) client->client_pk.public_key,
                     sizeof(client->client_pk.public_key),
-                    pubkey_b32, strlen(pubkey_b32)) < 0) {
+                    pubkey_b32, strlen(pubkey_b32)) !=
+      sizeof(client->client_pk.public_key)) {
     log_warn(LD_REND, "Client authorization public key cannot be decoded: %s",
              pubkey_b32);
     goto err;
@@ -1261,7 +1238,7 @@ load_client_keys(hs_service_t *service)
     client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
 
     /* If we cannot read the file, continue with the next file. */
-    if (!client_key_str)  {
+    if (!client_key_str) {
       log_warn(LD_REND, "Client authorization file %s can't be read. "
                         "Corrupted or verify permission? Ignoring.",
                client_key_file_path);
@@ -1556,7 +1533,7 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
                              hs_service_descriptor_t *desc, time_t now)
 {
   time_t *time_of_failure, *prev_ptr;
-  const hs_desc_link_specifier_t *legacy_ls;
+  const link_specifier_t *legacy_ls;
 
   tor_assert(ip);
   tor_assert(desc);
@@ -1565,22 +1542,13 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
   *time_of_failure = now;
   legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
   tor_assert(legacy_ls);
-  prev_ptr = digestmap_set(desc->intro_points.failed_id,
-                           (const char *) legacy_ls->u.legacy_id,
-                           time_of_failure);
+  prev_ptr = digestmap_set(
+    desc->intro_points.failed_id,
+    (const char *) link_specifier_getconstarray_un_legacy_id(legacy_ls),
+    time_of_failure);
   tor_free(prev_ptr);
 }
 
-/* Copy the descriptor link specifier object from src to dst. */
-static void
-link_specifier_copy(hs_desc_link_specifier_t *dst,
-                    const hs_desc_link_specifier_t *src)
-{
-  tor_assert(dst);
-  tor_assert(src);
-  memcpy(dst, src, sizeof(hs_desc_link_specifier_t));
-}
-
 /* Using a given descriptor signing keypair signing_kp, a service intro point
  * object ip and the time now, setup the content of an already allocated
  * descriptor intro desc_ip.
@@ -1615,9 +1583,14 @@ setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
 
   /* Copy link specifier(s). */
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, ls) {
-    hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy));
-    link_specifier_copy(copy, ls);
+                          const link_specifier_t *, ls) {
+    if (BUG(!ls)) {
+      goto done;
+    }
+    link_specifier_t *copy = link_specifier_dup(ls);
+    if (BUG(!copy)) {
+      goto done;
+    }
     smartlist_add(desc_ip->link_specifiers, copy);
   } SMARTLIST_FOREACH_END(ls);
 
@@ -1789,7 +1762,7 @@ build_service_desc_superencrypted(const hs_service_t *service,
          sizeof(curve25519_public_key_t));
 
   /* Test that subcred is not zero because we might use it below */
-  if (BUG(tor_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) {
+  if (BUG(fast_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) {
     return -1;
   }
 
@@ -1855,9 +1828,9 @@ build_service_desc_plaintext(const hs_service_t *service,
 
   tor_assert(service);
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->blinded_kp,
                               sizeof(desc->blinded_kp)));
-  tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->signing_kp,
                               sizeof(desc->signing_kp)));
 
   /* Set the subcredential. */
@@ -1907,7 +1880,7 @@ build_service_desc_keys(const hs_service_t *service,
   ed25519_keypair_t kp;
 
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk,
+  tor_assert(!fast_mem_is_zero((char *) &service->keys.identity_pk,
              ED25519_PUBKEY_LEN));
 
   /* XXX: Support offline key feature (#18098). */
@@ -2116,7 +2089,6 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
 {
   const or_options_t *options = get_options();
   const node_t *node;
-  extend_info_t *info = NULL;
   hs_service_intro_point_t *ip = NULL;
   /* Normal 3-hop introduction point flags. */
   router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC;
@@ -2145,43 +2117,17 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
    * we don't want to use that node anymore. */
   smartlist_add(exclude_nodes, (void *) node);
 
-  /* We do this to ease our life but also this call makes appropriate checks
-   * of the node object such as validating ntor support for instance.
-   *
-   * We must provide an extend_info for clients to connect over a 3-hop path,
-   * so we don't pass direct_conn here. */
-  info = extend_info_from_node(node, 0);
-  if (BUG(info == NULL)) {
-    goto err;
-  }
-
-  /* Let's do a basic sanity check here so that we don't end up advertising the
-   * ed25519 identity key of relays that don't actually support the link
-   * protocol */
-  if (!node_supports_ed25519_link_authentication(node, 0)) {
-    tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity));
-  } else {
-    /* Make sure we *do* have an ed key if we support the link authentication.
-     * Sending an empty key would result in a failure to extend. */
-    tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity));
-  }
+  /* Create our objects and populate them with the node information. */
+  ip = service_intro_point_new(node);
 
-  /* Create our objects and populate them with the node information.
-   * We don't care if the intro's link auth is compatible with us, because
-   * we are sending the ed25519 key to a remote client via the descriptor. */
-  ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node),
-                               node_supports_ed25519_link_authentication(node,
-                                                                         0));
   if (ip == NULL) {
     goto err;
   }
 
-  log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info));
-  extend_info_free(info);
+  log_info(LD_REND, "Picked intro point: %s", node_describe(node));
   return ip;
  err:
   service_intro_point_free(ip);
-  extend_info_free(info);
   return NULL;
 }
 
@@ -2387,15 +2333,70 @@ intro_point_should_expire(const hs_service_intro_point_t *ip,
   return 1;
 }
 
-/* Go over the given set of intro points for each service and remove any
- * invalid ones. The conditions for removal are:
+/* Return true iff we should remove the intro point ip from its service.
  *
- *    - The node doesn't exists anymore (not in consensus)
- *                          OR
- *    - The intro point maximum circuit retry count has been reached and no
- *      circuit can be found associated with it.
- *                          OR
- *    - The intro point has expired and we should pick a new one.
+ * We remove an intro point from the service descriptor list if one of
+ * these criteria is met:
+ *    - It has expired (either in INTRO2 count or in time).
+ *    - No node was found (fell off the consensus).
+ *    - We are over the maximum amount of retries.
+ *
+ * If an established or pending circuit is found for the given ip object, this
+ * return false indicating it should not be removed. */
+static bool
+should_remove_intro_point(hs_service_intro_point_t *ip, time_t now)
+{
+  bool ret = false;
+
+  tor_assert(ip);
+
+  /* Any one of the following needs to be True to furfill the criteria to
+   * remove an intro point. */
+  bool has_no_retries = (ip->circuit_retries >
+                         MAX_INTRO_POINT_CIRCUIT_RETRIES);
+  bool has_no_node = (get_node_from_intro_point(ip) == NULL);
+  bool has_expired = intro_point_should_expire(ip, now);
+
+  /* If the node fell off the consensus or the IP has expired, we have to
+   * remove it now. */
+  if (has_no_node || has_expired) {
+    ret = true;
+    goto end;
+  }
+
+  /* Pass this point, even though we might be over the retry limit, we check
+   * if a circuit (established or pending) exists. In that case, we should not
+   * remove it because it might simply be valid and opened at the previous
+   * scheduled event for the last retry. */
+
+  /* Did we established already? */
+  if (ip->circuit_established) {
+    goto end;
+  }
+  /* Do we simply have an existing circuit regardless of its state? */
+  if (hs_circ_service_get_intro_circ(ip)) {
+    goto end;
+  }
+
+  /* Getting here means we have _no_ circuits so then return if we have any
+   * remaining retries. */
+  ret = has_no_retries;
+
+ end:
+  /* Meaningful log in case we are about to remove the IP. */
+  if (ret) {
+    log_info(LD_REND, "Intro point %s%s (retried: %u times). "
+                      "Removing it.",
+             describe_intro_point(ip),
+             has_expired ? " has expired" :
+               (has_no_node) ?  " fell off the consensus" : "",
+             ip->circuit_retries);
+  }
+  return ret;
+}
+
+/* Go over the given set of intro points for each service and remove any
+ * invalid ones.
  *
  * If an intro point is removed, the circuit (if any) is immediately close.
  * If a circuit can't be found, the intro point is kept if it hasn't reached
@@ -2420,21 +2421,7 @@ cleanup_intro_points(hs_service_t *service, time_t now)
      * valid and remove any of them that aren't. */
     DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
                                 hs_service_intro_point_t *, ip) {
-      const node_t *node = get_node_from_intro_point(ip);
-      int has_expired = intro_point_should_expire(ip, now);
-
-      /* We cleanup an intro point if it has expired or if we do not know the
-       * node_t anymore (removed from our latest consensus) or if we've
-       * reached the maximum number of retry with a non existing circuit. */
-      if (has_expired || node == NULL ||
-          ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
-        log_info(LD_REND, "Intro point %s%s (retried: %u times). "
-                          "Removing it.",
-                 describe_intro_point(ip),
-                 has_expired ? " has expired" :
-                    (node == NULL) ?  " fell off the consensus" : "",
-                 ip->circuit_retries);
-
+      if (should_remove_intro_point(ip, now)) {
         /* We've retried too many times, remember it as a failed intro point
          * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */
         if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -2956,8 +2943,8 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now,
   /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */
   tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR);
 
-  log_info(LD_REND, "Encrypted revision counter %d to %ld",
-           (int) seconds_since_start_of_srv, (long int) rev_counter);
+  log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64,
+           (int) seconds_since_start_of_srv, rev_counter);
 
   hs_desc->desc->plaintext_data.revision_counter = rev_counter;
 }
@@ -3700,8 +3687,8 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk)
 }
 
 /* Return the number of service we have configured and usable. */
-unsigned int
-hs_service_get_num_services(void)
+MOCK_IMPL(unsigned int,
+hs_service_get_num_services,(void))
 {
   if (hs_service_map == NULL) {
     return 0;
diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h
index 5f43233ea1..c4bbb293bb 100644
--- a/src/feature/hs/hs_service.h
+++ b/src/feature/hs/hs_service.h
@@ -76,6 +76,10 @@ typedef struct hs_service_intro_point_t {
    * circuit associated with this intro point has received. This is used to
    * prevent replay attacks. */
   replaycache_t *replay_cache;
+
+  /* Support the INTRO2 DoS defense. If set, the DoS extension described by
+   * proposal 305 is sent. */
+  unsigned int support_intro2_dos_defense : 1;
 } hs_service_intro_point_t;
 
 /* Object handling introduction points of a service. */
@@ -241,6 +245,11 @@ typedef struct hs_service_config_t {
 
   /* Does this service export the circuit ID of its clients? */
   hs_circuit_id_protocol_t circuit_id_protocol;
+
+  /* DoS defenses. For the ESTABLISH_INTRO cell extension. */
+  unsigned int has_dos_defense_enabled : 1;
+  uint32_t intro_dos_rate_per_sec;
+  uint32_t intro_dos_burst_per_sec;
 } hs_service_config_t;
 
 /* Service state. */
@@ -310,7 +319,7 @@ hs_service_t *hs_service_new(const or_options_t *options);
 void hs_service_free_(hs_service_t *service);
 #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s))
 
-unsigned int hs_service_get_num_services(void);
+MOCK_DECL(unsigned int, hs_service_get_num_services,(void));
 void hs_service_stage_services(const smartlist_t *service_list);
 int hs_service_load_all_keys(void);
 int hs_service_get_version_from_key(const hs_service_t *service);
@@ -361,7 +370,7 @@ STATIC hs_service_t *get_first_service(void);
 STATIC hs_service_intro_point_t *service_intro_point_find_by_ident(
                                          const hs_service_t *service,
                                          const hs_ident_circuit_t *ident);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /* Service accessors. */
 STATIC hs_service_t *find_service(hs_service_ht *map,
@@ -369,10 +378,7 @@ STATIC hs_service_t *find_service(hs_service_ht *map,
 STATIC void remove_service(hs_service_ht *map, hs_service_t *service);
 STATIC int register_service(hs_service_ht *map, hs_service_t *service);
 /* Service introduction point functions. */
-STATIC hs_service_intro_point_t *service_intro_point_new(
-                            const extend_info_t *ei,
-                            unsigned int is_legacy,
-                            unsigned int supports_ed25519_link_handshake_any);
+STATIC hs_service_intro_point_t *service_intro_point_new(const node_t *node);
 STATIC void service_intro_point_free_(hs_service_intro_point_t *ip);
 #define service_intro_point_free(ip)                            \
   FREE_AND_NULL(hs_service_intro_point_t,             \
diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h
index d89440faca..6700eca15b 100644
--- a/src/feature/hs/hs_stats.h
+++ b/src/feature/hs/hs_stats.h
@@ -6,9 +6,13 @@
  * \brief Header file for hs_stats.c
  **/
 
+#ifndef TOR_HS_STATS_H
+#define TOR_HS_STATS_H
+
 void hs_stats_note_introduce2_cell(int is_hsv3);
 uint32_t hs_stats_get_n_introduce2_v3_cells(void);
 uint32_t hs_stats_get_n_introduce2_v2_cells(void);
 void hs_stats_note_service_rendezvous_launch(void);
 uint32_t hs_stats_get_n_rendezvous_launches(void);
 
+#endif /* !defined(TOR_HS_STATS_H) */
diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h
index 7d4116d8bb..6c86c02f47 100644
--- a/src/feature/hs/hsdir_index_st.h
+++ b/src/feature/hs/hsdir_index_st.h
@@ -20,5 +20,5 @@ struct hsdir_index_t {
   uint8_t store_second[DIGEST256_LEN];
 };
 
-#endif
+#endif /* !defined(HSDIR_INDEX_ST_H) */
 
diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h
index 95fe2c65ab..c90c52cfea 100644
--- a/src/feature/hs_common/shared_random_client.h
+++ b/src/feature/hs_common/shared_random_client.h
@@ -44,5 +44,5 @@ time_t get_start_time_of_current_round(void);
 
 #endif /* TOR_UNIT_TESTS */
 
-#endif /* TOR_SHARED_RANDOM_CLIENT_H */
+#endif /* !defined(TOR_SHARED_RANDOM_CLIENT_H) */
 
diff --git a/src/feature/keymgt/loadkey.h b/src/feature/keymgt/loadkey.h
index 8beee57a20..0a5af0b804 100644
--- a/src/feature/keymgt/loadkey.h
+++ b/src/feature/keymgt/loadkey.h
@@ -52,4 +52,4 @@ int read_encrypted_secret_key(ed25519_secret_key_t *out,
 int write_encrypted_secret_key(const ed25519_secret_key_t *out,
                                const char *fname);
 
-#endif
+#endif /* !defined(TOR_LOADKEY_H) */
diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c
index 7a065662a7..9fc3b62525 100644
--- a/src/feature/nodelist/authcert.c
+++ b/src/feature/nodelist/authcert.c
@@ -380,7 +380,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
   int added_trusted_cert = 0;
 
   for (s = contents; *s; s = eos) {
-    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s),
+                                                              &eos);
     cert_list_t *cl;
     if (!cert) {
       failure_code = -1;
diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h
index 2effdb06e6..071293f9ee 100644
--- a/src/feature/nodelist/authcert.h
+++ b/src/feature/nodelist/authcert.h
@@ -57,4 +57,4 @@ MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
 
 void authcert_free_all(void);
 
-#endif
+#endif /* !defined(TOR_AUTHCERT_H) */
diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h
index 68a84bc452..bf9b690c24 100644
--- a/src/feature/nodelist/authority_cert_st.h
+++ b/src/feature/nodelist/authority_cert_st.h
@@ -28,5 +28,5 @@ struct authority_cert_t {
   uint16_t dir_port;
 };
 
-#endif
+#endif /* !defined(AUTHORITY_CERT_ST_H) */
 
diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h
index b04a1abc7d..4d1378cdfa 100644
--- a/src/feature/nodelist/desc_store_st.h
+++ b/src/feature/nodelist/desc_store_st.h
@@ -36,4 +36,4 @@ struct desc_store_t {
   size_t bytes_dropped;
 };
 
-#endif
+#endif /* !defined(DESC_STORE_ST_H) */
diff --git a/src/feature/nodelist/describe.c b/src/feature/nodelist/describe.c
index 5c376408c0..1e46837685 100644
--- a/src/feature/nodelist/describe.c
+++ b/src/feature/nodelist/describe.c
@@ -9,66 +9,108 @@
  * \brief Format short descriptions of relays.
  */
 
+#define DESCRIBE_PRIVATE
+
 #include "core/or/or.h"
 #include "feature/nodelist/describe.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "core/or/extend_info_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
-
-/**
- * Longest allowed output of format_node_description, plus 1 character for
- * NUL.  This allows space for:
- * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
- * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
- * plus a terminating NUL.
- */
-#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+#include "feature/nodelist/microdesc_st.h"
 
 /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
  * hold a human-readable description of a node with identity digest
- * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
- * and address <b>addr</b> or <b>addr32h</b>.
+ * <b>id_digest</b>, nickname <b>nickname</b>, and addresses <b>addr32h</b> and
+ * <b>addr</b>.
  *
  * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
- * NULL.  The <b>addr32h</b> field is optional and may be set to 0.
+ * NULL or the null address.  The <b>addr32h</b> field is optional and may be
+ * set to 0.
  *
  * Return a pointer to the front of <b>buf</b>.
+ * If buf is NULL, return a string constant describing the error.
  */
-static const char *
+STATIC const char *
 format_node_description(char *buf,
                         const char *id_digest,
-                        int is_named,
                         const char *nickname,
                         const tor_addr_t *addr,
                         uint32_t addr32h)
 {
-  char *cp;
+  size_t rv = 0;
+  bool has_addr = addr && !tor_addr_is_null(addr);
 
   if (!buf)
     return "<NULL BUFFER>";
 
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
-  cp = buf+1+HEX_DIGEST_LEN;
+  memset(buf, 0, NODE_DESC_BUF_LEN);
+
+  if (!id_digest) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<NULL ID DIGEST>", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    return buf;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", NODE_DESC_BUF_LEN);
+  tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  id_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   if (nickname) {
-    buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
-    strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
-    cp += strlen(cp);
+    rv = strlcat(buf, "~", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    rv = strlcat(buf, nickname, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
-  if (addr32h || addr) {
-    memcpy(cp, " at ", 4);
-    cp += 4;
-    if (addr) {
-      tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
-    } else {
-      struct in_addr in;
-      in.s_addr = htonl(addr32h);
-      tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
-    }
+  if (addr32h || has_addr) {
+    rv = strlcat(buf, " at ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
+  if (addr32h) {
+    int ntoa_rv = 0;
+    char ipv4_addr_str[INET_NTOA_BUF_LEN];
+    memset(ipv4_addr_str, 0, sizeof(ipv4_addr_str));
+    struct in_addr in;
+    memset(&in, 0, sizeof(in));
+
+    in.s_addr = htonl(addr32h);
+    ntoa_rv = tor_inet_ntoa(&in, ipv4_addr_str, sizeof(ipv4_addr_str));
+    tor_assert_nonfatal(ntoa_rv >= 0);
+
+    rv = strlcat(buf, ipv4_addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  /* Both addresses are valid */
+  if (addr32h && has_addr) {
+    rv = strlcat(buf, " and ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  if (has_addr) {
+    const char *str_rv = NULL;
+    char addr_str[TOR_ADDR_BUF_LEN];
+    memset(addr_str, 0, sizeof(addr_str));
+
+    str_rv = tor_addr_to_str(addr_str, addr, sizeof(addr_str), 1);
+    tor_assert_nonfatal(str_rv == addr_str);
+
+    rv = strlcat(buf, addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   return buf;
 }
 
@@ -84,11 +126,11 @@ router_describe(const routerinfo_t *ri)
 
   if (!ri)
     return "<null>";
+
   return format_node_description(buf,
                                  ri->cache_info.identity_digest,
-                                 0,
                                  ri->nickname,
-                                 NULL,
+                                 &ri->ipv6_addr,
                                  ri->addr);
 }
 
@@ -103,25 +145,33 @@ node_describe(const node_t *node)
   static char buf[NODE_DESC_BUF_LEN];
   const char *nickname = NULL;
   uint32_t addr32h = 0;
-  int is_named = 0;
+  const tor_addr_t *ipv6_addr = NULL;
 
   if (!node)
     return "<null>";
 
   if (node->rs) {
     nickname = node->rs->nickname;
-    is_named = node->rs->is_named;
     addr32h = node->rs->addr;
+    ipv6_addr = &node->rs->ipv6_addr;
+    /* Support consensus versions less than 28, when IPv6 addresses were in
+     * microdescs. This code can be removed when 0.2.9 is no longer supported,
+     * and the MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC macro is removed. */
+    if (node->md && tor_addr_is_null(ipv6_addr)) {
+      ipv6_addr = &node->md->ipv6_addr;
+    }
   } else if (node->ri) {
     nickname = node->ri->nickname;
     addr32h = node->ri->addr;
+    ipv6_addr = &node->ri->ipv6_addr;
+  } else {
+    return "<null rs and ri>";
   }
 
   return format_node_description(buf,
                                  node->identity,
-                                 is_named,
                                  nickname,
-                                 NULL,
+                                 ipv6_addr,
                                  addr32h);
 }
 
@@ -137,11 +187,11 @@ routerstatus_describe(const routerstatus_t *rs)
 
   if (!rs)
     return "<null>";
+
   return format_node_description(buf,
                                  rs->identity_digest,
-                                 rs->is_named,
                                  rs->nickname,
-                                 NULL,
+                                 &rs->ipv6_addr,
                                  rs->addr);
 }
 
@@ -157,9 +207,9 @@ extend_info_describe(const extend_info_t *ei)
 
   if (!ei)
     return "<null>";
+
   return format_node_description(buf,
                                  ei->identity_digest,
-                                 0,
                                  ei->nickname,
                                  &ei->addr,
                                  0);
@@ -175,9 +225,39 @@ extend_info_describe(const extend_info_t *ei)
 void
 router_get_verbose_nickname(char *buf, const routerinfo_t *router)
 {
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
-                DIGEST_LEN);
-  buf[1+HEX_DIGEST_LEN] = '~';
-  strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
+  size_t rv = 0;
+
+  if (!buf)
+    return;
+
+  memset(buf, 0, MAX_VERBOSE_NICKNAME_LEN+1);
+
+  if (!router) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<null>", MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+    return;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  router->cache_info.identity_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+  }
+
+  rv = strlcat(buf, "~", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  rv = strlcat(buf, router->nickname, MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
 }
diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h
index 018af6470e..6c0d6dc48d 100644
--- a/src/feature/nodelist/describe.h
+++ b/src/feature/nodelist/describe.h
@@ -22,4 +22,36 @@ const char *node_describe(const struct node_t *node);
 const char *router_describe(const struct routerinfo_t *ri);
 const char *routerstatus_describe(const struct routerstatus_t *ri);
 
-#endif
+void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
+
+#if defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS)
+
+/**
+ * Longest allowed output for an IPv4 address "255.255.255.255", with NO
+ * terminating NUL.
+ */
+#define IPV4_BUF_LEN_NO_NUL 15
+
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL.  This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " 255.255.255.255 and [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN \
+  (MAX_VERBOSE_NICKNAME_LEN+4+IPV4_BUF_LEN_NO_NUL+5+TOR_ADDR_BUF_LEN)
+
+#endif /* defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS) */
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC const char *format_node_description(char *buf,
+                                           const char *id_digest,
+                                           const char *nickname,
+                                           const tor_addr_t *addr,
+                                           uint32_t addr32h);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_DESCRIBE_H) */
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
index 93baa6e4e0..ccbb378513 100644
--- a/src/feature/nodelist/dirlist.c
+++ b/src/feature/nodelist/dirlist.c
@@ -28,7 +28,7 @@
 
 #include "app/config/config.h"
 #include "core/or/policies.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/dirlist.h"
@@ -49,6 +49,37 @@ static smartlist_t *trusted_dir_servers = NULL;
  * and all fallback directory servers. */
 static smartlist_t *fallback_dir_servers = NULL;
 
+/** Helper: From a given trusted directory entry, add the v4 or/and v6 address
+ * to the nodelist address set. */
+static void
+add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir)
+{
+  tor_assert(dir);
+  tor_assert(dir->is_authority);
+
+  /* Add IPv4 and then IPv6 if applicable. */
+  nodelist_add_addr4_to_address_set(dir->addr);
+  if (!tor_addr_is_null(&dir->ipv6_addr)) {
+    nodelist_add_addr6_to_address_set(&dir->ipv6_addr);
+  }
+}
+
+/** Go over the trusted directory server list and add their address(es) to the
+ * nodelist address set. This is called everytime a new consensus is set. */
+MOCK_IMPL(void,
+dirlist_add_trusted_dir_addresses, (void))
+{
+  if (!trusted_dir_servers) {
+    return;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(trusted_dir_servers, const dir_server_t *, ent) {
+    if (ent->is_authority) {
+      add_trusted_dir_to_nodelist_addr_set(ent);
+    }
+  } SMARTLIST_FOREACH_END(ent);
+}
+
 /** Return the number of directory authorities whose type matches some bit set
  * in <b>type</b>  */
 int
diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h
index 9fabd0a44a..c49162f1e9 100644
--- a/src/feature/nodelist/dirlist.h
+++ b/src/feature/nodelist/dirlist.h
@@ -44,4 +44,6 @@ void dir_server_add(dir_server_t *ent);
 void clear_dir_servers(void);
 void dirlist_free_all(void);
 
-#endif
+MOCK_DECL(void, dirlist_add_trusted_dir_addresses, (void));
+
+#endif /* !defined(TOR_DIRLIST_H) */
diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h
index 66e32c422f..ac2a803252 100644
--- a/src/feature/nodelist/document_signature_st.h
+++ b/src/feature/nodelist/document_signature_st.h
@@ -25,5 +25,5 @@ struct document_signature_t {
                                      * as good. */
 };
 
-#endif
+#endif /* !defined(DOCUMENT_SIGNATURE_ST_H) */
 
diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h
index c54277b05e..22c708f018 100644
--- a/src/feature/nodelist/extrainfo_st.h
+++ b/src/feature/nodelist/extrainfo_st.h
@@ -26,5 +26,5 @@ struct extrainfo_t {
   size_t pending_sig_len;
 };
 
-#endif
+#endif /* !defined(EXTRAINFO_ST_H) */
 
diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c
index 75cab7a0af..fea7cf4c65 100644
--- a/src/feature/nodelist/fmt_routerstatus.c
+++ b/src/feature/nodelist/fmt_routerstatus.c
@@ -14,55 +14,14 @@
 #include "core/or/or.h"
 #include "feature/nodelist/fmt_routerstatus.h"
 
-/* #include "lib/container/buffers.h" */
-/* #include "app/config/config.h" */
-/* #include "app/config/confparse.h" */
-/* #include "core/or/channel.h" */
-/* #include "core/or/channeltls.h" */
-/* #include "core/or/command.h" */
-/* #include "core/mainloop/connection.h" */
-/* #include "core/or/connection_or.h" */
-/* #include "feature/dircache/conscache.h" */
-/* #include "feature/dircache/consdiffmgr.h" */
-/* #include "feature/control/control.h" */
-/* #include "feature/dircache/directory.h" */
-/* #include "feature/dircache/dirserv.h" */
-/* #include "feature/hibernate/hibernate.h" */
-/* #include "feature/dirauth/keypin.h" */
-/* #include "core/mainloop/mainloop.h" */
-/* #include "feature/nodelist/microdesc.h" */
-/* #include "feature/nodelist/networkstatus.h" */
-/* #include "feature/nodelist/nodelist.h" */
 #include "core/or/policies.h"
-/* #include "core/or/protover.h" */
-/* #include "feature/stats/rephist.h" */
-/* #include "feature/relay/router.h" */
-/* #include "feature/nodelist/dirlist.h" */
 #include "feature/nodelist/routerlist.h"
-
-/* #include "feature/nodelist/routerparse.h" */
-/* #include "feature/nodelist/routerset.h" */
-/* #include "feature/nodelist/torcert.h" */
-/* #include "feature/dircommon/voting_schedule.h" */
-
 #include "feature/dirauth/dirvote.h"
 
-/* #include "feature/dircache/cached_dir_st.h" */
-/* #include "feature/dircommon/dir_connection_st.h" */
-/* #include "feature/nodelist/extrainfo_st.h" */
-/* #include "feature/nodelist/microdesc_st.h" */
-/* #include "feature/nodelist/node_st.h" */
 #include "feature/nodelist/routerinfo_st.h"
-/* #include "feature/nodelist/routerlist_st.h" */
-/* #include "core/or/tor_version_st.h" */
 #include "feature/nodelist/vote_routerstatus_st.h"
 
-/* #include "lib/compress/compress.h" */
-/* #include "lib/container/order.h" */
 #include "lib/crypt_ops/crypto_format.h"
-/* #include "lib/encoding/confline.h" */
-
-/* #include "lib/encoding/keyval.h" */
 
 /** Helper: write the router-status information in <b>rs</b> into a newly
  * allocated character buffer.  Use the same format as in network-status
@@ -135,7 +94,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     goto done;
 
   smartlist_add_asprintf(chunks,
-                   "s%s%s%s%s%s%s%s%s%s%s\n",
+                   "s%s%s%s%s%s%s%s%s%s%s%s\n",
                   /* These must stay in alphabetical order. */
                    rs->is_authority?" Authority":"",
                    rs->is_bad_exit?" BadExit":"",
@@ -145,6 +104,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                    rs->is_hs_dir?" HSDir":"",
                    rs->is_flagged_running?" Running":"",
                    rs->is_stable?" Stable":"",
+                   rs->is_staledesc?" StaleDesc":"",
                    rs->is_v2_dir?" V2Dir":"",
                    rs->is_valid?" Valid":"");
 
@@ -232,7 +192,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     }
 
     if (format == NS_V3_VOTE && vrs) {
-      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
+      if (fast_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
         smartlist_add_strdup(chunks, "id ed25519 none\n");
       } else {
         char ed_b64[BASE64_DIGEST256_LEN+1];
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
index dafaabb5e5..89ac0a2f83 100644
--- a/src/feature/nodelist/microdesc.c
+++ b/src/feature/nodelist/microdesc.c
@@ -23,6 +23,7 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -69,6 +70,8 @@ struct microdesc_cache_t {
 };
 
 static microdesc_cache_t *get_microdesc_cache_noload(void);
+static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                              const char *activity);
 
 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
 static inline unsigned int
@@ -110,8 +113,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest)
 
   /* If we have a reasonably live consensus, then most of our dirservers should
    * still be caching all the microdescriptors in it. Reasonably live
-   * consensuses are up to a day old. But microdescriptors expire 7 days after
-   * the last consensus that referenced them. */
+   * consensuses are up to a day old (or a day in the future). But
+   * microdescriptors expire 7 days after the last consensus that referenced
+   * them. */
   if (!networkstatus_get_reasonably_live_consensus(approx_time(),
                                                    FLAV_MICRODESC)) {
     return;
@@ -221,6 +225,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
   }
 
   md->off = tor_fd_getpos(fd);
+  warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
+                    "dumping a microdescriptor");
   written = write_all_to_fd(fd, md->body, md->bodylen);
   if (written != (ssize_t)md->bodylen) {
     written = written < 0 ? 0 : written;
@@ -480,6 +486,27 @@ microdesc_cache_clear(microdesc_cache_t *cache)
   cache->bytes_dropped = 0;
 }
 
+static void
+warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                  const char *activity)
+{
+  const char *nul_found = memchr(inp, 0, len);
+  if (BUG(nul_found)) {
+    log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
+             "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
+             activity, offset, (nul_found - inp), len);
+    const char *start_excerpt_at, *eos = inp + len;
+    if ((nul_found - inp) >= 16)
+      start_excerpt_at = nul_found - 16;
+    else
+      start_excerpt_at = inp;
+    size_t excerpt_len = MIN(32, eos - start_excerpt_at);
+    char tmp[65];
+    base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
+    log_warn(LD_BUG, "      surrounding string: %s", tmp);
+  }
+}
+
 /** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
  * for the first time.  Return 0 on success, -1 on failure. */
 int
@@ -497,6 +524,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
 
   mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
   if (mm) {
+    warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
     added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
                                     SAVED_IN_CACHE, 0, -1, NULL);
     if (added) {
@@ -508,7 +536,9 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   journal_content = read_file_to_str(cache->journal_fname,
                                      RFTS_IGNORE_MISSING, &st);
   if (journal_content) {
-    cache->journal_len = (size_t) st.st_size;
+    cache->journal_len = strlen(journal_content);
+    warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
+                      "reading microdesc journal");
     added = microdescs_add_to_cache(cache, journal_content,
                                     journal_content+st.st_size,
                                     SAVED_IN_JOURNAL, 0, -1, NULL);
@@ -544,8 +574,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
   size_t bytes_dropped = 0;
   time_t now = time(NULL);
 
-  /* If we don't know a live consensus, don't believe last_listed values: we
-   * might be starting up after being down for a while. */
+  /* If we don't know a reasonably live consensus, don't believe last_listed
+   * values: we might be starting up after being down for a while. */
   if (! force &&
       ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
       return;
@@ -884,10 +914,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno)
   if (md->body && md->saved_location != SAVED_IN_CACHE)
     tor_free(md->body);
 
-  if (md->family) {
-    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
-    smartlist_free(md->family);
-  }
+  nodefamily_free(md->family);
   short_policy_free(md->exit_policy);
   short_policy_free(md->ipv6_exit_policy);
 
@@ -943,7 +970,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
       continue;
     if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
       continue;
-    if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
+    if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
       continue;
     /* XXXX Also skip if we're a noncache and wouldn't use this router.
      * XXXX NM Microdesc
@@ -973,6 +1000,7 @@ update_microdesc_downloads(time_t now)
   if (directory_too_idle_to_fetch_descriptors(options, now))
     return;
 
+  /* Give up if we don't have a reasonably live consensus. */
   consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
   if (!consensus)
     return;
diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h
index bb8b23d664..e017c46c79 100644
--- a/src/feature/nodelist/microdesc_st.h
+++ b/src/feature/nodelist/microdesc_st.h
@@ -9,6 +9,7 @@
 
 struct curve25519_public_key_t;
 struct ed25519_public_key_t;
+struct nodefamily_t;
 struct short_policy_t;
 
 /** A microdescriptor is the smallest amount of information needed to build a
@@ -32,6 +33,8 @@ struct microdesc_t {
   unsigned int no_save : 1;
   /** If true, this microdesc has an entry in the microdesc_map */
   unsigned int held_in_map : 1;
+  /** True iff the exit policy for this router rejects everything. */
+  unsigned int policy_is_reject_star : 1;
   /** Reference count: how many node_ts have a reference to this microdesc? */
   unsigned int held_by_nodes;
 
@@ -69,12 +72,12 @@ struct microdesc_t {
   tor_addr_t ipv6_addr;
   /** As routerinfo_t.ipv6_orport */
   uint16_t ipv6_orport;
-  /** As routerinfo_t.family */
-  smartlist_t *family;
+  /** As routerinfo_t.family, with readable members parsed. */
+  struct nodefamily_t *family;
   /** IPv4 exit policy summary */
   struct short_policy_t *exit_policy;
   /** IPv6 exit policy summary */
   struct short_policy_t *ipv6_exit_policy;
 };
 
-#endif
+#endif /* !defined(MICRODESC_ST_H) */
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
index c74acd8b74..496bafb865 100644
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -44,6 +44,7 @@
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitstats.h"
@@ -57,7 +58,7 @@
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
 #include "feature/dircache/dirserv.h"
@@ -67,6 +68,7 @@
 #include "feature/dircommon/voting_schedule.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/fmt_routerstatus.h"
@@ -81,6 +83,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
+#include "feature/dirauth/dirauth_periodic.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
@@ -116,8 +119,6 @@ STATIC networkstatus_t *current_md_consensus = NULL;
 typedef struct consensus_waiting_for_certs_t {
   /** The consensus itself. */
   networkstatus_t *consensus;
-  /** The encoded version of the consensus, nul-terminated. */
-  char *body;
   /** When did we set the current value of consensus_waiting_for_certs?  If
    * this is too recent, we shouldn't try to fetch a new consensus for a
    * little while, to give ourselves time to get certificates for this one. */
@@ -179,6 +180,10 @@ static void update_consensus_bootstrap_multiple_downloads(
 static int networkstatus_check_required_protocols(const networkstatus_t *ns,
                                                   int client_mode,
                                                   char **warning_out);
+static int reload_consensus_from_file(const char *fname,
+                                      const char *flavor,
+                                      unsigned flags,
+                                      const char *source_dir);
 
 /** Forget that we've warned about anything networkstatus-related, so we will
  * give fresh warnings if the same behavior happens again. */
@@ -210,14 +215,11 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
-/**
- * Read and and return the cached consensus of type <b>flavorname</b>.  If
- * <b>unverified</b> is true, get the one we haven't verified. Return NULL if
- * the file isn't there. */
+/** Return the filename used to cache the consensus of a given flavor */
 static char *
-networkstatus_read_cached_consensus_impl(int flav,
-                                         const char *flavorname,
-                                         int unverified_consensus)
+networkstatus_get_cache_fname(int flav,
+                              const char *flavorname,
+                              int unverified_consensus)
 {
   char buf[128];
   const char *prefix;
@@ -232,21 +234,35 @@ networkstatus_read_cached_consensus_impl(int flav,
     tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
   }
 
-  char *filename = get_cachedir_fname(buf);
-  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  return get_cachedir_fname(buf);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static tor_mmap_t *
+networkstatus_map_cached_consensus_impl(int flav,
+                                        const char *flavorname,
+                                        int unverified_consensus)
+{
+  char *filename = networkstatus_get_cache_fname(flav,
+                                                 flavorname,
+                                                 unverified_consensus);
+  tor_mmap_t *result = tor_mmap_file(filename);
   tor_free(filename);
   return result;
 }
 
-/** Return a new string containing the current cached consensus of flavor
- * <b>flavorname</b>. */
-char *
-networkstatus_read_cached_consensus(const char *flavorname)
- {
+/** Map the file containing the current cached consensus of flavor
+ * <b>flavorname</b> */
+tor_mmap_t *
+networkstatus_map_cached_consensus(const char *flavorname)
+{
   int flav = networkstatus_parse_flavor_name(flavorname);
   if (flav < 0)
     return NULL;
-  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+  return networkstatus_map_cached_consensus_impl(flav, flavorname, 0);
 }
 
 /** Read every cached v3 consensus networkstatus from the disk. */
@@ -259,25 +275,15 @@ router_reload_consensus_networkstatus(void)
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
     const char *flavor = networkstatus_get_flavor_name(flav);
-    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
-        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
-                 flavor);
-      }
-      tor_free(s);
-    }
+    char *fname = networkstatus_get_cache_fname(flav, flavor, 0);
+    reload_consensus_from_file(fname, flavor, flags, NULL);
+    tor_free(fname);
 
-    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor,
-                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
-                                     NULL)) {
-        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
-                 "from cache", flavor);
-      }
-      tor_free(s);
-    }
+    fname = networkstatus_get_cache_fname(flav, flavor, 1);
+    reload_consensus_from_file(fname, flavor,
+                               flags | NSSET_WAS_WAITING_FOR_CERTS,
+                               NULL);
+    tor_free(fname);
   }
 
   update_certificate_downloads(time(NULL));
@@ -713,8 +719,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest)
 
 /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or
  * NULL if none was found. */
-const routerstatus_t *
-networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+MOCK_IMPL(const routerstatus_t *,
+networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest))
 {
   return networkstatus_vote_find_mutable_entry(ns, digest);
 }
@@ -1377,7 +1383,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor))
 }
 
 /** Return the most recent consensus that we have downloaded, or NULL if we
- * don't have one. */
+ * don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus,(void))
 {
@@ -1388,7 +1394,7 @@ networkstatus_get_latest_consensus,(void))
 }
 
 /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL
- * if we don't have one. */
+ * if we don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
 {
@@ -1422,10 +1428,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now)
   return (ns->valid_after <= now && now <= ns->valid_until);
 }
 
-/** Determine if <b>consensus</b> is valid or expired recently enough that
- * we can still use it.
+/** Determine if <b>consensus</b> is valid, or expired recently enough, or not
+ * too far in the future, so that we can still use it.
  *
- * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old or
+ * too new.
  */
 int
 networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
@@ -1434,29 +1441,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
   if (BUG(!consensus))
     return 0;
 
-  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+  return networkstatus_valid_after_is_reasonably_live(consensus->valid_after,
+                                                      now) &&
+         networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
                                                       now);
 }
 
+#define REASONABLY_LIVE_TIME (24*60*60)
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_after
+ * time, and checks to see if it is in the past, or not too far in the future.
+ */
+int
+networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                             time_t now)
+{
+  return (now >= valid_after - REASONABLY_LIVE_TIME);
+}
+
 /** As networkstatus_consensus_reasonably_live, but takes a valid_until
- * time rather than an entire consensus. */
+ * time, and checks to see if it is in the future, or not too far in the past.
+ */
 int
 networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                              time_t now)
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   return (now <= valid_until + REASONABLY_LIVE_TIME);
 }
 
 /** As networkstatus_get_live_consensus(), but is way more tolerant of expired
- * consensuses. */
+ *  and future consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_reasonably_live_consensus,(time_t now, int flavor))
 {
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
-      consensus->valid_after <= now &&
       networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
@@ -1655,6 +1675,7 @@ notify_before_networkstatus_changes(const networkstatus_t *old_c,
   notify_control_networkstatus_changed(old_c, new_c);
   dos_consensus_has_changed(new_c);
   relay_consensus_has_changed(new_c);
+  hs_dos_consensus_has_changed(new_c);
 }
 
 /* Called after a new consensus has been put in the global state. It is safe
@@ -1725,6 +1746,44 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
 #endif /* defined(TOR_UNIT_TESTS) */
 
 /**
+ * Helper: Read the current consensus of type <b>flavor</b> from
+ * <b>fname</b>.  Flags and return values are as for
+ * networkstatus_set_current_consensus().
+ **/
+static int
+reload_consensus_from_file(const char *fname,
+                           const char *flavor,
+                           unsigned flags,
+                           const char *source_dir)
+{
+  tor_mmap_t *map = tor_mmap_file(fname);
+  if (!map)
+    return 0;
+
+  int rv = networkstatus_set_current_consensus(map->data, map->size,
+                                               flavor, flags, source_dir);
+#ifdef _WIN32
+  if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) {
+    log_notice(LD_GENERAL, "Looks like the above failures are probably "
+               "because of a CRLF in consensus file %s; falling back to "
+               "read_file_to_string. Nothing to worry about: this file "
+               "was probably saved by an earlier version of Tor.",
+               escaped(fname));
+    char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+    rv = networkstatus_set_current_consensus(content, strlen(content),
+                                             flavor, flags, source_dir);
+    tor_free(content);
+  }
+#endif /* defined(_WIN32) */
+  if (rv < -1) {
+    log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s",
+             escaped(fname));
+  }
+  tor_munmap_file(map);
+  return rv;
+}
+
+/**
  * Helper for handle_missing_protocol_warning: handles either the
  * client case (if <b>is_client</b> is set) or the server case otherwise.
  */
@@ -1841,6 +1900,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor,
  */
 int
 networkstatus_set_current_consensus(const char *consensus,
+                                    size_t consensus_len,
                                     const char *flavor,
                                     unsigned flags,
                                     const char *source_dir)
@@ -1869,7 +1929,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   /* Make sure it's parseable. */
-  c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS);
+  c = networkstatus_parse_vote_from_string(consensus,
+                                           consensus_len,
+                                           NULL, NS_TYPE_CONSENSUS);
   if (!c) {
     log_warn(LD_DIR, "Unable to parse networkstatus consensus");
     result = -2;
@@ -1957,14 +2019,12 @@ networkstatus_set_current_consensus(const char *consensus,
           c->valid_after > current_valid_after) {
         waiting = &consensus_waiting_for_certs[flav];
         networkstatus_vote_free(waiting->consensus);
-        tor_free(waiting->body);
         waiting->consensus = c;
         free_consensus = 0;
-        waiting->body = tor_strdup(consensus);
         waiting->set_at = now;
         waiting->dl_failed = 0;
         if (!from_cache) {
-          write_str_to_file(unverified_fname, consensus, 0);
+          write_bytes_to_file(unverified_fname, consensus, consensus_len, 1);
         }
         if (dl_certs)
           authority_certs_fetch_missing(c, now, source_dir);
@@ -1976,9 +2036,9 @@ networkstatus_set_current_consensus(const char *consensus,
          * latest consensus. */
         if (was_waiting_for_certs && from_cache)
           if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
           }
       }
       goto done;
@@ -1991,9 +2051,9 @@ networkstatus_set_current_consensus(const char *consensus,
       }
       if (was_waiting_for_certs && (r < -1) && from_cache) {
         if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
         }
       }
       goto done;
@@ -2055,16 +2115,12 @@ networkstatus_set_current_consensus(const char *consensus,
       waiting->consensus->valid_after <= c->valid_after) {
     networkstatus_vote_free(waiting->consensus);
     waiting->consensus = NULL;
-    if (consensus != waiting->body)
-      tor_free(waiting->body);
-    else
-      waiting->body = NULL;
     waiting->set_at = 0;
     waiting->dl_failed = 0;
     if (unlink(unverified_fname) != 0) {
-      log_warn(LD_FS,
-               "Failed to unlink %s: %s",
-               unverified_fname, strerror(errno));
+      log_debug(LD_FS,
+                "Failed to unlink %s: %s",
+                unverified_fname, strerror(errno));
     }
   }
 
@@ -2082,7 +2138,6 @@ networkstatus_set_current_consensus(const char *consensus,
 
     nodelist_set_consensus(c);
 
-    /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
     update_consensus_networkstatus_fetch_time(now);
 
     /* Change the cell EWMA settings */
@@ -2095,6 +2150,7 @@ networkstatus_set_current_consensus(const char *consensus,
     circuit_build_times_new_consensus_params(
                                get_circuit_build_times_mutable(), c);
     channelpadding_new_consensus_params(c);
+    circpad_new_consensus_params(c);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
@@ -2108,17 +2164,18 @@ networkstatus_set_current_consensus(const char *consensus,
   if (we_want_to_fetch_flavor(options, flav)) {
     if (dir_server_mode(get_options())) {
       dirserv_set_cached_consensus_networkstatus(consensus,
+                                                 consensus_len,
                                                  flavor,
                                                  &c->digests,
                                                  c->digest_sha3_as_signed,
                                                  c->valid_after);
 
-      consdiffmgr_add_consensus(consensus, c);
+      consdiffmgr_add_consensus(consensus, consensus_len, c);
     }
   }
 
   if (!from_cache) {
-    write_str_to_file(consensus_fname, consensus, 0);
+    write_bytes_to_file(consensus_fname, consensus, consensus_len, 1);
   }
 
   warn_early_consensus(c, flavor, now);
@@ -2154,14 +2211,10 @@ networkstatus_note_certs_arrived(const char *source_dir)
     if (!waiting->consensus)
       continue;
     if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
-      char *waiting_body = waiting->body;
-      if (!networkstatus_set_current_consensus(
-                                 waiting_body,
-                                 flavor_name,
-                                 NSSET_WAS_WAITING_FOR_CERTS,
-                                 source_dir)) {
-        tor_free(waiting_body);
-      }
+      char *fname = networkstatus_get_cache_fname(i, flavor_name, 1);
+      reload_consensus_from_file(fname, flavor_name,
+                                 NSSET_WAS_WAITING_FOR_CERTS, source_dir);
+      tor_free(fname);
     }
   }
 }
@@ -2315,6 +2368,49 @@ networkstatus_getinfo_helper_single(const routerstatus_t *rs)
                                    NULL);
 }
 
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
+ *
+ * We assume that node-\>is_running has already been set, e.g. by
+ *   dirserv_set_router_is_running(ri, now);
+ */
+void
+set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                 const node_t *node,
+                                 const routerinfo_t *ri)
+{
+  memset(rs, 0, sizeof(routerstatus_t));
+
+  rs->is_authority =
+    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
+
+  /* Set by compute_performance_thresholds or from consensus */
+  rs->is_exit = node->is_exit;
+  rs->is_stable = node->is_stable;
+  rs->is_fast = node->is_fast;
+  rs->is_flagged_running = node->is_running;
+  rs->is_valid = node->is_valid;
+  rs->is_possible_guard = node->is_possible_guard;
+  rs->is_bad_exit = node->is_bad_exit;
+  rs->is_hs_dir = node->is_hs_dir;
+  rs->is_named = rs->is_unnamed = 0;
+
+  rs->published_on = ri->cache_info.published_on;
+  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
+  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  rs->addr = ri->addr;
+  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
+  rs->or_port = ri->or_port;
+  rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
+
+  tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
+  rs->ipv6_orport = ri->ipv6_orport;
+}
+
 /** Alloc and return a string describing routerstatuses for the most
  * recent info of each router we know about that is of purpose
  * <b>purpose_string</b>. Return NULL if unrecognized purpose.
@@ -2331,7 +2427,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   smartlist_t *statuses;
   const uint8_t purpose = router_purpose_from_string(purpose_string);
   routerstatus_t rs;
-  const int bridge_auth = authdir_mode_bridge(get_options());
 
   if (purpose == ROUTER_PURPOSE_UNKNOWN) {
     log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
@@ -2348,11 +2443,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
       continue;
     if (ri->purpose != purpose)
       continue;
-    /* TODO: modifying the running flag in a getinfo is a bad idea */
-    if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
-      dirserv_set_router_is_running(ri, now);
-    /* then generate and write out status lines for each of them */
-    set_routerstatus_from_routerinfo(&rs, node, ri, now, 0);
+    set_routerstatus_from_routerinfo(&rs, node, ri);
     smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs));
   } SMARTLIST_FOREACH_END(ri);
 
@@ -2362,41 +2453,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   return answer;
 }
 
-/** Write out router status entries for all our bridge descriptors. */
-void
-networkstatus_dump_bridge_status_to_file(time_t now)
-{
-  char *status = networkstatus_getinfo_by_purpose("bridge", now);
-  char *fname = NULL;
-  char *thresholds = NULL;
-  char *published_thresholds_and_status = NULL;
-  char published[ISO_TIME_LEN+1];
-  const routerinfo_t *me = router_get_my_routerinfo();
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *fingerprint_line = NULL;
-
-  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
-                                      fingerprint, 0) >= 0) {
-    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
-  } else {
-    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
-  }
-  format_iso_time(published, now);
-  dirserv_compute_bridge_flag_thresholds();
-  thresholds = dirserv_get_flag_thresholds_line();
-  tor_asprintf(&published_thresholds_and_status,
-               "published %s\nflag-thresholds %s\n%s%s",
-               published, thresholds, fingerprint_line ? fingerprint_line : "",
-               status);
-  fname = get_datadir_fname("networkstatus-bridges");
-  write_str_to_file(fname,published_thresholds_and_status,0);
-  tor_free(thresholds);
-  tor_free(published_thresholds_and_status);
-  tor_free(fname);
-  tor_free(status);
-  tor_free(fingerprint_line);
-}
-
 /* DOCDOC get_net_param_from_list */
 static int32_t
 get_net_param_from_list(smartlist_t *net_params, const char *param_name,
@@ -2668,6 +2724,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
   const char *required, *recommended;
   char *missing = NULL;
 
+  const bool consensus_postdates_this_release =
+    ns->valid_after >= tor_get_approx_release_date();
+
   tor_assert(warning_out);
 
   if (client_mode) {
@@ -2685,7 +2744,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
                  "%s on the Tor network. The missing protocols are: %s",
                  func, missing);
     tor_free(missing);
-    return 1;
+    return consensus_postdates_this_release ? 1 : 0;
   }
 
   if (! protover_all_supported(recommended, &missing)) {
@@ -2718,6 +2777,5 @@ networkstatus_free_all(void)
       networkstatus_vote_free(waiting->consensus);
       waiting->consensus = NULL;
     }
-    tor_free(waiting->body);
   }
 }
diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h
index 9e7b0f1bb0..600fd7fbd5 100644
--- a/src/feature/nodelist/networkstatus.h
+++ b/src/feature/nodelist/networkstatus.h
@@ -16,7 +16,7 @@
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
-char *networkstatus_read_cached_consensus(const char *flavorname);
+tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
 void routerstatus_free_(routerstatus_t *rs);
 #define routerstatus_free(rs) \
@@ -40,8 +40,9 @@ int compare_digest_to_routerstatus_entry(const void *_key,
                                          const void **_member);
 int compare_digest_to_vote_routerstatus_entry(const void *_key,
                                               const void **_member);
-const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns,
-                                              const char *digest);
+MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,(
+                                              networkstatus_t *ns,
+                                              const char *digest));
 routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns,
                                               const char *digest);
 int networkstatus_vote_find_entry_idx(networkstatus_t *ns,
@@ -87,6 +88,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
 int networkstatus_is_live(const networkstatus_t *ns, time_t now);
 int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
                                             time_t now);
+int networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                                 time_t now);
 int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                                  time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus,
@@ -106,6 +109,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options);
 #define NSSET_ACCEPT_OBSOLETE 8
 #define NSSET_REQUIRE_FLAVOR 16
 int networkstatus_set_current_consensus(const char *consensus,
+                                        size_t consensus_len,
                                         const char *flavor,
                                         unsigned flags,
                                         const char *source_dir);
@@ -118,7 +122,6 @@ void signed_descs_update_status_from_consensus_networkstatus(
 
 char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
 char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
-void networkstatus_dump_bridge_status_to_file(time_t now);
 MOCK_DECL(int32_t, networkstatus_get_param,
           (const networkstatus_t *ns, const char *param_name,
            int32_t default_val, int32_t min_val, int32_t max_val));
@@ -145,6 +148,10 @@ void vote_routerstatus_free_(vote_routerstatus_t *rs);
 #define vote_routerstatus_free(rs) \
   FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
 
+void set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                      const node_t *node,
+                                      const routerinfo_t *ri);
+
 #ifdef NETWORKSTATUS_PRIVATE
 #ifdef TOR_UNIT_TESTS
 STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
@@ -157,4 +164,3 @@ extern networkstatus_t *current_md_consensus;
 #endif /* defined(NETWORKSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_NETWORKSTATUS_H) */
-
diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h
index 677d8ed811..420c3d61e4 100644
--- a/src/feature/nodelist/networkstatus_sr_info_st.h
+++ b/src/feature/nodelist/networkstatus_sr_info_st.h
@@ -19,5 +19,5 @@ struct networkstatus_sr_info_t {
   smartlist_t *commits;
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_SR_INFO_ST_H) */
 
diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h
index 6160f12361..6e84c170d6 100644
--- a/src/feature/nodelist/networkstatus_st.h
+++ b/src/feature/nodelist/networkstatus_st.h
@@ -99,6 +99,9 @@ struct networkstatus_t {
 
   /** List of key=value strings from the headers of the bandwidth list file */
   smartlist_t *bw_file_headers;
+
+  /** A SHA256 digest of the bandwidth file used in a vote. */
+  uint8_t bw_file_digest256[DIGEST256_LEN];
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_ST_H) */
diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h
index 4037fcdeca..66af82a8e3 100644
--- a/src/feature/nodelist/networkstatus_voter_info_st.h
+++ b/src/feature/nodelist/networkstatus_voter_info_st.h
@@ -27,4 +27,4 @@ struct networkstatus_voter_info_t {
   smartlist_t *sigs;
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_VOTER_INFO_ST_H) */
diff --git a/src/feature/nodelist/nickname.h b/src/feature/nodelist/nickname.h
index 9bdc6b50e8..78db2a5f91 100644
--- a/src/feature/nodelist/nickname.h
+++ b/src/feature/nodelist/nickname.h
@@ -16,4 +16,4 @@ int is_legal_nickname(const char *s);
 int is_legal_nickname_or_hexdigest(const char *s);
 int is_legal_hexdigest(const char *s);
 
-#endif
+#endif /* !defined(TOR_NICKNAME_H) */
diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c
index e31abb247f..719b4b1b27 100644
--- a/src/feature/nodelist/node_select.c
+++ b/src/feature/nodelist/node_select.c
@@ -30,6 +30,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
+#include "lib/container/bitarray.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/math/fp.h"
 
@@ -585,6 +586,7 @@ compute_weighted_bandwidths(const smartlist_t *sl,
   }
 
   weight_scale = networkstatus_get_weight_scale_param(NULL);
+  tor_assert(weight_scale >= 1);
 
   if (rule == WEIGHT_FOR_GUARD) {
     Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
@@ -825,6 +827,58 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
   nodelist_add_node_and_family(sl, node);
 }
 
+/**
+ * Remove every node_t that appears in <b>excluded</b> from <b>sl</b>.
+ *
+ * Behaves like smartlist_subtract, but uses nodelist_idx values to deliver
+ * linear performance when smartlist_subtract would be quadratic.
+ **/
+static void
+nodelist_subtract(smartlist_t *sl, const smartlist_t *excluded)
+{
+  const smartlist_t *nodelist = nodelist_get_list();
+  const int nodelist_len = smartlist_len(nodelist);
+  bitarray_t *excluded_idx = bitarray_init_zero(nodelist_len);
+
+  /* We haven't used nodelist_idx in this way previously, so I'm going to be
+   * paranoid in this code, and check that nodelist_idx is correct for every
+   * node before we use it.  If we fail, we fall back to smartlist_subtract().
+   */
+
+  /* Set the excluded_idx bit corresponding to every excluded node...
+   */
+  SMARTLIST_FOREACH_BEGIN(excluded, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    bitarray_set(excluded_idx, idx);
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Then remove them from sl.
+   */
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    if (bitarray_is_set(excluded_idx, idx)) {
+      SMARTLIST_DEL_CURRENT(sl, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  bitarray_free(excluded_idx);
+  return;
+
+ internal_error:
+  log_warn(LD_BUG, "Internal error prevented us from using the fast method "
+           "for subtracting nodelists. Falling back to the quadratic way.");
+  smartlist_subtract(sl, excluded);
+  bitarray_free(excluded_idx);
+}
+
 /** Return a random running node from the nodelist. Never
  * pick a node that is in
  * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
@@ -859,6 +913,7 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
   const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
 
+  const smartlist_t *node_list = nodelist_get_list();
   smartlist_t *sl=smartlist_new(),
     *excludednodes=smartlist_new();
   const node_t *choice = NULL;
@@ -869,17 +924,17 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   rule = weight_for_exit ? WEIGHT_FOR_EXIT :
     (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
 
-  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+  SMARTLIST_FOREACH_BEGIN(node_list, const node_t *, node) {
     if (node_allows_single_hop_exits(node)) {
       /* Exclude relays that allow single hop exit circuits. This is an
        * obsolete option since 0.2.9.2-alpha and done by default in
        * 0.3.1.0-alpha. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     } else if (rendezvous_v3 &&
                !node_supports_v3_rendezvous_point(node)) {
       /* Exclude relays that do not support to rendezvous for a hidden service
        * version 3. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     }
   } SMARTLIST_FOREACH_END(node);
 
@@ -896,19 +951,11 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
            "We found %d running nodes.",
             smartlist_len(sl));
 
-  smartlist_subtract(sl,excludednodes);
-  log_debug(LD_CIRC,
-            "We removed %d excludednodes, leaving %d nodes.",
-            smartlist_len(excludednodes),
-            smartlist_len(sl));
-
   if (excludedsmartlist) {
-    smartlist_subtract(sl,excludedsmartlist);
-    log_debug(LD_CIRC,
-              "We removed %d excludedsmartlist, leaving %d nodes.",
-              smartlist_len(excludedsmartlist),
-              smartlist_len(sl));
+    smartlist_add_all(excludednodes, excludedsmartlist);
   }
+  nodelist_subtract(sl, excludednodes);
+
   if (excludedset) {
     routerset_subtract_nodes(sl,excludedset);
     log_debug(LD_CIRC,
diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h
index ed7450b92c..d8b4aca5c1 100644
--- a/src/feature/nodelist/node_select.h
+++ b/src/feature/nodelist/node_select.h
@@ -97,6 +97,6 @@ STATIC const routerstatus_t *router_pick_directory_server_impl(
                                            int *n_busy_out);
 STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
                                           int serverdesc, int microdesc);
-#endif
+#endif /* defined(NODE_SELECT_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NODE_SELECT_H) */
diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h
index 53ffde29e4..c63a535a19 100644
--- a/src/feature/nodelist/node_st.h
+++ b/src/feature/nodelist/node_st.h
@@ -99,4 +99,4 @@ struct node_t {
   struct hsdir_index_t hsdir_index;
 };
 
-#endif
+#endif /* !defined(NODE_ST_H) */
diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c
new file mode 100644
index 0000000000..2ec9d5fa40
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.c
@@ -0,0 +1,416 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.c
+ * \brief Code to manipulate encoded, reference-counted node families.  We
+ *  use these tricks to save space, since these families would otherwise
+ *  require a large number of tiny allocations.
+ **/
+
+#include "core/or/or.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
+#include "feature/nodelist/nodefamily_st.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/routerlist.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * Allocate and return a blank node family with space to hold <b>n_members</b>
+ * members.
+ */
+static nodefamily_t *
+nodefamily_alloc(int n_members)
+{
+  size_t alloc_len = offsetof(nodefamily_t, family_members) +
+    NODEFAMILY_ARRAY_SIZE(n_members);
+  nodefamily_t *nf = tor_malloc_zero(alloc_len);
+  nf->n_members = n_members;
+  return nf;
+}
+
+/**
+ * Hashtable hash implementation.
+ */
+static inline unsigned int
+nodefamily_hash(const nodefamily_t *nf)
+{
+  return (unsigned) siphash24g(nf->family_members,
+                               NODEFAMILY_ARRAY_SIZE(nf->n_members));
+}
+
+/**
+ * Hashtable equality implementation.
+ */
+static inline unsigned int
+nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b)
+{
+  return (a->n_members == b->n_members) &&
+    fast_memeq(a->family_members, b->family_members,
+               NODEFAMILY_ARRAY_SIZE(a->n_members));
+}
+
+static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families
+  = HT_INITIALIZER();
+
+HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             nodefamily_eq)
+HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             node_family_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/**
+ * Parse the family declaration in <b>s</b>, returning the canonical
+ * <b>nodefamily_t</b> for its members.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * If NF_WARN_MALFORMED is set in <b>flags</b>, warn about any
+ * elements that we can't parse.  (By default, we log at info.)
+ *
+ * If NF_REJECT_MALFORMED is set in <b>flags</b>, treat any unparseable
+ * elements as an error. (By default, we simply omit them.)
+ **/
+nodefamily_t *
+nodefamily_parse(const char *s, const uint8_t *rsa_id_self,
+                 unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Canonicalize the family list <b>s</b>, returning a newly allocated string.
+ *
+ * The canonicalization rules are fully specified in dir-spec.txt, but,
+ * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are
+ * truncated, nicknames are put into lowercase, unrecognized entries are left
+ * alone, and everything is sorted.
+ **/
+char *
+nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                        unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_t *result_members = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags,
+                                             result_members);
+
+  char *formatted = nodefamily_format(nf);
+  smartlist_split_string(result_members, formatted, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  smartlist_sort_strings(result_members);
+  char *combined = smartlist_join_strings(result_members, " ", 0, NULL);
+
+  nodefamily_free(nf);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp));
+  smartlist_free(result_members);
+  tor_free(formatted);
+
+  return combined;
+}
+
+/**
+ * qsort helper for encoded nodefamily elements.
+ **/
+static int
+compare_members(const void *a, const void *b)
+{
+  return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN);
+}
+
+/**
+ * Parse the member strings in <b>members</b>, returning their canonical
+ * <b>nodefamily_t</b>.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * The <b>flags</b> element is interpreted as in nodefamily_parse().
+ *
+ * If <b>unrecognized</b> is provided, fill it copies of any unrecognized
+ * members.  (Note that malformed $hexids are not considered unrecognized.)
+ **/
+nodefamily_t *
+nodefamily_from_members(const smartlist_t *members,
+                        const uint8_t *rsa_id_self,
+                        unsigned flags,
+                        smartlist_t *unrecognized_out)
+{
+  const int n_self = rsa_id_self ? 1 : 0;
+  int n_bad_elements = 0;
+  int n_members = smartlist_len(members) + n_self;
+  nodefamily_t *tmp = nodefamily_alloc(n_members);
+  uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0);
+
+  SMARTLIST_FOREACH_BEGIN(members, const char *, cp) {
+    bool bad_element = true;
+    if (is_legal_nickname(cp)) {
+      ptr[0] = NODEFAMILY_BY_NICKNAME;
+      tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname
+      memcpy(ptr+1, cp, strlen(cp));
+      tor_strlower((char*) ptr+1);
+      bad_element = false;
+    } else if (is_legal_hexdigest(cp)) {
+      char digest_buf[DIGEST_LEN];
+      char nn_buf[MAX_NICKNAME_LEN+1];
+      char nn_char=0;
+      if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) {
+        bad_element = false;
+        ptr[0] = NODEFAMILY_BY_RSA_ID;
+        memcpy(ptr+1, digest_buf, DIGEST_LEN);
+      }
+    } else {
+      if (unrecognized_out)
+        smartlist_add_strdup(unrecognized_out, cp);
+    }
+
+    if (bad_element) {
+      const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO;
+      log_fn(severity, LD_GENERAL,
+             "Bad element %s while parsing a node family.",
+             escaped(cp));
+      ++n_bad_elements;
+    } else {
+      ptr += NODEFAMILY_MEMBER_LEN;
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  if (n_bad_elements && (flags & NF_REJECT_MALFORMED))
+    goto err;
+
+  if (rsa_id_self) {
+    /* Add self. */
+    ptr[0] = NODEFAMILY_BY_RSA_ID;
+    memcpy(ptr+1, rsa_id_self, DIGEST_LEN);
+  }
+
+  n_members -= n_bad_elements;
+
+  /* Sort tmp into canonical order. */
+  qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN,
+        compare_members);
+
+  /* Remove duplicates. */
+  int i;
+  for (i = 0; i < n_members-1; ++i) {
+    uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i);
+    uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1);
+    if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) {
+      memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN);
+      --n_members;
+      --i;
+    }
+  }
+  int n_members_alloc = tmp->n_members;
+  tmp->n_members = n_members;
+
+  /* See if we already allocated this family. */
+  nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp);
+  if (found) {
+    /* If we did, great: incref it and return it. */
+    ++found->refcnt;
+    tor_free(tmp);
+    return found;
+  } else {
+    /* If not, insert it into the hashtable. */
+    if (n_members_alloc != n_members) {
+      /* Compact the family if needed */
+      nodefamily_t *tmp2 = nodefamily_alloc(n_members);
+      memcpy(tmp2->family_members, tmp->family_members,
+             n_members * NODEFAMILY_MEMBER_LEN);
+      tor_free(tmp);
+      tmp = tmp2;
+    }
+
+    tmp->refcnt = 1;
+    HT_INSERT(nodefamily_map, &the_node_families, tmp);
+    return tmp;
+  }
+
+ err:
+  tor_free(tmp);
+  return NULL;
+}
+
+/**
+ * Drop our reference to <b>family</b>, freeing it if there are no more
+ * references.
+ */
+void
+nodefamily_free_(nodefamily_t *family)
+{
+  if (family == NULL)
+    return;
+
+  --family->refcnt;
+
+  if (family->refcnt == 0) {
+    HT_REMOVE(nodefamily_map, &the_node_families, family);
+    tor_free(family);
+  }
+}
+
+/**
+ * Return true iff <b>family</b> contains the SHA1 RSA1024 identity
+ * <b>rsa_id</b>.
+ */
+bool
+nodefamily_contains_rsa_id(const nodefamily_t *family,
+                           const uint8_t *rsa_id)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    if (ptr[0] == NODEFAMILY_BY_RSA_ID &&
+        fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff <b>family</b> contains the nickname <b>name</b>.
+ */
+bool
+nodefamily_contains_nickname(const nodefamily_t *family,
+                             const char *name)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    // note that the strcasecmp() is safe because there is always at least one
+    // NUL in the encoded nickname, because all legal nicknames are less than
+    // DIGEST_LEN bytes long.
+    if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true if <b>family</b> contains the nickname or the RSA ID for
+ * <b>node</b>
+ **/
+bool
+nodefamily_contains_node(const nodefamily_t *family,
+                         const node_t *node)
+{
+  return
+    nodefamily_contains_nickname(family, node_get_nickname(node))
+    ||
+    nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node));
+}
+
+/**
+ * Look up every entry in <b>family</b>, and add add the corresponding
+ * node_t to <b>out</b>.
+ **/
+void
+nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                  smartlist_t *out)
+{
+  if (!family)
+    return;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    const node_t *node = NULL;
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED);
+        break;
+      case NODEFAMILY_BY_RSA_ID:
+        node = node_get_by_id((char*)ptr+1);
+        break;
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+    if (node)
+      smartlist_add(out, (void *)node);
+  }
+}
+
+/**
+ * Encode <b>family</b> as a space-separated string.
+ */
+char *
+nodefamily_format(const nodefamily_t *family)
+{
+  if (!family)
+    return tor_strdup("");
+
+  unsigned i;
+  smartlist_t *sl = smartlist_new();
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        smartlist_add_strdup(sl, (char*)ptr+1);
+        break;
+      case NODEFAMILY_BY_RSA_ID: {
+        char buf[HEX_DIGEST_LEN+2];
+        buf[0]='$';
+        base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN);
+        tor_strupper(buf);
+        smartlist_add_strdup(sl, buf);
+        break;
+      }
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+  }
+
+  char *result = smartlist_join_strings(sl, " ", 0, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Free all storage held in the nodefamily map.
+ **/
+void
+nodefamily_free_all(void)
+{
+  HT_CLEAR(nodefamily_map, &the_node_families);
+}
diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h
new file mode 100644
index 0000000000..31b71e77a0
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.h
+ * \brief Header file for nodefamily.c.
+ **/
+
+#ifndef TOR_NODEFAMILY_H
+#define TOR_NODEFAMILY_H
+
+#include "lib/malloc/malloc.h"
+#include <stdbool.h>
+
+typedef struct nodefamily_t nodefamily_t;
+struct node_t;
+struct smartlist_t;
+
+#define NF_WARN_MALFORMED    (1u<<0)
+#define NF_REJECT_MALFORMED  (1u<<1)
+
+nodefamily_t *nodefamily_parse(const char *s,
+                               const uint8_t *rsa_id_self,
+                               unsigned flags);
+nodefamily_t *nodefamily_from_members(const struct smartlist_t *members,
+                                      const uint8_t *rsa_id_self,
+                                      unsigned flags,
+                                      smartlist_t *unrecognized_out);
+void nodefamily_free_(nodefamily_t *family);
+#define nodefamily_free(family) \
+  FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family))
+
+bool nodefamily_contains_rsa_id(const nodefamily_t *family,
+                                const uint8_t *rsa_id);
+bool nodefamily_contains_nickname(const nodefamily_t *family,
+                                  const char *name);
+bool nodefamily_contains_node(const nodefamily_t *family,
+                              const struct node_t *node);
+void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                       struct smartlist_t *out);
+char *nodefamily_format(const nodefamily_t *family);
+char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                              unsigned flags);
+
+void nodefamily_free_all(void);
+
+#endif /* !defined(TOR_NODEFAMILY_H) */
diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h
new file mode 100644
index 0000000000..20390c9308
--- /dev/null
+++ b/src/feature/nodelist/nodefamily_st.h
@@ -0,0 +1,48 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NODEFAMILY_ST_H
+#define TOR_NODEFAMILY_ST_H
+
+#include "orconfig.h"
+#include "ht.h"
+
+struct nodefamily_t {
+  /** Entry for this nodefamily_t within the hashtable. */
+  HT_ENTRY(nodefamily_t) ht_ent;
+  /** Reference count.  (The hashtable is not treated as a reference */
+  uint32_t refcnt;
+  /** Number of items encoded in <b>family_members</b>. */
+  uint32_t n_members;
+  /* A byte-array encoding the members of this family.  We encode each member
+   * as one byte to indicate whether it's a nickname or a fingerprint, plus
+   * DIGEST_LEN bytes of data.  The entries are lexically sorted.
+   */
+  uint8_t family_members[FLEXIBLE_ARRAY_MEMBER];
+};
+
+#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN)
+
+/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID.
+ */
+#define NODEFAMILY_BY_RSA_ID 0
+/** Tag byte, indicates that the following bytes are a NUL-padded nickname.
+ */
+#define NODEFAMILY_BY_NICKNAME 1
+
+/**
+ * Number of bytes to allocate in the array for a nodefamily_t with N members.
+ **/
+#define NODEFAMILY_ARRAY_SIZE(n) \
+  ((n) * NODEFAMILY_MEMBER_LEN)
+
+/**
+ * Pointer to the i'th member of <b>nf</b>, as encoded.
+ */
+#define NODEFAMILY_MEMBER_PTR(nf, i) \
+  (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN]))
+
+#endif /* !defined(TOR_NODEFAMILY_ST_H) */
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
index 99d7f746a8..9191173c3b 100644
--- a/src/feature/nodelist/nodelist.c
+++ b/src/feature/nodelist/nodelist.c
@@ -49,7 +49,7 @@
 #include "core/or/protover.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/hs/hs_client.h"
@@ -59,6 +59,7 @@
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
@@ -454,22 +455,43 @@ node_add_to_address_set(const node_t *node)
 
   if (node->rs) {
     if (node->rs->addr)
-      address_set_add_ipv4h(the_nodelist->node_addrs, node->rs->addr);
+      nodelist_add_addr4_to_address_set(node->rs->addr);
     if (!tor_addr_is_null(&node->rs->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->rs->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->rs->ipv6_addr);
   }
   if (node->ri) {
     if (node->ri->addr)
-      address_set_add_ipv4h(the_nodelist->node_addrs, node->ri->addr);
+      nodelist_add_addr4_to_address_set(node->ri->addr);
     if (!tor_addr_is_null(&node->ri->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->ri->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->ri->ipv6_addr);
   }
   if (node->md) {
     if (!tor_addr_is_null(&node->md->ipv6_addr))
-      address_set_add(the_nodelist->node_addrs, &node->md->ipv6_addr);
+      nodelist_add_addr6_to_address_set(&node->md->ipv6_addr);
   }
 }
 
+/** Add the given v4 address into the nodelist address set. */
+void
+nodelist_add_addr4_to_address_set(const uint32_t addr)
+{
+  if (!the_nodelist || !the_nodelist->node_addrs || addr == 0) {
+    return;
+  }
+  address_set_add_ipv4h(the_nodelist->node_addrs, addr);
+}
+
+/** Add the given v6 address into the nodelist address set. */
+void
+nodelist_add_addr6_to_address_set(const tor_addr_t *addr)
+{
+  if (BUG(!addr) || tor_addr_is_null(addr) || tor_addr_is_v4(addr) ||
+      !the_nodelist || !the_nodelist->node_addrs) {
+    return;
+  }
+  address_set_add(the_nodelist->node_addrs, addr);
+}
+
 /** Return true if <b>addr</b> is the address of some node in the nodelist.
  * If not, probably return false. */
 int
@@ -611,9 +633,12 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH(the_nodelist->nodes, node_t *, node,
                     node->rs = NULL);
 
-  /* Conservatively estimate that every node will have 2 addresses. */
-  const int estimated_addresses = smartlist_len(ns->routerstatus_list) *
-                                  get_estimated_address_per_node();
+  /* Conservatively estimate that every node will have 2 addresses (v4 and
+   * v6). Then we add the number of configured trusted authorities we have. */
+  int estimated_addresses = smartlist_len(ns->routerstatus_list) *
+                            get_estimated_address_per_node();
+  estimated_addresses += (get_n_authorities(V3_DIRINFO & BRIDGE_DIRINFO) *
+                          get_estimated_address_per_node());
   address_set_free(the_nodelist->node_addrs);
   the_nodelist->node_addrs = address_set_new(estimated_addresses);
 
@@ -664,6 +689,9 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
     node_add_to_address_set(node);
   } SMARTLIST_FOREACH_END(node);
+  /* Then, add all trusted configured directories. Some might not be in the
+   * consensus so make sure we know them. */
+  dirlist_add_trusted_dir_addresses();
 
   if (! authdir) {
     SMARTLIST_FOREACH_BEGIN(the_nodelist->nodes, node_t *, node) {
@@ -943,7 +971,7 @@ nodelist_ensure_freshness(networkstatus_t *ns)
 /** Return a list of a node_t * for every node we know about.  The caller
  * MUST NOT modify the list. (You can set and clear flags in the nodes if
  * you must, but you must not add or remove nodes.) */
-MOCK_IMPL(smartlist_t *,
+MOCK_IMPL(const smartlist_t *,
 nodelist_get_list,(void))
 {
   init_nodelist();
@@ -1105,7 +1133,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
 /** Dummy object that should be unreturnable.  Used to ensure that
  * node_get_protover_summary_flags() always returns non-NULL. */
 static const protover_summary_flags_t zero_protover_flags = {
-  0,0,0,0,0,0,0
+  0,0,0,0,0,0,0,0,0
 };
 
 /** Return the protover_summary_flags for a given node. */
@@ -1165,6 +1193,17 @@ node_supports_ed25519_hs_intro(const node_t *node)
   return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
 }
 
+/** Return true iff <b>node</b> supports the DoS ESTABLISH_INTRO cell
+ * extenstion. */
+int
+node_supports_establish_intro_dos_extension(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->
+                           supports_establish_intro_dos_extension;
+}
+
 /** Return true iff <b>node</b> supports to be a rendezvous point for hidden
  * service version 3 (HSRend=2). */
 int
@@ -1188,6 +1227,102 @@ node_get_rsa_id_digest(const node_t *node)
   return (const uint8_t*)node->identity;
 }
 
+/* Returns a new smartlist with all possible link specifiers from node:
+ *  - legacy ID is mandatory thus MUST be present in node;
+ *  - include ed25519 link specifier if present in the node, and the node
+ *    supports ed25519 link authentication, and:
+ *    - if direct_conn is true, its link versions are compatible with us,
+ *    - if direct_conn is false, regardless of its link versions;
+ *  - include IPv4 link specifier, if the primary address is not IPv4, log a
+ *    BUG() warning, and return an empty smartlist;
+ *  - include IPv6 link specifier if present in the node.
+ *
+ * If node is NULL, returns an empty smartlist.
+ *
+ * The smartlist must be freed using link_specifier_smartlist_free(). */
+smartlist_t *
+node_get_link_specifier_smartlist(const node_t *node, bool direct_conn)
+{
+  link_specifier_t *ls;
+  tor_addr_port_t ap;
+  smartlist_t *lspecs = smartlist_new();
+
+  if (!node)
+    return lspecs;
+
+  /* Get the relay's IPv4 address. */
+  node_get_prim_orport(node, &ap);
+
+  /* We expect the node's primary address to be a valid IPv4 address.
+   * This conforms to the protocol, which requires either an IPv4 or IPv6
+   * address (or both). */
+  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
+      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
+    return lspecs;
+  }
+
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
+  link_specifier_set_un_ipv4_port(ls, ap.port);
+  /* Four bytes IPv4 and two bytes port. */
+  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
+                            sizeof(ap.port));
+  smartlist_add(lspecs, ls);
+
+  /* Legacy ID is mandatory and will always be present in node. */
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
+         link_specifier_getlen_un_legacy_id(ls));
+  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
+  smartlist_add(lspecs, ls);
+
+  /* ed25519 ID is only included if the node has it, and the node declares a
+   protocol version that supports ed25519 link authentication.
+   If direct_conn is true, we also require that the node's link version is
+   compatible with us. (Otherwise, we will be sending the ed25519 key
+   to another tor, which may support different link versions.) */
+  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
+      node_supports_ed25519_link_authentication(node, direct_conn)) {
+    ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, LS_ED25519_ID);
+    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
+           link_specifier_getlen_un_ed25519_id(ls));
+    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
+    smartlist_add(lspecs, ls);
+  }
+
+  /* Check for IPv6. If so, include it as well. */
+  if (node_has_ipv6_orport(node)) {
+    ls = link_specifier_new();
+    node_get_pref_ipv6_orport(node, &ap);
+    link_specifier_set_ls_type(ls, LS_IPV6);
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
+    smartlist_add(lspecs, ls);
+  }
+
+  return lspecs;
+}
+
+/* Free a link specifier list. */
+void
+link_specifier_smartlist_free_(smartlist_t *ls_list)
+{
+  if (!ls_list)
+    return;
+
+  SMARTLIST_FOREACH(ls_list, link_specifier_t *, lspec,
+                    link_specifier_free(lspec));
+  smartlist_free(ls_list);
+}
+
 /** Return the nickname of <b>node</b>, or NULL if we can't find one. */
 const char *
 node_get_nickname(const node_t *node)
@@ -1327,8 +1462,7 @@ node_exit_policy_rejects_all(const node_t *node)
   if (node->ri)
     return node->ri->policy_is_reject_star;
   else if (node->md)
-    return node->md->exit_policy == NULL ||
-      short_policy_is_reject_star(node->md->exit_policy);
+    return node->md->policy_is_reject_star;
   else
     return 1;
 }
@@ -1503,19 +1637,6 @@ node_is_me(const node_t *node)
   return router_digest_is_me(node->identity);
 }
 
-/** Return <b>node</b> declared family (as a list of names), or NULL if
- * the node didn't declare a family. */
-const smartlist_t *
-node_get_declared_family(const node_t *node)
-{
-  if (node->ri && node->ri->declared_family)
-    return node->ri->declared_family;
-  else if (node->md && node->md->family)
-    return node->md->family;
-  else
-    return NULL;
-}
-
 /* Does this node have a valid IPv6 address?
  * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for
  * checking specific ports. */
@@ -1775,7 +1896,7 @@ microdesc_has_curve25519_onion_key(const microdesc_t *md)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -1855,7 +1976,7 @@ node_set_country(node_t *node)
 void
 nodelist_refresh_countries(void)
 {
-  smartlist_t *nodes = nodelist_get_list();
+  const smartlist_t *nodes = nodelist_get_list();
   SMARTLIST_FOREACH(nodes, node_t *, node,
                     node_set_country(node));
 }
@@ -1866,6 +1987,9 @@ int
 addrs_in_same_network_family(const tor_addr_t *a1,
                              const tor_addr_t *a2)
 {
+  if (tor_addr_is_null(a1) || tor_addr_is_null(a2))
+    return 0;
+
   switch (tor_addr_family(a1)) {
     case AF_INET:
       return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
@@ -1881,7 +2005,7 @@ addrs_in_same_network_family(const tor_addr_t *a1,
  * (case-insensitive), or if <b>node's</b> identity key digest
  * matches a hexadecimal value stored in <b>nickname</b>.  Return
  * false otherwise. */
-static int
+STATIC int
 node_nickname_matches(const node_t *node, const char *nickname)
 {
   const char *n = node_get_nickname(node);
@@ -1893,7 +2017,7 @@ node_nickname_matches(const node_t *node, const char *nickname)
 }
 
 /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
-static inline int
+STATIC int
 node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
 {
   if (!lst) return 0;
@@ -1904,6 +2028,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
   return 0;
 }
 
+/** Return true iff n1's declared family contains n2. */
+STATIC int
+node_family_contains(const node_t *n1, const node_t *n2)
+{
+  if (n1->ri && n1->ri->declared_family) {
+    return node_in_nickname_smartlist(n1->ri->declared_family, n2);
+  } else if (n1->md) {
+    return nodefamily_contains_node(n1->md->family, n2);
+  } else {
+    return 0;
+  }
+}
+
+/**
+ * Return true iff <b>node</b> has declared a nonempty family.
+ **/
+STATIC bool
+node_has_declared_family(const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    return true;
+  }
+
+  if (node->md && node->md->family) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Add to <b>out</b> every node_t that is listed by <b>node</b> as being in
+ * its family.  (Note that these nodes are not in node's family unless they
+ * also agree that node is in their family.)
+ **/
+STATIC void
+node_lookup_declared_family(smartlist_t *out, const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) {
+      const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED);
+      if (n2) {
+        smartlist_add(out, (node_t *)n2);
+      }
+    } SMARTLIST_FOREACH_END(name);
+    return;
+  }
+
+  if (node->md && node->md->family) {
+    nodefamily_add_nodes_to_smartlist(node->md->family, out);
+  }
+}
+
 /** Return true iff r1 and r2 are in the same family, but not the same
  * router. */
 int
@@ -1916,19 +2095,20 @@ nodes_in_same_family(const node_t *node1, const node_t *node2)
     tor_addr_t a1, a2;
     node_get_addr(node1, &a1);
     node_get_addr(node2, &a2);
-    if (addrs_in_same_network_family(&a1, &a2))
+
+    tor_addr_port_t ap6_1, ap6_2;
+    node_get_pref_ipv6_orport(node1, &ap6_1);
+    node_get_pref_ipv6_orport(node2, &ap6_2);
+
+    if (addrs_in_same_network_family(&a1, &a2) ||
+        addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr))
       return 1;
   }
 
   /* Are they in the same family because the agree they are? */
-  {
-    const smartlist_t *f1, *f2;
-    f1 = node_get_declared_family(node1);
-    f2 = node_get_declared_family(node2);
-    if (f1 && f2 &&
-        node_in_nickname_smartlist(f1, node2) &&
-        node_in_nickname_smartlist(f2, node1))
-      return 1;
+  if (node_family_contains(node1, node2) &&
+      node_family_contains(node2, node1)) {
+    return 1;
   }
 
   /* Are they in the same family because the user says they are? */
@@ -1956,13 +2136,10 @@ void
 nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
 {
   const smartlist_t *all_nodes = nodelist_get_list();
-  const smartlist_t *declared_family;
   const or_options_t *options = get_options();
 
   tor_assert(node);
 
-  declared_family = node_get_declared_family(node);
-
   /* Let's make sure that we have the node itself, if it's a real node. */
   {
     const node_t *real_node = node_get_by_id(node->identity);
@@ -1973,35 +2150,35 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
   /* First, add any nodes with similar network addresses. */
   if (options->EnforceDistinctSubnets) {
     tor_addr_t node_addr;
+    tor_addr_port_t node_ap6;
     node_get_addr(node, &node_addr);
+    node_get_pref_ipv6_orport(node, &node_ap6);
 
     SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) {
       tor_addr_t a;
+      tor_addr_port_t ap6;
       node_get_addr(node2, &a);
-      if (addrs_in_same_network_family(&a, &node_addr))
+      node_get_pref_ipv6_orport(node2, &ap6);
+      if (addrs_in_same_network_family(&a, &node_addr) ||
+          addrs_in_same_network_family(&ap6.addr, &node_ap6.addr))
         smartlist_add(sl, (void*)node2);
     } SMARTLIST_FOREACH_END(node2);
   }
 
-  /* Now, add all nodes in the declared_family of this node, if they
+  /* Now, add all nodes in the declared family of this node, if they
    * also declare this node to be in their family. */
-  if (declared_family) {
+  if (node_has_declared_family(node)) {
+    smartlist_t *declared_family = smartlist_new();
+    node_lookup_declared_family(declared_family, node);
+
     /* Add every r such that router declares familyness with node, and node
      * declares familyhood with router. */
-    SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
-      const node_t *node2;
-      const smartlist_t *family2;
-      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
-        continue;
-      if (!(family2 = node_get_declared_family(node2)))
-        continue;
-      SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) {
-          if (node_nickname_matches(node, name2)) {
-            smartlist_add(sl, (void*)node2);
-            break;
-          }
-      } SMARTLIST_FOREACH_END(name2);
-    } SMARTLIST_FOREACH_END(name);
+    SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) {
+      if (node_family_contains(node2, node)) {
+        smartlist_add(sl, (void*)node2);
+      }
+    } SMARTLIST_FOREACH_END(node2);
+    smartlist_free(declared_family);
   }
 
   /* If the user declared any families locally, honor those too. */
@@ -2306,7 +2483,7 @@ compute_frac_paths_available(const networkstatus_t *consensus,
   const int authdir = authdir_mode_v3(options);
 
   count_usable_descriptors(num_present_out, num_usable_out,
-                           mid, consensus, now, NULL,
+                           mid, consensus, now, options->MiddleNodes,
                            USABLE_DESCRIPTOR_ALL);
   log_debug(LD_NET,
             "%s: %d present, %d usable",
@@ -2508,7 +2685,7 @@ count_loading_descriptors_progress(void)
   if (fraction > 1.0)
     return 0; /* it's not the number of descriptors holding us back */
   return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int)
-    (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 -
+    (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 -
                BOOTSTRAP_STATUS_LOADING_DESCRIPTORS));
 }
 
@@ -2595,7 +2772,7 @@ update_router_have_minimum_dir_info(void)
   /* If paths have just become available in this update. */
   if (res && !have_min_dir_info) {
     control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO");
-    control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+    control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
     log_info(LD_DIR,
              "We now have enough directory information to build circuits.");
   }
diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h
index c430f497d5..87cfa48e25 100644
--- a/src/feature/nodelist/nodelist.h
+++ b/src/feature/nodelist/nodelist.h
@@ -35,6 +35,8 @@ node_t *nodelist_add_microdesc(microdesc_t *md);
 void nodelist_set_consensus(networkstatus_t *ns);
 void nodelist_ensure_freshness(networkstatus_t *ns);
 int nodelist_probably_contains_address(const tor_addr_t *addr);
+void nodelist_add_addr4_to_address_set(const uint32_t addr);
+void nodelist_add_addr6_to_address_set(const tor_addr_t *addr);
 
 void nodelist_remove_microdesc(const char *identity_digest, microdesc_t *md);
 void nodelist_remove_routerinfo(routerinfo_t *ri);
@@ -68,7 +70,6 @@ const char *node_get_platform(const node_t *node);
 uint32_t node_get_prim_addr_ipv4h(const node_t *node);
 void node_get_address_string(const node_t *node, char *cp, size_t len);
 long node_get_declared_uptime(const node_t *node);
-const smartlist_t *node_get_declared_family(const node_t *node);
 const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
 int node_ed25519_id_matches(const node_t *node,
                             const struct ed25519_public_key_t *id);
@@ -77,7 +78,13 @@ int node_supports_ed25519_link_authentication(const node_t *node,
 int node_supports_v3_hsdir(const node_t *node);
 int node_supports_ed25519_hs_intro(const node_t *node);
 int node_supports_v3_rendezvous_point(const node_t *node);
+int node_supports_establish_intro_dos_extension(const node_t *node);
 const uint8_t *node_get_rsa_id_digest(const node_t *node);
+smartlist_t *node_get_link_specifier_smartlist(const node_t *node,
+                                               bool direct_conn);
+void link_specifier_smartlist_free_(smartlist_t *ls_list);
+#define link_specifier_smartlist_free(ls_list) \
+  FREE_AND_NULL(smartlist_t, link_specifier_smartlist_free_, (ls_list))
 
 int node_has_ipv6_addr(const node_t *node);
 int node_has_ipv6_orport(const node_t *node);
@@ -97,7 +104,7 @@ const struct curve25519_public_key_t *node_get_curve25519_onion_key(
                                   const node_t *node);
 crypto_pk_t *node_get_rsa_onion_key(const node_t *node);
 
-MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
+MOCK_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 /* Temporary during transition to multiple addresses.  */
 void node_get_addr(const node_t *node, tor_addr_t *addr_out);
@@ -155,10 +162,16 @@ int count_loading_descriptors_progress(void);
 
 #ifdef NODELIST_PRIVATE
 
+STATIC int node_nickname_matches(const node_t *node, const char *nickname);
+STATIC int node_in_nickname_smartlist(const smartlist_t *lst,
+                                      const node_t *node);
+STATIC int node_family_contains(const node_t *n1, const node_t *n2);
+STATIC bool node_has_declared_family(const node_t *node);
+STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node);
+
 #ifdef TOR_UNIT_TESTS
 
-STATIC void
-node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h
index bfa28c7754..8465060f93 100644
--- a/src/feature/nodelist/routerinfo.h
+++ b/src/feature/nodelist/routerinfo.h
@@ -17,11 +17,9 @@ void router_get_prim_orport(const routerinfo_t *router,
 int router_has_orport(const routerinfo_t *router,
                       const tor_addr_port_t *orport);
 
-void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
-
 smartlist_t *router_get_all_orports(const routerinfo_t *ri);
 
 const char *router_purpose_to_string(uint8_t p);
 uint8_t router_purpose_from_string(const char *s);
 
-#endif
+#endif /* !defined(TOR_ROUTERINFO_H) */
diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h
index 59656818c1..59fd56d0a0 100644
--- a/src/feature/nodelist/routerinfo_st.h
+++ b/src/feature/nodelist/routerinfo_st.h
@@ -112,4 +112,4 @@ struct routerinfo_t {
   uint8_t purpose;
 };
 
-#endif
+#endif /* !defined(ROUTERINFO_ST_H) */
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
index c7fa868929..b3cf3e2394 100644
--- a/src/feature/nodelist/routerlist.c
+++ b/src/feature/nodelist/routerlist.c
@@ -67,7 +67,7 @@
 #include "core/mainloop/mainloop.h"
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dirauth/reachability.h"
@@ -160,7 +160,7 @@ static time_t last_descriptor_download_attempted = 0;
  *
  * From time to time, we replace "cached-descriptors" with a new file
  * containing only the live, non-superseded descriptors, and clear
- * cached-routers.new.
+ * cached-descriptors.new.
  *
  * On startup, we read both files.
  */
@@ -1463,12 +1463,13 @@ router_descriptor_is_older_than,(const routerinfo_t *router, int seconds))
 }
 
 /** Add <b>router</b> to the routerlist, if we don't already have it.  Replace
- * older entries (if any) with the same key.  Note: Callers should not hold
- * their pointers to <b>router</b> if this function fails; <b>router</b>
- * will either be inserted into the routerlist or freed. Similarly, even
- * if this call succeeds, they should not hold their pointers to
- * <b>router</b> after subsequent calls with other routerinfo's -- they
- * might cause the original routerinfo to get freed.
+ * older entries (if any) with the same key.
+ *
+ * Note: Callers should not hold their pointers to <b>router</b> if this
+ * function fails; <b>router</b> will either be inserted into the routerlist or
+ * freed. Similarly, even if this call succeeds, they should not hold their
+ * pointers to <b>router</b> after subsequent calls with other routerinfo's --
+ * they might cause the original routerinfo to get freed.
  *
  * Returns the status for the operation. Might set *<b>msg</b> if it wants
  * the poster of the router to know something.
@@ -1930,6 +1931,8 @@ routerlist_remove_old_routers(void)
 void
 routerlist_descriptors_added(smartlist_t *sl, int from_cache)
 {
+  // XXXX use pubsub mechanism here.
+
   tor_assert(sl);
   control_event_descriptors_changed(sl);
   SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) {
@@ -1937,7 +1940,9 @@ routerlist_descriptors_added(smartlist_t *sl, int from_cache)
       learned_bridge_descriptor(ri, from_cache);
     if (ri->needs_retest_if_added) {
       ri->needs_retest_if_added = 0;
+#ifdef HAVE_MODULE_DIRAUTH
       dirserv_single_reachability_test(approx_time(), ri);
+#endif
     }
   } SMARTLIST_FOREACH_END(ri);
 }
@@ -2975,7 +2980,7 @@ routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey,
   digest256_matches = tor_memeq(ei->digest256,
                                 sd->extra_info_digest256, DIGEST256_LEN);
   digest256_matches |=
-    tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
+    fast_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
 
   /* The identity must match exactly to have been generated at the same time
    * by the same router. */
@@ -3059,7 +3064,7 @@ routerinfo_has_curve25519_onion_key(const routerinfo_t *ri)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -3227,6 +3232,8 @@ refresh_all_country_info(void)
     routerset_refresh_countries(options->EntryNodes);
   if (options->ExitNodes)
     routerset_refresh_countries(options->ExitNodes);
+  if (options->MiddleNodes)
+    routerset_refresh_countries(options->MiddleNodes);
   if (options->ExcludeNodes)
     routerset_refresh_countries(options->ExcludeNodes);
   if (options->ExcludeExitNodes)
diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h
index 5771ebb1ab..dc9203e015 100644
--- a/src/feature/nodelist/routerlist.h
+++ b/src/feature/nodelist/routerlist.h
@@ -37,9 +37,12 @@ typedef enum was_router_added_t {
   ROUTER_WAS_NOT_WANTED = -6,
   /* Router descriptor was rejected because it was older than
    * OLD_ROUTER_DESC_MAX_AGE. */
-  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
-  /* DOCDOC */
-  ROUTER_CERTS_EXPIRED = -8
+  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'ROUTER_IS_ALREADY_KNOWN' */
+  /* Some certs on this router are expired. */
+  ROUTER_CERTS_EXPIRED = -8,
+  /* We couldn't format the annotations for this router. This is a directory
+   * authority bug. */
+  ROUTER_AUTHDIR_BUG_ANNOTATIONS = -10
 } was_router_added_t;
 
 /** How long do we avoid using a directory server after it's given us a 503? */
diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h
index 7446ead3cb..10b919a1bf 100644
--- a/src/feature/nodelist/routerlist_st.h
+++ b/src/feature/nodelist/routerlist_st.h
@@ -36,5 +36,5 @@ struct routerlist_t {
   desc_store_t extrainfo_store;
 };
 
-#endif
+#endif /* !defined(ROUTERLIST_ST_H) */
 
diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c
index 55e2756959..9a205d39b7 100644
--- a/src/feature/nodelist/routerset.c
+++ b/src/feature/nodelist/routerset.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2001 Matej Pfajfar.
-n * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
@@ -34,6 +34,9 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/nickname.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
 #include "lib/geoip/geoip.h"
 
 #include "core/or/addr_policy_st.h"
@@ -41,6 +44,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
+#include "lib/confmgt/var_type_def_st.h"
 
 /** Return a new empty routerset. */
 routerset_t *
@@ -378,7 +382,7 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
   } else {
     /* We need to iterate over the routerlist to get all the ones of the
      * right kind. */
-    smartlist_t *nodes = nodelist_get_list();
+    const smartlist_t *nodes = nodelist_get_list();
     SMARTLIST_FOREACH(nodes, const node_t *, node, {
         if (running_only && !node->is_running)
           continue;
@@ -461,3 +465,111 @@ routerset_free_(routerset_t *routerset)
   bitarray_free(routerset->countries);
   tor_free(routerset);
 }
+
+/**
+ * config helper: parse a routerset-typed variable.
+ *
+ * Takes as input as a single line in <b>line</b>; writes its results into a
+ * routerset_t** passed as <b>target</b>.  On success return 0; on failure
+ * return -1 and store an error message into *<b>errmsg</b>.
+ **/
+/*
+ * Warning: For this type, the default value (NULL) and "" are sometimes
+ * considered different values.  That is generally risky, and best avoided for
+ * other types in the future.  For cases where we want the default to be "all
+ * routers" (like EntryNodes) we should add a new routerset value indicating
+ * "all routers" (see #31908)
+ */
+static int
+routerset_kv_parse(void *target, const config_line_t *line, char **errmsg,
+                  const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)target;
+  routerset_free(*p); // clear the old value, if any.
+  routerset_t *rs = routerset_new();
+  if (routerset_parse(rs, line->value, line->key) < 0) {
+    routerset_free(rs);
+    *errmsg = tor_strdup("Invalid router list.");
+    return -1;
+  } else {
+    if (routerset_is_empty(rs)) {
+      /* Represent empty sets as NULL. */
+      routerset_free(rs);
+    }
+    *p = rs;
+    return 0;
+  }
+}
+
+/**
+ * config helper: encode a routerset-typed variable.
+ *
+ * Return a newly allocated string containing the value of the
+ * routerset_t** passed as <b>value</b>.
+ */
+static char *
+routerset_encode(const void *value, const void *params)
+{
+  (void)params;
+  const routerset_t **p = (const routerset_t**)value;
+  return routerset_to_string(*p);
+}
+
+/**
+ * config helper: free and clear a routerset-typed variable.
+ *
+ * Clear the routerset_t** passed as <b>value</b>.
+ */
+static void
+routerset_clear(void *value, const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)value;
+  routerset_free(*p); // sets *p to NULL.
+}
+
+/**
+ * config helper: copy a routerset-typed variable.
+ *
+ * Takes it input from a routerset_t** in <b>src</b>; writes its output to a
+ * routerset_t** in <b>dest</b>.  Returns 0 on success, -1 on (impossible)
+ * failure.
+ **/
+static int
+routerset_copy(void *dest, const void *src, const void *params)
+{
+  (void)params;
+  routerset_t **output = (routerset_t**)dest;
+  const routerset_t *input = *(routerset_t**)src;
+  routerset_free(*output); // sets *output to NULL
+  if (! routerset_is_empty(input)) {
+    *output = routerset_new();
+    routerset_union(*output, input);
+  }
+  return 0;
+}
+
+/**
+ * Function table to implement a routerset_t-based configuration type.
+ **/
+static const var_type_fns_t routerset_type_fns = {
+  .kv_parse = routerset_kv_parse,
+  .encode = routerset_encode,
+  .clear = routerset_clear,
+  .copy = routerset_copy
+};
+
+/**
+ * Definition of a routerset_t-based configuration type.
+ *
+ * Values are mapped to and from strings using the format defined in
+ * routerset_parse(): nicknames, IP address patterns, and fingerprints--with
+ * optional space, separated by commas.
+ *
+ * Empty sets are represented as NULL.
+ **/
+const var_type_def_t ROUTERSET_type_defn = {
+  .name = "RouterList",
+  .fns = &routerset_type_fns
+};
diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h
index ca8b6fed93..f3bf4a1f7c 100644
--- a/src/feature/nodelist/routerset.h
+++ b/src/feature/nodelist/routerset.h
@@ -44,6 +44,9 @@ void routerset_free_(routerset_t *routerset);
 #define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
 int routerset_len(const routerset_t *set);
 
+struct var_type_def_t;
+extern const struct var_type_def_t ROUTERSET_type_defn;
+
 #ifdef ROUTERSET_PRIVATE
 #include "lib/container/bitarray.h"
 
diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h
index 288edf5982..46337c9e52 100644
--- a/src/feature/nodelist/routerstatus_st.h
+++ b/src/feature/nodelist/routerstatus_st.h
@@ -47,6 +47,8 @@ struct routerstatus_t {
   unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
                              * or it claims to accept tunnelled dir requests.
                              */
+  unsigned int is_staledesc:1; /** True iff the authorities think this router
+                                * should upload a new descriptor soon. */
 
   unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
   unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
@@ -76,5 +78,5 @@ struct routerstatus_t {
 
 };
 
-#endif
+#endif /* !defined(ROUTERSTATUS_ST_H) */
 
diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h
index bdcebf184a..64c28f7440 100644
--- a/src/feature/nodelist/signed_descriptor_st.h
+++ b/src/feature/nodelist/signed_descriptor_st.h
@@ -57,5 +57,5 @@ struct signed_descriptor_t {
   unsigned int send_unencrypted : 1;
 };
 
-#endif
+#endif /* !defined(SIGNED_DESCRIPTOR_ST_H) */
 
diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c
index b0197e9f13..270c14eb1c 100644
--- a/src/feature/nodelist/torcert.c
+++ b/src/feature/nodelist/torcert.c
@@ -74,7 +74,7 @@ tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
   tor_assert(real_len == alloc_len);
   tor_assert(real_len > ED25519_SIG_LEN);
   uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
-  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
+  tor_assert(fast_mem_is_zero((char*)sig, ED25519_SIG_LEN));
 
   ed25519_signature_t signature;
   if (ed25519_sign(&signature, encoded,
@@ -290,8 +290,8 @@ tor_cert_describe_signature_status(const tor_cert_t *cert)
 }
 
 /** Return a new copy of <b>cert</b> */
-tor_cert_t *
-tor_cert_dup(const tor_cert_t *cert)
+MOCK_IMPL(tor_cert_t *,
+tor_cert_dup,(const tor_cert_t *cert))
 {
   tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
   if (cert->encoded)
diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h
index 492275b514..03d5bdca93 100644
--- a/src/feature/nodelist/torcert.h
+++ b/src/feature/nodelist/torcert.h
@@ -71,7 +71,7 @@ int tor_cert_checksig(tor_cert_t *cert,
                       const ed25519_public_key_t *pubkey, time_t now);
 const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
 
-tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
+MOCK_DECL(tor_cert_t *,tor_cert_dup,(const tor_cert_t *cert));
 int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 
diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h
index 366754c166..0d909da260 100644
--- a/src/feature/nodelist/vote_routerstatus_st.h
+++ b/src/feature/nodelist/vote_routerstatus_st.h
@@ -38,4 +38,4 @@ struct vote_routerstatus_t {
   uint8_t ed25519_id[ED25519_PUBKEY_LEN];
 };
 
-#endif
+#endif /* !defined(VOTE_ROUTERSTATUS_ST_H) */
diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c
index e20a39482f..d62598d46f 100644
--- a/src/feature/relay/dns.c
+++ b/src/feature/relay/dns.c
@@ -59,7 +59,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
@@ -1360,6 +1360,42 @@ evdns_err_is_transient(int err)
   }
 }
 
+/**
+ * Return number of configured nameservers in <b>the_evdns_base</b>.
+ */
+size_t
+number_of_configured_nameservers(void)
+{
+  return evdns_base_count_nameservers(the_evdns_base);
+}
+
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+/**
+ * Return address of configured nameserver in <b>the_evdns_base</b>
+ * at index <b>idx</b>.
+ */
+tor_addr_t *
+configured_nameserver_address(const size_t idx)
+{
+ struct sockaddr_storage sa;
+ ev_socklen_t sa_len = sizeof(sa);
+
+ if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx,
+                                    (struct sockaddr *)&sa,
+                                    sa_len) > 0) {
+   tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t));
+   if (tor_addr_from_sockaddr(tor_addr,
+                              (const struct sockaddr *)&sa,
+                              NULL) == 0) {
+     return tor_addr;
+   }
+   tor_free(tor_addr);
+ }
+
+ return NULL;
+}
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
 /** Configure eventdns nameservers if force is true, or if the configuration
  * has changed since the last time we called this function, or if we failed on
  * our last attempt.  On Unix, this reads from /etc/resolv.conf or
@@ -1391,16 +1427,23 @@ configure_nameservers(int force)
   evdns_set_log_fn(evdns_log_cb);
   if (conf_fname) {
     log_debug(LD_FS, "stat()ing %s", conf_fname);
-    if (stat(sandbox_intern_string(conf_fname), &st)) {
+    int missing_resolv_conf = 0;
+    int stat_res = stat(sandbox_intern_string(conf_fname), &st);
+
+    if (stat_res) {
       log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s",
                conf_fname, strerror(errno));
-      goto err;
-    }
-    if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname)
+      missing_resolv_conf = 1;
+    } else if (!force && resolv_conf_fname &&
+               !strcmp(conf_fname,resolv_conf_fname)
         && st.st_mtime == resolv_conf_mtime) {
       log_info(LD_EXIT, "No change to '%s'", conf_fname);
       return 0;
     }
+
+    if (stat_res == 0 && st.st_size == 0)
+      missing_resolv_conf = 1;
+
     if (nameservers_configured) {
       evdns_base_search_clear(the_evdns_base);
       evdns_base_clear_nameservers_and_suspend(the_evdns_base);
@@ -1413,20 +1456,34 @@ configure_nameservers(int force)
           sandbox_intern_string("/etc/hosts"));
     }
 #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */
-    log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
-    if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
-        sandbox_intern_string(conf_fname)))) {
-      log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)",
-               conf_fname, conf_fname, r);
-      goto err;
-    }
-    if (evdns_base_count_nameservers(the_evdns_base) == 0) {
-      log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname);
-      goto err;
+
+    if (!missing_resolv_conf) {
+      log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
+      if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
+          sandbox_intern_string(conf_fname)))) {
+        log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers "
+                          "in '%s' (%d)", conf_fname, conf_fname, r);
+
+        if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" -
+          goto err; // in which case we expect libevent to add 127.0.0.1 as
+                    // fallback.
+      }
+      if (evdns_base_count_nameservers(the_evdns_base) == 0) {
+        log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.",
+                 conf_fname);
+      }
+
+      tor_free(resolv_conf_fname);
+      resolv_conf_fname = tor_strdup(conf_fname);
+      resolv_conf_mtime = st.st_mtime;
+    } else {
+      log_warn(LD_EXIT, "Could not read your DNS config from '%s' - "
+                        "please investigate your DNS configuration. "
+                        "This is possibly a problem. Meanwhile, falling"
+                        " back to local DNS at 127.0.0.1.", conf_fname);
+      evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1");
     }
-    tor_free(resolv_conf_fname);
-    resolv_conf_fname = tor_strdup(conf_fname);
-    resolv_conf_mtime = st.st_mtime;
+
     if (nameservers_configured)
       evdns_base_resume(the_evdns_base);
   }
diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h
index e4474cdf43..7b2a31a311 100644
--- a/src/feature/relay/dns.h
+++ b/src/feature/relay/dns.h
@@ -45,6 +45,11 @@ size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes);
 #ifdef DNS_PRIVATE
 #include "feature/relay/dns_structs.h"
 
+size_t number_of_configured_nameservers(void);
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+tor_addr_t *configured_nameserver_address(const size_t idx);
+#endif
+
 MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn,
 int is_resolve,or_circuit_t *oncirc, char **hostname_out,
 int *made_connection_pending_out, cached_resolve_t **resolve_out));
diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c
index 56c5bb96f5..c343d19b8d 100644
--- a/src/feature/relay/ext_orport.c
+++ b/src/feature/relay/ext_orport.c
@@ -20,7 +20,7 @@
 #include "core/or/or.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn)
 
   conn->base_.type = CONN_TYPE_OR;
   TO_CONN(conn)->state = 0; // set the state to a neutral value
-  control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0);
   connection_tls_start_handshake(conn, 1);
 }
 
@@ -659,4 +659,3 @@ ext_orport_free_all(void)
   if (ext_or_auth_cookie) /* Free the auth cookie */
     tor_free(ext_or_auth_cookie);
 }
-
diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c
index 696905cf5e..c37745cf33 100644
--- a/src/feature/relay/onion_queue.c
+++ b/src/feature/relay/onion_queue.c
@@ -212,10 +212,12 @@ num_ntors_per_tap(void)
 #define MIN_NUM_NTORS_PER_TAP 1
 #define MAX_NUM_NTORS_PER_TAP 100000
 
-  return networkstatus_get_param(NULL, "NumNTorsPerTAP",
-                                 DEFAULT_NUM_NTORS_PER_TAP,
-                                 MIN_NUM_NTORS_PER_TAP,
-                                 MAX_NUM_NTORS_PER_TAP);
+  int result = networkstatus_get_param(NULL, "NumNTorsPerTAP",
+                                       DEFAULT_NUM_NTORS_PER_TAP,
+                                       MIN_NUM_NTORS_PER_TAP,
+                                       MAX_NUM_NTORS_PER_TAP);
+  tor_assert(result > 0);
+  return result;
 }
 
 /** Choose which onion queue we'll pull from next. If one is empty choose
diff --git a/src/feature/relay/onion_queue.h b/src/feature/relay/onion_queue.h
index 0df921e057..cf478bc1a0 100644
--- a/src/feature/relay/onion_queue.h
+++ b/src/feature/relay/onion_queue.h
@@ -20,4 +20,4 @@ int onion_num_pending(uint16_t handshake_type);
 void onion_pending_remove(or_circuit_t *circ);
 void clear_pending_onions(void);
 
-#endif
+#endif /* !defined(TOR_ONION_QUEUE_H) */
diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c
new file mode 100644
index 0000000000..b48b495895
--- /dev/null
+++ b/src/feature/relay/relay_periodic.c
@@ -0,0 +1,308 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.c
+ * @brief Periodic functions for the relay subsytem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+#include "core/mainloop/cpuworker.h" // XXXX use a pubsub event.
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/circuituse.h" // XXXX move have_performed_bandwidth_test
+
+#include "feature/relay/dns.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/stats/predict_ports.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/control/control_events.h"
+
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: If we're a server and initializing dns failed, retry.
+ */
+static int
+retry_dns_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+#define RETRY_DNS_INTERVAL (10*60)
+  if (server_mode(options) && has_dns_init_failed())
+    dns_init();
+  return RETRY_DNS_INTERVAL;
+}
+
+DECLARE_EVENT(retry_dns, ROUTER, 0);
+
+static int dns_honesty_first_time = 1;
+
+/**
+ * Periodic event: if we're an exit, see if our DNS server is telling us
+ * obvious lies.
+ */
+static int
+check_dns_honesty_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  /* 9. and if we're an exit node, check whether our DNS is telling stories
+   * to us. */
+  if (net_is_disabled() ||
+      ! public_server_mode(options) ||
+      router_my_exit_policy_is_reject_star())
+    return PERIODIC_EVENT_NO_UPDATE;
+
+  if (dns_honesty_first_time) {
+    /* Don't launch right when we start */
+    dns_honesty_first_time = 0;
+    return crypto_rand_int_range(60, 180);
+  }
+
+  dns_launch_correctness_checks();
+  return 12*3600 + crypto_rand_int(12*3600);
+}
+
+DECLARE_EVENT(check_dns_honesty, RELAY, FL(NEED_NET));
+
+/* Periodic callback: rotate the onion keys after the period defined by the
+ * "onion-key-rotation-days" consensus parameter, shut down and restart all
+ * cpuworkers, and update our descriptor if necessary.
+ */
+static int
+rotate_onion_key_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_lifetime = get_onion_key_lifetime();
+    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
+    if (rotation_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL,"Rotating onion key.");
+    rotate_onion_key();
+    cpuworkers_rotate_keyinfo();
+    if (router_rebuild_descriptor(1)<0) {
+      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
+    }
+    if (advertised_server_mode() && !net_is_disabled())
+      router_upload_dir_desc_to_dirservers(0);
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(rotate_onion_key, ROUTER, 0);
+
+/** Periodic callback: consider rebuilding or and re-uploading our descriptor
+ * (if we've passed our internal checks). */
+static int
+check_descriptor_callback(time_t now, const or_options_t *options)
+{
+/** How often do we check whether part of our router info has changed in a
+ * way that would require an upload? That includes checking whether our IP
+ * address has changed. */
+#define CHECK_DESCRIPTOR_INTERVAL (60)
+
+  (void)options;
+
+  /* 2b. Once per minute, regenerate and upload the descriptor if the old
+   * one is inaccurate. */
+  if (!net_is_disabled()) {
+    check_descriptor_bandwidth_changed(now);
+    check_descriptor_ipaddress_changed(now);
+    mark_my_descriptor_dirty_if_too_old(now);
+    consider_publishable_server(0);
+  }
+
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_descriptor, ROUTER, FL(NEED_NET));
+
+static int dirport_reachability_count = 0;
+
+/**
+ * Periodic callback: check whether we're reachable (as a relay), and
+ * whether our bandwidth has changed enough that we need to
+ * publish a new descriptor.
+ */
+static int
+check_for_reachability_bw_callback(time_t now, const or_options_t *options)
+{
+  /* XXXX This whole thing was stuck in the middle of what is now
+   * XXXX check_descriptor_callback.  I'm not sure it's right. */
+
+  /* also, check religiously for reachability, if it's within the first
+   * 20 minutes of our uptime. */
+  if (server_mode(options) &&
+      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
+      !net_is_disabled()) {
+    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+      router_do_reachability_checks(1, dirport_reachability_count==0);
+      if (++dirport_reachability_count > 5)
+        dirport_reachability_count = 0;
+      return 1;
+    } else {
+      /* If we haven't checked for 12 hours and our bandwidth estimate is
+       * low, do another bandwidth test. This is especially important for
+       * bridges, since they might go long periods without much use. */
+      const routerinfo_t *me = router_get_my_routerinfo();
+      static int first_time = 1;
+      if (!first_time && me &&
+          me->bandwidthcapacity < me->bandwidthrate &&
+          me->bandwidthcapacity < 51200) {
+        reset_bandwidth_test();
+      }
+      first_time = 0;
+#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
+      return BANDWIDTH_RECHECK_INTERVAL;
+    }
+  }
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_for_reachability_bw, ROUTER, FL(NEED_NET));
+
+/**
+ * Callback: Send warnings if Tor doesn't find its ports reachable.
+ */
+static int
+reachability_warnings_callback(time_t now, const or_options_t *options)
+{
+  (void) now;
+
+  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
+  }
+
+  if (server_mode(options) &&
+      !net_is_disabled() &&
+      have_completed_a_circuit()) {
+    /* every 20 minutes, check and complain if necessary */
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me && !check_whether_orport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
+               "its ORPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->or_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
+                                  address, me->or_port);
+      tor_free(address);
+    }
+
+    if (me && !check_whether_dirport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      log_warn(LD_CONFIG,
+               "Your server (%s:%d) has not managed to confirm that its "
+               "DirPort is reachable. Relays do not publish descriptors "
+               "until their ORPort and DirPort are reachable. Please check "
+               "your firewalls, ports, address, /etc/hosts file, etc.",
+               address, me->dir_port);
+      control_event_server_status(LOG_WARN,
+                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
+                                  address, me->dir_port);
+      tor_free(address);
+    }
+  }
+
+  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
+}
+
+DECLARE_EVENT(reachability_warnings, ROUTER, FL(NEED_NET));
+
+/* Periodic callback: Every 30 seconds, check whether it's time to make new
+ * Ed25519 subkeys.
+ */
+static int
+check_ed_keys_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    if (should_make_new_ed_keys(options, now)) {
+      int new_signing_key = load_ed_keys(options, now);
+      if (new_signing_key < 0 ||
+          generate_ed_link_cert(options, now, new_signing_key > 0)) {
+        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
+        tor_shutdown_event_loop_and_exit(1);
+      }
+    }
+    return 30;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_ed_keys, ROUTER, 0);
+
+/* Period callback: Check if our old onion keys are still valid after the
+ * period of time defined by the consensus parameter
+ * "onion-key-grace-period-days", otherwise expire them by setting them to
+ * NULL.
+ */
+static int
+check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_grace_period = get_onion_key_grace_period();
+    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
+    if (expiry_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL, "Expiring old onion keys.");
+    expire_old_onion_keys();
+    cpuworkers_rotate_keyinfo();
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_onion_keys_expiry_time, ROUTER, 0);
+
+void
+relay_register_periodic_events(void)
+{
+  periodic_events_register(&retry_dns_event);
+  periodic_events_register(&check_dns_honesty_event);
+  periodic_events_register(&rotate_onion_key_event);
+  periodic_events_register(&check_descriptor_event);
+  periodic_events_register(&check_for_reachability_bw_event);
+  periodic_events_register(&reachability_warnings_event);
+  periodic_events_register(&check_ed_keys_event);
+  periodic_events_register(&check_onion_keys_expiry_time_event);
+
+  dns_honesty_first_time = 1;
+  dirport_reachability_count = 0;
+}
+
+/**
+ * Update our schedule so that we'll check whether we need to update our
+ * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
+ * seconds.
+ */
+void
+reschedule_descriptor_update_check(void)
+{
+  periodic_event_reschedule(&check_descriptor_event);
+}
diff --git a/src/feature/relay/relay_periodic.h b/src/feature/relay/relay_periodic.h
new file mode 100644
index 0000000000..b6ea83c749
--- /dev/null
+++ b/src/feature/relay/relay_periodic.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.h
+ * @brief Header for feature/relay/relay_periodic.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+#define TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+
+void relay_register_periodic_events(void);
+void reschedule_descriptor_update_check(void);
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_PERIODIC_H) */
diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c
new file mode 100644
index 0000000000..106e88b2a5
--- /dev/null
+++ b/src/feature/relay/relay_sys.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.c
+ * @brief Subsystem definitions for the relay module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/onion_queue.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/relay_sys.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/router.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_relay_initialize(void)
+{
+  relay_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_relay_shutdown(void)
+{
+  dns_free_all();
+  ext_orport_free_all();
+  clear_pending_onions();
+  routerkeys_free_all();
+  router_free_all();
+}
+
+const struct subsys_fns_t sys_relay = {
+  .name = "relay",
+  .supported = true,
+  .level = 50,
+  .initialize = subsys_relay_initialize,
+  .shutdown = subsys_relay_shutdown,
+};
diff --git a/src/feature/relay/relay_sys.h b/src/feature/relay/relay_sys.h
new file mode 100644
index 0000000000..32e21d90d8
--- /dev/null
+++ b/src/feature/relay/relay_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.h
+ * @brief Header for feature/relay/relay_sys.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_SYS_H
+#define TOR_FEATURE_RELAY_RELAY_SYS_H
+
+extern const struct subsys_fns_t sys_relay;
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_SYS_H) */
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index e91550a78c..a46b522bd6 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -16,7 +16,7 @@
 #include "core/or/policies.h"
 #include "core/or/protover.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -30,6 +30,7 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
@@ -49,6 +50,7 @@
 #include "lib/encoding/confline.h"
 #include "lib/osinfo/uname.h"
 #include "lib/tls/tortls.h"
+#include "lib/version/torversion.h"
 
 #include "feature/dirauth/authmode.h"
 
@@ -58,6 +60,7 @@
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
@@ -149,6 +152,8 @@ routerinfo_err_to_string(int err)
       return "Cannot generate descriptor";
     case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
       return "Descriptor still rebuilding - not ready yet";
+    case TOR_ROUTERINFO_ERROR_INTERNAL_BUG:
+      return "Internal bug, see logs for details";
   }
 
   log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
@@ -191,8 +196,8 @@ set_onion_key(crypto_pk_t *k)
 
 /** Return the current onion key.  Requires that the onion key has been
  * loaded or generated. */
-crypto_pk_t *
-get_onion_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_onion_key,(void))
 {
   tor_assert(onionkey);
   return onionkey;
@@ -239,7 +244,7 @@ expire_old_onion_keys(void)
     lastonionkey = NULL;
   }
 
-  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+  /* We zero out the keypair. See the fast_mem_is_zero() check made in
    * construct_ntor_key_map() below. */
   memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
@@ -266,11 +271,12 @@ expire_old_onion_keys(void)
 
 /** Return the current secret onion key for the ntor handshake. Must only
  * be called from the main thread. */
-static const curve25519_keypair_t *
-get_current_curve25519_keypair(void)
+MOCK_IMPL(STATIC const struct curve25519_keypair_t *,
+get_current_curve25519_keypair,(void))
 {
   return &curve25519_onion_key;
 }
+
 /** Return a map from KEYID (the key itself) to keypairs for use in the ntor
  * handshake. Must only be called from the main thread. */
 di_digest256_map_t *
@@ -281,12 +287,12 @@ construct_ntor_key_map(void)
   const uint8_t *cur_pk = curve25519_onion_key.pubkey.public_key;
   const uint8_t *last_pk = last_curve25519_onion_key.pubkey.public_key;
 
-  if (!tor_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
+  if (!fast_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, cur_pk,
                     tor_memdup(&curve25519_onion_key,
                                sizeof(curve25519_keypair_t)));
   }
-  if (!tor_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
+  if (!fast_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
       tor_memneq(cur_pk, last_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, last_pk,
                     tor_memdup(&last_curve25519_onion_key,
@@ -337,6 +343,16 @@ set_server_identity_key(crypto_pk_t *k)
   }
 }
 
+#ifdef TOR_UNIT_TESTS
+/** Testing only -- set the server's RSA identity digest to
+ * be <b>digest</b> */
+void
+set_server_identity_key_digest_testing(const uint8_t *digest)
+{
+  memcpy(server_identitykey_digest, digest, DIGEST_LEN);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Make sure that we have set up our identity keys to match or not match as
  * appropriate, and die with an assertion if we have not. */
 static void
@@ -359,8 +375,8 @@ assert_identity_keys_ok(void)
 /** Returns the current server identity key; requires that the key has
  * been set, and that we are running as a Tor server.
  */
-crypto_pk_t *
-get_server_identity_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_server_identity_key,(void))
 {
   tor_assert(server_identitykey);
   tor_assert(server_mode(get_options()));
@@ -634,7 +650,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out,
                fname);
     goto done;
   }
-  parsed = authority_cert_parse_from_string(cert, &eos);
+  parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos);
   if (!parsed) {
     log_warn(LD_DIR, "Unable to parse certificate in %s", fname);
     goto done;
@@ -1031,7 +1047,7 @@ init_keys(void)
       return -1;
 
     keydir = get_keydir_fname("secret_onion_key_ntor.old");
-    if (tor_mem_is_zero((const char *)
+    if (fast_mem_is_zero((const char *)
                            last_curve25519_onion_key.pubkey.public_key,
                         CURVE25519_PUBKEY_LEN) &&
         file_status(keydir) == FN_FILE) {
@@ -1467,9 +1483,9 @@ static extrainfo_t *desc_extrainfo = NULL;
 static const char *desc_gen_reason = "uninitialized reason";
 /** Since when has our descriptor been "clean"?  0 if we need to regenerate it
  * now. */
-static time_t desc_clean_since = 0;
+STATIC time_t desc_clean_since = 0;
 /** Why did we mark the descriptor dirty? */
-static const char *desc_dirty_reason = "Tor just started";
+STATIC const char *desc_dirty_reason = "Tor just started";
 /** Boolean: do we need to regenerate the above? */
 static int desc_needs_upload = 0;
 
@@ -1688,10 +1704,6 @@ router_get_descriptor_gen_reason(void)
   return desc_gen_reason;
 }
 
-/** A list of nicknames that we've warned about including in our family
- * declaration verbatim rather than as digests. */
-static smartlist_t *warned_nonexistent_family = NULL;
-
 static int router_guess_address_from_dir_headers(uint32_t *guess);
 
 /** Make a current best guess at our address, either because
@@ -1804,26 +1816,159 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr)
                                                    CONN_TYPE_DIR_LISTENER);
 }
 
-/** Build a fresh routerinfo, signed server descriptor, and extra-info document
- * for this OR. Set r to the generated routerinfo, e to the generated
- * extra-info document. Return 0 on success, -1 on temporary error. Failure to
- * generate an extra-info document is not an error and is indicated by setting
- * e to NULL. Caller is responsible for freeing generated documents if 0 is
- * returned.
+/** A list of nicknames that we've warned about including in our family,
+ * for one reason or another. */
+static smartlist_t *warned_family = NULL;
+
+/**
+ * Return a new smartlist containing the family members configured in
+ * <b>options</b>.  Warn about invalid or missing entries.  Return NULL
+ * if this relay should not declare a family.
+ **/
+STATIC smartlist_t *
+get_my_declared_family(const or_options_t *options)
+{
+  if (!options->MyFamily)
+    return NULL;
+
+  if (options->BridgeRelay)
+    return NULL;
+
+  if (!warned_family)
+    warned_family = smartlist_new();
+
+  smartlist_t *declared_family = smartlist_new();
+  config_line_t *family;
+
+  /* First we try to get the whole family in the form of hexdigests. */
+  for (family = options->MyFamily; family; family = family->next) {
+    char *name = family->value;
+    const node_t *member;
+    if (options->Nickname && !strcasecmp(name, options->Nickname))
+      continue; /* Don't list ourself by nickname, that's redundant */
+    else
+      member = node_get_by_nickname(name, 0);
+
+    if (!member) {
+      /* This node doesn't seem to exist, so warn about it if it is not
+       * a hexdigest. */
+      int is_legal = is_legal_nickname_or_hexdigest(name);
+      if (!smartlist_contains_string(warned_family, name) &&
+          !is_legal_hexdigest(name)) {
+        if (is_legal)
+          log_warn(LD_CONFIG,
+                   "There is a router named %s in my declared family, but "
+                   "I have no descriptor for it. I'll use the nickname "
+                   "as is, but this may confuse clients. Please list it "
+                   "by identity digest instead.", escaped(name));
+        else
+          log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                   "family, but that isn't a legal digest or nickname. "
+                   "Skipping it.", escaped(name));
+        smartlist_add_strdup(warned_family, name);
+      }
+      if (is_legal) {
+        smartlist_add_strdup(declared_family, name);
+      }
+    } else {
+      /* List the node by digest. */
+      char *fp = tor_malloc(HEX_DIGEST_LEN+2);
+      fp[0] = '$';
+      base16_encode(fp+1,HEX_DIGEST_LEN+1,
+                    member->identity, DIGEST_LEN);
+      smartlist_add(declared_family, fp);
+
+      if (! is_legal_hexdigest(name) &&
+          !smartlist_contains_string(warned_family, name)) {
+        /* Warn if this node was not specified by hexdigest. */
+        log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                 "family, but it wasn't listed by digest. Please consider "
+                 "saying %s instead, if that's what you meant.",
+                 escaped(name), fp);
+        smartlist_add_strdup(warned_family, name);
+      }
+    }
+  }
+
+  /* Now declared_family should have the closest we can come to the
+   * identities that the user wanted.
+   *
+   * Unlike older versions of Tor, we _do_ include our own identity: this
+   * helps microdescriptor compression, and helps in-memory compression
+   * on clients. */
+  nodefamily_t *nf = nodefamily_from_members(declared_family,
+                                     router_get_my_id_digest(),
+                                     NF_WARN_MALFORMED,
+                                     NULL);
+  SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s));
+  smartlist_free(declared_family);
+  if (!nf) {
+    return NULL;
+  }
+
+  char *s = nodefamily_format(nf);
+  nodefamily_free(nf);
+
+  smartlist_t *result = smartlist_new();
+  smartlist_split_string(result, s, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  tor_free(s);
+
+  if (smartlist_len(result) == 1) {
+    /* This is a one-element list containing only ourself; instead return
+     * nothing */
+    const char *singleton = smartlist_get(result, 0);
+    bool is_me = false;
+    if (singleton[0] == '$') {
+      char d[DIGEST_LEN];
+      int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1));
+      if (n == DIGEST_LEN &&
+          fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) {
+        is_me = true;
+      }
+    }
+    if (!is_me) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Found a singleton family list with an element "
+               "that wasn't us! Element was %s", escaped(singleton));
+      // LCOV_EXCL_STOP
+    } else {
+      SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+      smartlist_free(result);
+      return NULL;
+    }
+  }
+
+  return result;
+}
+
+/** Allocate a fresh, unsigned routerinfo for this OR, without any of the
+ * fields that depend on the corresponding extrainfo.
+ *
+ * On success, set ri_out to the new routerinfo, and return 0.
+ * Caller is responsible for freeing the generated routerinfo.
+ *
+ * Returns a negative value and sets ri_out to NULL on temporary error.
  */
-int
-router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+MOCK_IMPL(STATIC int,
+router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out))
 {
-  routerinfo_t *ri;
-  extrainfo_t *ei;
+  routerinfo_t *ri = NULL;
   uint32_t addr;
   char platform[256];
   int hibernating = we_are_hibernating();
   const or_options_t *options = get_options();
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
+  if (BUG(!ri_out)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
 
   if (router_pick_published_address(options, &addr, 0) < 0) {
     log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
-    return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    result = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    goto err;
   }
 
   /* Log a message if the address in the descriptor doesn't match the ORPort
@@ -1880,8 +2025,8 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key());
   if (BUG(crypto_pk_get_digest(ri->identity_pkey,
                            ri->cache_info.identity_digest) < 0)) {
-    routerinfo_free(ri);
-    return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    result = TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    goto err;
   }
   ri->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
@@ -1918,134 +2063,260 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
     tor_free(p_tmp);
   }
 
-  if (options->MyFamily && ! options->BridgeRelay) {
-    if (!warned_nonexistent_family)
-      warned_nonexistent_family = smartlist_new();
-    ri->declared_family = smartlist_new();
-    config_line_t *family;
-    for (family = options->MyFamily; family; family = family->next) {
-       char *name = family->value;
-       const node_t *member;
-       if (!strcasecmp(name, options->Nickname))
-         continue; /* Don't list ourself, that's redundant */
-       else
-         member = node_get_by_nickname(name, 0);
-       if (!member) {
-         int is_legal = is_legal_nickname_or_hexdigest(name);
-         if (!smartlist_contains_string(warned_nonexistent_family, name) &&
-             !is_legal_hexdigest(name)) {
-           if (is_legal)
-             log_warn(LD_CONFIG,
-                      "I have no descriptor for the router named \"%s\" in my "
-                      "declared family; I'll use the nickname as is, but "
-                      "this may confuse clients.", name);
-           else
-             log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
-                      "declared family, but that isn't a legal nickname. "
-                      "Skipping it.", escaped(name));
-           smartlist_add_strdup(warned_nonexistent_family, name);
-         }
-         if (is_legal) {
-           smartlist_add_strdup(ri->declared_family, name);
-         }
-       } else if (router_digest_is_me(member->identity)) {
-         /* Don't list ourself in our own family; that's redundant */
-         /* XXX shouldn't be possible */
-       } else {
-         char *fp = tor_malloc(HEX_DIGEST_LEN+2);
-         fp[0] = '$';
-         base16_encode(fp+1,HEX_DIGEST_LEN+1,
-                       member->identity, DIGEST_LEN);
-         smartlist_add(ri->declared_family, fp);
-         if (smartlist_contains_string(warned_nonexistent_family, name))
-           smartlist_string_remove(warned_nonexistent_family, name);
-       }
-    }
+  ri->declared_family = get_my_declared_family(options);
 
-    /* remove duplicates from the list */
-    smartlist_sort_strings(ri->declared_family);
-    smartlist_uniq_strings(ri->declared_family);
+  if (options->BridgeRelay) {
+    ri->purpose = ROUTER_PURPOSE_BRIDGE;
+    /* Bridges shouldn't be able to send their descriptors unencrypted,
+     anyway, since they don't have a DirPort, and always connect to the
+     bridge authority anonymously.  But just in case they somehow think of
+     sending them on an unencrypted connection, don't allow them to try. */
+    ri->cache_info.send_unencrypted = 0;
+  } else {
+    ri->purpose = ROUTER_PURPOSE_GENERAL;
+    ri->cache_info.send_unencrypted = 1;
   }
 
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  *ri_out = NULL;
+  return result;
+
+ done:
+  *ri_out = ri;
+  return 0;
+}
+
+/** Allocate and return a fresh, unsigned extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * Uses options->Nickname to set the nickname, and options->BridgeRelay to set
+ * ei->cache_info.send_unencrypted.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+static extrainfo_t *
+router_build_fresh_unsigned_extrainfo(const routerinfo_t *ri)
+{
+  extrainfo_t *ei = NULL;
+  const or_options_t *options = get_options();
+
+  if (BUG(!ri))
+    return NULL;
+
   /* Now generate the extrainfo. */
   ei = tor_malloc_zero(sizeof(extrainfo_t));
   ei->cache_info.is_extrainfo = 1;
-  strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname));
+  strlcpy(ei->nickname, options->Nickname, sizeof(ei->nickname));
   ei->cache_info.published_on = ri->cache_info.published_on;
   ei->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
 
   memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest,
          DIGEST_LEN);
+
+  if (options->BridgeRelay) {
+    /* See note in router_build_fresh_routerinfo(). */
+    ei->cache_info.send_unencrypted = 0;
+  } else {
+    ei->cache_info.send_unencrypted = 1;
+  }
+
+  return ei;
+}
+
+/** Dump the extrainfo descriptor body for ei, sign it, and add the body and
+ * signature to ei->cache_info. Note that the extrainfo body is determined by
+ * ei, and some additional config and statistics state: see
+ * extrainfo_dump_to_string() for details.
+ *
+ * Return 0 on success, -1 on temporary error.
+ * If ei is NULL, logs a BUG() warning and returns -1.
+ * On error, ei->cache_info is not modified.
+ */
+static int
+router_dump_and_sign_extrainfo_descriptor_body(extrainfo_t *ei)
+{
+  if (BUG(!ei))
+    return -1;
+
   if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body,
                                ei, get_server_identity_key(),
                                get_master_signing_keypair()) < 0) {
     log_warn(LD_BUG, "Couldn't generate extra-info descriptor.");
-    extrainfo_free(ei);
-    ei = NULL;
-  } else {
-    ei->cache_info.signed_descriptor_len =
-      strlen(ei->cache_info.signed_descriptor_body);
-    router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
-                              ei->cache_info.signed_descriptor_len,
-                              ei->cache_info.signed_descriptor_digest);
-    crypto_digest256((char*) ei->digest256,
-                     ei->cache_info.signed_descriptor_body,
-                     ei->cache_info.signed_descriptor_len,
-                     DIGEST_SHA256);
+    return -1;
   }
 
-  /* Now finish the router descriptor. */
-  if (ei) {
-    memcpy(ri->cache_info.extra_info_digest,
-           ei->cache_info.signed_descriptor_digest,
-           DIGEST_LEN);
-    memcpy(ri->cache_info.extra_info_digest256,
-           ei->digest256,
-           DIGEST256_LEN);
-  } else {
-    /* ri was allocated with tor_malloc_zero, so there is no need to
-     * zero ri->cache_info.extra_info_digest here. */
+  ei->cache_info.signed_descriptor_len =
+    strlen(ei->cache_info.signed_descriptor_body);
+
+  router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
+                            ei->cache_info.signed_descriptor_len,
+                            ei->cache_info.signed_descriptor_digest);
+  crypto_digest256((char*) ei->digest256,
+                   ei->cache_info.signed_descriptor_body,
+                   ei->cache_info.signed_descriptor_len,
+                   DIGEST_SHA256);
+
+  return 0;
+}
+
+/** Allocate and return a fresh, signed extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+STATIC extrainfo_t *
+router_build_fresh_signed_extrainfo(const routerinfo_t *ri)
+{
+  int result = -1;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!ri))
+    return NULL;
+
+  ei = router_build_fresh_unsigned_extrainfo(ri);
+  /* router_build_fresh_unsigned_extrainfo() should not fail. */
+  if (BUG(!ei))
+    goto err;
+
+  result = router_dump_and_sign_extrainfo_descriptor_body(ei);
+  if (result < 0)
+    goto err;
+
+  goto done;
+
+ err:
+  extrainfo_free(ei);
+  return NULL;
+
+ done:
+  return ei;
+}
+
+/** Set the fields in ri that depend on ei.
+ *
+ * If ei is NULL, logs a BUG() warning and zeroes the relevant fields.
+ */
+STATIC void
+router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                        const extrainfo_t *ei)
+{
+  if (BUG(!ei)) {
+    /* Just to be safe, zero ri->cache_info.extra_info_digest here. */
+    memset(ri->cache_info.extra_info_digest, 0, DIGEST_LEN);
+    memset(ri->cache_info.extra_info_digest256, 0, DIGEST256_LEN);
+    return;
   }
+
+  /* Now finish the router descriptor. */
+  memcpy(ri->cache_info.extra_info_digest,
+         ei->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  memcpy(ri->cache_info.extra_info_digest256,
+         ei->digest256,
+         DIGEST256_LEN);
+}
+
+/** Dump the descriptor body for ri, sign it, and add the body and signature to
+ * ri->cache_info. Note that the descriptor body is determined by ri, and some
+ * additional config and state: see router_dump_router_to_string() for details.
+ *
+ * Return 0 on success, and a negative value on temporary error.
+ * If ri is NULL, logs a BUG() warning and returns a negative value.
+ * On error, ri->cache_info is not modified.
+ */
+STATIC int
+router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri)
+{
+  if (BUG(!ri))
+    return TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
   if (! (ri->cache_info.signed_descriptor_body =
           router_dump_router_to_string(ri, get_server_identity_key(),
                                        get_onion_key(),
                                        get_current_curve25519_keypair(),
                                        get_master_signing_keypair())) ) {
     log_warn(LD_BUG, "Couldn't generate router descriptor.");
-    routerinfo_free(ri);
-    extrainfo_free(ei);
     return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
   }
+
   ri->cache_info.signed_descriptor_len =
     strlen(ri->cache_info.signed_descriptor_body);
 
-  ri->purpose =
-    options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-  if (options->BridgeRelay) {
-    /* Bridges shouldn't be able to send their descriptors unencrypted,
-       anyway, since they don't have a DirPort, and always connect to the
-       bridge authority anonymously.  But just in case they somehow think of
-       sending them on an unencrypted connection, don't allow them to try. */
-    ri->cache_info.send_unencrypted = 0;
-    if (ei)
-      ei->cache_info.send_unencrypted = 0;
-  } else {
-    ri->cache_info.send_unencrypted = 1;
-    if (ei)
-      ei->cache_info.send_unencrypted = 1;
-  }
-
   router_get_router_hash(ri->cache_info.signed_descriptor_body,
                          strlen(ri->cache_info.signed_descriptor_body),
                          ri->cache_info.signed_descriptor_digest);
 
+  return 0;
+}
+
+/** Build a fresh routerinfo, signed server descriptor, and signed extrainfo
+ * document for this OR.
+ *
+ * Set r to the generated routerinfo, e to the generated extrainfo document.
+ * Failure to generate an extra-info document is not an error and is indicated
+ * by setting e to NULL.
+ * Return 0 on success, and a negative value on temporary error.
+ * Caller is responsible for freeing generated documents on success.
+ */
+int
+router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+{
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+  routerinfo_t *ri = NULL;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!r))
+    goto err;
+
+  if (BUG(!e))
+    goto err;
+
+  result = router_build_fresh_unsigned_routerinfo(&ri);
+  if (result < 0) {
+    goto err;
+  }
+  /* If ri is NULL, then result should be negative. So this check should be
+   * unreachable. */
+  if (BUG(!ri)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
+
+  ei = router_build_fresh_signed_extrainfo(ri);
+
+  /* Failing to create an ei is not an error. */
+  if (ei) {
+    router_update_routerinfo_from_extrainfo(ri, ei);
+  }
+
+  result = router_dump_and_sign_routerinfo_descriptor_body(ri);
+  if (result < 0)
+    goto err;
+
   if (ei) {
-    tor_assert(!
-          routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
-                                                 &ri->cache_info, NULL));
+     if (BUG(routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                    &ri->cache_info, NULL))) {
+       result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+       goto err;
+     }
   }
 
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  extrainfo_free(ei);
+  *r = NULL;
+  *e = NULL;
+  return result;
+
+ done:
   *r = ri;
   *e = ei;
   return 0;
@@ -2131,7 +2402,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
   /* Now we see whether we want to be retrying frequently or no.  The
    * rule here is that we'll retry frequently if we aren't listed in the
    * live consensus we have, or if the publication time of the
-   * descriptor listed for us in the consensus is very old. */
+   * descriptor listed for us in the consensus is very old, or if the
+   * consensus lists us as "stale" and we haven't regenerated since the
+   * consensus was published. */
   ns = networkstatus_get_live_consensus(now);
   if (ns) {
     rs = networkstatus_vote_find_entry(ns, server_identitykey_digest);
@@ -2139,6 +2412,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
       retry_fast_reason = "not listed in consensus";
     else if (rs->published_on < slow_cutoff)
       retry_fast_reason = "version listed in consensus is quite old";
+    else if (rs->is_staledesc && ns->valid_after > desc_clean_since)
+      retry_fast_reason = "listed as stale in consensus";
   }
 
   if (retry_fast_reason && desc_clean_since < fast_cutoff)
@@ -2384,6 +2659,10 @@ get_platform_str(char *platform, size_t len)
 /** OR only: Given a routerinfo for this router, and an identity key to sign
  * with, encode the routerinfo as a signed server descriptor and return a new
  * string encoding the result, or NULL on failure.
+ *
+ * In addition to the fields in router, this function calls
+ * onion_key_lifetime(), get_options(), and we_are_hibernating(), and uses the
+ * results to populate some fields in the descriptor.
  */
 char *
 router_dump_router_to_string(routerinfo_t *router,
@@ -2447,11 +2726,8 @@ router_dump_router_to_string(routerinfo_t *router,
       log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
       goto err;
     }
-    if (ed25519_public_to_base64(ed_fp_base64,
-                       &router->cache_info.signing_key_cert->signing_key)<0) {
-      log_err(LD_BUG,"Couldn't base64-encode identity key\n");
-      goto err;
-    }
+    ed25519_public_to_base64(ed_fp_base64,
+                            &router->cache_info.signing_key_cert->signing_key);
     tor_asprintf(&ed_cert_line, "identity-ed25519\n"
                  "-----BEGIN ED25519 CERT-----\n"
                  "%s"
@@ -2701,8 +2977,7 @@ router_dump_router_to_string(routerinfo_t *router,
     if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN,
                      signing_keypair) < 0)
       goto err;
-    if (ed25519_signature_to_base64(buf, &sig) < 0)
-      goto err;
+    ed25519_signature_to_base64(buf, &sig);
 
     smartlist_add_asprintf(chunks, "%s\n", buf);
   }
@@ -2841,34 +3116,26 @@ load_stats_file(const char *filename, const char *end_line, time_t now,
   return r;
 }
 
-/** Write the contents of <b>extrainfo</b> and aggregated statistics to
- * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on
- * success, negative on failure. */
-int
-extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
-                         crypto_pk_t *ident_key,
-                         const ed25519_keypair_t *signing_keypair)
+/** Add header strings to chunks, based on the extrainfo object extrainfo,
+ * and ed25519 keypair signing_keypair, if emit_ed_sigs is true.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_header_helper(
+                                     smartlist_t *chunks,
+                                     const extrainfo_t *extrainfo,
+                                     const ed25519_keypair_t *signing_keypair,
+                                     int emit_ed_sigs)
 {
-  const or_options_t *options = get_options();
   char identity[HEX_DIGEST_LEN+1];
   char published[ISO_TIME_LEN+1];
-  char digest[DIGEST_LEN];
-  char *bandwidth_usage;
-  int result;
-  static int write_stats_to_extrainfo = 1;
-  char sig[DIROBJ_MAX_SIG_LEN+1];
-  char *s = NULL, *pre, *contents, *cp, *s_dup = NULL;
-  time_t now = time(NULL);
-  smartlist_t *chunks = smartlist_new();
-  extrainfo_t *ei_tmp = NULL;
-  const int emit_ed_sigs = signing_keypair &&
-    extrainfo->cache_info.signing_key_cert;
   char *ed_cert_line = NULL;
+  char *pre = NULL;
+  int rv = -1;
 
   base16_encode(identity, sizeof(identity),
                 extrainfo->cache_info.identity_digest, DIGEST_LEN);
   format_iso_time(published, extrainfo->cache_info.published_on);
-  bandwidth_usage = rep_hist_get_bandwidth_lines();
   if (emit_ed_sigs) {
     if (!extrainfo->cache_info.signing_key_cert->signing_key_included ||
         !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key,
@@ -2894,21 +3161,64 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     ed_cert_line = tor_strdup("");
   }
 
-  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s",
+  /* This is the first chunk in the file. If the file is too big, other chunks
+   * are removed. So we must only add one chunk here. */
+  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n",
                extrainfo->nickname, identity,
                ed_cert_line,
-               published, bandwidth_usage);
+               published);
   smartlist_add(chunks, pre);
 
-  if (geoip_is_loaded(AF_INET))
-    smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
-                           geoip_db_digest(AF_INET));
-  if (geoip_is_loaded(AF_INET6))
-    smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
-                           geoip_db_digest(AF_INET6));
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  tor_free(ed_cert_line);
+  return rv;
+}
+
+/** Add pluggable transport and statistics strings to chunks, skipping
+ * statistics if write_stats_to_extrainfo is false.
+ * Helper for extrainfo_dump_to_string().
+ * Can not fail. */
+static void
+extrainfo_dump_to_string_stats_helper(smartlist_t *chunks,
+                                      int write_stats_to_extrainfo)
+{
+  const or_options_t *options = get_options();
+  char *contents = NULL;
+  time_t now = time(NULL);
+
+  /* If the file is too big, these chunks are removed, starting with the last
+   * chunk. So each chunk must be a complete line, and the file must be valid
+   * after each chunk. */
+
+  /* Add information about the pluggable transports we support, even if we
+   * are not publishing statistics. This information is needed by BridgeDB
+   * to distribute bridges. */
+  if (options->ServerTransportPlugin) {
+    char *pluggable_transports = pt_get_extra_info_descriptor_string();
+    if (pluggable_transports)
+      smartlist_add(chunks, pluggable_transports);
+  }
 
   if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
     log_info(LD_GENERAL, "Adding stats to extra-info descriptor.");
+    /* Bandwidth usage stats don't have their own option */
+    {
+      contents = rep_hist_get_bandwidth_lines();
+      smartlist_add(chunks, contents);
+    }
+    /* geoip hashes aren't useful unless we are publishing other stats */
+    if (geoip_is_loaded(AF_INET))
+      smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
+                             geoip_db_digest(AF_INET));
+    if (geoip_is_loaded(AF_INET6))
+      smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
+                             geoip_db_digest(AF_INET6));
     if (options->DirReqStatistics &&
         load_stats_file("stats"PATH_SEPARATOR"dirreq-stats",
                         "dirreq-stats-end", now, &contents) > 0) {
@@ -2944,50 +3254,140 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
       if (contents)
         smartlist_add(chunks, contents);
     }
+    /* bridge statistics */
+    if (should_record_bridge_info(options)) {
+      const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
+      if (bridge_stats) {
+        smartlist_add_strdup(chunks, bridge_stats);
+      }
+    }
   }
+}
 
-  /* Add information about the pluggable transports we support. */
-  if (options->ServerTransportPlugin) {
-    char *pluggable_transports = pt_get_extra_info_descriptor_string();
-    if (pluggable_transports)
-      smartlist_add(chunks, pluggable_transports);
-  }
+/** Add an ed25519 signature of chunks to chunks, using the ed25519 keypair
+ * signing_keypair.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_ed_sig_helper(
+                                     smartlist_t *chunks,
+                                     const ed25519_keypair_t *signing_keypair)
+{
+  char sha256_digest[DIGEST256_LEN];
+  ed25519_signature_t ed_sig;
+  char buf[ED25519_SIG_BASE64_LEN+1];
+  int rv = -1;
+
+  /* These are two of the three final chunks in the file. If the file is too
+   * big, other chunks are removed. So we must only add two chunks here. */
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+  crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
+                                 ED_DESC_SIGNATURE_PREFIX,
+                                 chunks, "", DIGEST_SHA256);
+  if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
+                   signing_keypair) < 0)
+    goto err;
+  ed25519_signature_to_base64(buf, &ed_sig);
 
-  if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
-    const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
-    if (bridge_stats) {
-      smartlist_add_strdup(chunks, bridge_stats);
-    }
+  smartlist_add_asprintf(chunks, "%s\n", buf);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Add an RSA signature of extrainfo_string to chunks, using the RSA key
+ * ident_key.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_rsa_sig_helper(smartlist_t *chunks,
+                                        crypto_pk_t *ident_key,
+                                        const char *extrainfo_string)
+{
+  char sig[DIROBJ_MAX_SIG_LEN+1];
+  char digest[DIGEST_LEN];
+  int rv = -1;
+
+  memset(sig, 0, sizeof(sig));
+  if (router_get_extrainfo_hash(extrainfo_string, strlen(extrainfo_string),
+                                digest) < 0 ||
+      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
+                                     ident_key) < 0) {
+    log_warn(LD_BUG, "Could not append signature to extra-info "
+                     "descriptor.");
+    goto err;
   }
+  smartlist_add_strdup(chunks, sig);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Write the contents of <b>extrainfo</b>, to * *<b>s_out</b>, signing them
+ * with <b>ident_key</b>.
+ *
+ * If ExtraInfoStatistics is 1, also write aggregated statistics and related
+ * configuration data before signing. Most statistics also have an option that
+ * enables or disables that particular statistic.
+ *
+ * Always write pluggable transport lines.
+ *
+ * Return 0 on success, negative on failure. */
+int
+extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
+                         crypto_pk_t *ident_key,
+                         const ed25519_keypair_t *signing_keypair)
+{
+  int result;
+  static int write_stats_to_extrainfo = 1;
+  char *s = NULL, *cp, *s_dup = NULL;
+  smartlist_t *chunks = smartlist_new();
+  extrainfo_t *ei_tmp = NULL;
+  const int emit_ed_sigs = signing_keypair &&
+    extrainfo->cache_info.signing_key_cert;
+  int rv = 0;
+
+  rv = extrainfo_dump_to_string_header_helper(chunks, extrainfo,
+                                              signing_keypair,
+                                              emit_ed_sigs);
+  if (rv < 0)
+    goto err;
+
+  extrainfo_dump_to_string_stats_helper(chunks, write_stats_to_extrainfo);
 
   if (emit_ed_sigs) {
-    char sha256_digest[DIGEST256_LEN];
-    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
-    crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
-                                   ED_DESC_SIGNATURE_PREFIX,
-                                   chunks, "", DIGEST_SHA256);
-    ed25519_signature_t ed_sig;
-    char buf[ED25519_SIG_BASE64_LEN+1];
-    if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
-                     signing_keypair) < 0)
-      goto err;
-    if (ed25519_signature_to_base64(buf, &ed_sig) < 0)
+    rv = extrainfo_dump_to_string_ed_sig_helper(chunks, signing_keypair);
+    if (rv < 0)
       goto err;
-
-    smartlist_add_asprintf(chunks, "%s\n", buf);
   }
 
+  /* This is one of the three final chunks in the file. If the file is too big,
+   * other chunks are removed. So we must only add one chunk here. */
   smartlist_add_strdup(chunks, "router-signature\n");
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
   while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
     /* So long as there are at least two chunks (one for the initial
      * extra-info line and one for the router-signature), we can keep removing
-     * things. */
-    if (smartlist_len(chunks) > 2) {
-      /* We remove the next-to-last element (remember, len-1 is the last
-         element), since we need to keep the router-signature element. */
-      int idx = smartlist_len(chunks) - 2;
+     * things. If emit_ed_sigs is true, we also keep 2 additional chunks at the
+     * end for the ed25519 signature. */
+    const int required_chunks = emit_ed_sigs ? 4 : 2;
+    if (smartlist_len(chunks) > required_chunks) {
+      /* We remove the next-to-last or 4th-last element (remember, len-1 is the
+       * last element), since we need to keep the router-signature elements. */
+      int idx = smartlist_len(chunks) - required_chunks;
       char *e = smartlist_get(chunks, idx);
       smartlist_del_keeporder(chunks, idx);
       log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
@@ -3004,15 +3404,10 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     }
   }
 
-  memset(sig, 0, sizeof(sig));
-  if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 ||
-      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
-                                     ident_key) < 0) {
-    log_warn(LD_BUG, "Could not append signature to extra-info "
-                     "descriptor.");
+  rv = extrainfo_dump_to_string_rsa_sig_helper(chunks, ident_key, s);
+  if (rv < 0)
     goto err;
-  }
-  smartlist_add_strdup(chunks, sig);
+
   tor_free(s);
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -3048,9 +3443,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk));
   smartlist_free(chunks);
   tor_free(s_dup);
-  tor_free(ed_cert_line);
   extrainfo_free(ei_tmp);
-  tor_free(bandwidth_usage);
 
   return result;
 }
@@ -3060,9 +3453,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
 void
 router_reset_warnings(void)
 {
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_clear(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_clear(warned_family);
   }
 }
 
@@ -3075,6 +3468,10 @@ router_free_all(void)
   crypto_pk_free(server_identitykey);
   crypto_pk_free(client_identitykey);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
   tor_mutex_free(key_lock);
   routerinfo_free(desc_routerinfo);
   extrainfo_free(desc_extrainfo);
@@ -3086,11 +3483,12 @@ router_free_all(void)
   memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key));
   memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_free(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_free(warned_family);
   }
 }
+
 /* From the given RSA key object, convert it to ASN-1 encoded format and set
  * the newly allocated object in onion_pkey_out. The length of the key is set
  * in onion_pkey_len_out. */
diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h
index bd6a8a012e..55b9ef9e68 100644
--- a/src/feature/relay/router.h
+++ b/src/feature/relay/router.h
@@ -23,11 +23,12 @@ struct ed25519_keypair_t;
 #define TOR_ROUTERINFO_ERROR_DIGEST_FAILED   (-4)
 #define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5)
 #define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6)
+#define TOR_ROUTERINFO_ERROR_INTERNAL_BUG    (-7)
 
-crypto_pk_t *get_onion_key(void);
+MOCK_DECL(crypto_pk_t *,get_onion_key,(void));
 time_t get_onion_key_set_at(void);
 void set_server_identity_key(crypto_pk_t *k);
-crypto_pk_t *get_server_identity_key(void);
+MOCK_DECL(crypto_pk_t *,get_server_identity_key,(void));
 int server_identity_key_is_set(void);
 void set_client_identity_key(crypto_pk_t *k);
 crypto_pk_t *get_tlsclient_identity_key(void);
@@ -114,9 +115,27 @@ void router_reset_reachability(void);
 void router_free_all(void);
 
 #ifdef ROUTER_PRIVATE
-/* Used only by router.c and test.c */
+/* Used only by router.c and the unit tests */
 STATIC void get_platform_str(char *platform, size_t len);
 STATIC int router_write_fingerprint(int hashed);
-#endif
+STATIC smartlist_t *get_my_declared_family(const or_options_t *options);
+
+#ifdef TOR_UNIT_TESTS
+extern time_t desc_clean_since;
+extern const char *desc_dirty_reason;
+void set_server_identity_key_digest_testing(const uint8_t *digest);
+MOCK_DECL(STATIC const struct curve25519_keypair_t *,
+                                       get_current_curve25519_keypair,(void));
+
+MOCK_DECL(STATIC int,
+              router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out));
+STATIC extrainfo_t *router_build_fresh_signed_extrainfo(
+                                                      const routerinfo_t *ri);
+STATIC void router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                                    const extrainfo_t *ei);
+STATIC int router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(ROUTER_PRIVATE) */
 
 #endif /* !defined(TOR_ROUTER_H) */
diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c
index f639fc91e7..a9190b2e13 100644
--- a/src/feature/relay/routerkeys.c
+++ b/src/feature/relay/routerkeys.c
@@ -226,7 +226,7 @@ load_ed_keys(const or_options_t *options, time_t now)
         tor_free(fname);
       }
     }
-    if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
+    if (safe_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
       sign_signing_key_with_id = NULL;
     else
       sign_signing_key_with_id = id;
@@ -631,14 +631,14 @@ get_master_identity_keypair(void)
 }
 #endif /* defined(TOR_UNIT_TESTS) */
 
-const ed25519_keypair_t *
-get_master_signing_keypair(void)
+MOCK_IMPL(const ed25519_keypair_t *,
+get_master_signing_keypair,(void))
 {
   return master_signing_key;
 }
 
-const struct tor_cert_st *
-get_master_signing_key_cert(void)
+MOCK_IMPL(const struct tor_cert_st *,
+get_master_signing_key_cert,(void))
 {
   return signing_key_cert;
 }
@@ -706,6 +706,8 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
 
   *len_out = 0;
   if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    log_info(LD_OR, "crypto_pk_get_digest failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
   }
   memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
@@ -713,8 +715,12 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
   int r = crypto_pk_private_sign(onion_key,
                                (char*)signature, sizeof(signature),
                                (const char*)signed_data, sizeof(signed_data));
-  if (r < 0)
+  if (r < 0) {
+    /* It's probably missing the private key */
+    log_info(LD_OR, "crypto_pk_private_sign failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
+  }
 
   *len_out = r;
 
diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h
index 0badd34191..cde07b52c3 100644
--- a/src/feature/relay/routerkeys.h
+++ b/src/feature/relay/routerkeys.h
@@ -7,8 +7,8 @@
 #include "lib/crypt_ops/crypto_ed25519.h"
 
 const ed25519_public_key_t *get_master_identity_key(void);
-const ed25519_keypair_t *get_master_signing_keypair(void);
-const struct tor_cert_st *get_master_signing_key_cert(void);
+MOCK_DECL(const ed25519_keypair_t *, get_master_signing_keypair,(void));
+MOCK_DECL(const struct tor_cert_st *, get_master_signing_key_cert,(void));
 
 const ed25519_keypair_t *get_current_auth_keypair(void);
 const struct tor_cert_st *get_current_link_cert_cert(void);
diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c
index 064eea6c46..f8b54ff45d 100644
--- a/src/feature/relay/selftest.c
+++ b/src/feature/relay/selftest.c
@@ -26,7 +26,7 @@
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/authority_cert_st.h"
@@ -35,6 +35,7 @@
 #include "feature/nodelist/routerlist.h" // but...
 #include "feature/nodelist/routerset.h"
 #include "feature/nodelist/torcert.h"
+#include "feature/relay/relay_periodic.h"
 #include "feature/relay/router.h"
 #include "feature/relay/selftest.h"
 
diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h
index a80ec8936e..aea77ec791 100644
--- a/src/feature/relay/selftest.h
+++ b/src/feature/relay/selftest.h
@@ -21,4 +21,4 @@ void router_orport_found_reachable(void);
 void router_dirport_found_reachable(void);
 void router_perform_bandwidth_test(int num_circs, time_t now);
 
-#endif
+#endif /* !defined(TOR_SELFTEST_H) */
diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h
index 7bd4f2fe8c..51a1798fcb 100644
--- a/src/feature/rend/rend_authorized_client_st.h
+++ b/src/feature/rend/rend_authorized_client_st.h
@@ -14,5 +14,5 @@ struct rend_authorized_client_t {
   crypto_pk_t *client_key;
 };
 
-#endif
+#endif /* !defined(REND_AUTHORIZED_CLIENT_ST_H) */
 
diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
index 05ff145d53..bd8a60f0d9 100644
--- a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
+++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
@@ -13,5 +13,5 @@ struct rend_encoded_v2_service_descriptor_t {
   char *desc_str; /**< Descriptor string. */
 };
 
-#endif
+#endif /* !defined(REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H) */
 
diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h
index de6987e569..4882b62752 100644
--- a/src/feature/rend/rend_intro_point_st.h
+++ b/src/feature/rend/rend_intro_point_st.h
@@ -73,4 +73,4 @@ struct rend_intro_point_t {
   unsigned int circuit_established:1;
 };
 
-#endif
+#endif /* !defined(REND_INTRO_POINT_ST_H) */
diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h
index aeb3178064..ff7627ce96 100644
--- a/src/feature/rend/rend_service_descriptor_st.h
+++ b/src/feature/rend/rend_service_descriptor_st.h
@@ -30,5 +30,5 @@ struct rend_service_descriptor_t {
   smartlist_t *successful_uploads;
 };
 
-#endif
+#endif /* !defined(REND_SERVICE_DESCRIPTOR_ST_H) */
 
diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c
index 1c3badaff3..c3f86d8c82 100644
--- a/src/feature/rend/rendcache.c
+++ b/src/feature/rend/rendcache.c
@@ -19,6 +19,8 @@
 #include "feature/rend/rend_intro_point_st.h"
 #include "feature/rend/rend_service_descriptor_st.h"
 
+#include "lib/ctime/di_ops.h"
+
 /** Map from service id (as generated by rend_get_service_id) to
  * rend_cache_entry_t. */
 STATIC strmap_t *rend_cache = NULL;
@@ -45,7 +47,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL;
  * looked up in this cache and if present, it is discarded from the fetched
  * descriptor. At the end, all IP(s) in the cache, for a specific service
  * ID, that were NOT present in the descriptor are removed from this cache.
- * Which means that if at least one IP was not in this cache, thus usuable,
+ * Which means that if at least one IP was not in this cache, thus usable,
  * it's considered a new descriptor so we keep it. Else, if all IPs were in
  * this cache, we discard the descriptor as it's considered unusable.
  *
@@ -593,10 +595,10 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc)
   char desc_id_digest[DIGEST_LEN];
   tor_assert(rend_cache_v2_dir);
   if (base32_decode(desc_id_digest, DIGEST_LEN,
-                    desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) {
+                    desc_id, REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Rejecting v2 rendezvous descriptor request -- descriptor ID "
-           "contains illegal characters: %s",
+           "has wrong length or illegal characters: %s",
            safe_str(desc_id));
     return -1;
   }
@@ -854,7 +856,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     *entry = NULL;
   }
   if (base32_decode(want_desc_id, sizeof(want_desc_id),
-                    desc_id_base32, strlen(desc_id_base32)) != 0) {
+                    desc_id_base32, strlen(desc_id_base32)) !=
+      sizeof(want_desc_id)) {
     log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.",
              escaped_safe_str_client(desc_id_base32));
     goto err;
@@ -888,8 +891,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   if (intro_content && intro_size > 0) {
     int n_intro_points;
     if (rend_data->auth_type != REND_NO_AUTH &&
-        !tor_mem_is_zero(rend_data->descriptor_cookie,
-                         sizeof(rend_data->descriptor_cookie))) {
+        !safe_mem_is_zero(rend_data->descriptor_cookie,
+                          sizeof(rend_data->descriptor_cookie))) {
       char *ipos_decrypted = NULL;
       size_t ipos_decrypted_size;
       if (rend_decrypt_introduction_points(&ipos_decrypted,
@@ -1005,4 +1008,3 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   tor_free(intro_content);
   return retval;
 }
-
diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c
index 5c9dbea8e3..1446dc2269 100644
--- a/src/feature/rend/rendclient.c
+++ b/src/feature/rend/rendclient.c
@@ -17,7 +17,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/relay.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
@@ -119,7 +119,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   char tmp[RELAY_PAYLOAD_SIZE];
   rend_cache_entry_t *entry = NULL;
   crypt_path_t *cpath;
-  off_t dh_offset;
+  ptrdiff_t dh_offset;
   crypto_pk_t *intro_key = NULL;
   int status = 0;
   const char *onion_address;
@@ -403,14 +403,23 @@ rend_client_introduction_acked(origin_circuit_t *circ,
     } else {
       log_info(LD_REND,"...Found no rend circ. Dropping on the floor.");
     }
+    /* Save the rend data digest to a temporary object so that we don't access
+     * it after we mark the circuit for close. */
+    const uint8_t *rend_digest_tmp = NULL;
+    size_t digest_len;
+    uint8_t *cached_rend_digest = NULL;
+    rend_digest_tmp = rend_data_get_pk_digest(circ->rend_data, &digest_len);
+    cached_rend_digest = tor_malloc_zero(digest_len);
+    memcpy(cached_rend_digest, rend_digest_tmp, digest_len);
+
     /* close the circuit: we won't need it anymore. */
     circuit_change_purpose(TO_CIRCUIT(circ),
                            CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
 
     /* close any other intros launched in parallel */
-    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
-                                                           NULL));
+    rend_client_close_other_intros(cached_rend_digest);
+    tor_free(cached_rend_digest); /* free the temporary digest */
   } else {
     /* It's a NAK; the introduction point didn't relay our request. */
     circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
@@ -469,16 +478,19 @@ directory_get_from_hs_dir(const char *desc_id,
 
   /* Automatically pick an hs dir if none given. */
   if (!rs_hsdir) {
+    bool rate_limited = false;
+
     /* Determine responsible dirs. Even if we can't get all we want, work with
      * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */
     smartlist_t *responsible_dirs = smartlist_new();
     hid_serv_get_responsible_directories(responsible_dirs, desc_id);
 
-    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32);
+    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32, &rate_limited);
     if (!hs_dir) {
       /* No suitable hs dir can be found, stop right now. */
-      control_event_hsv2_descriptor_failed(rend_query, NULL,
-                                           "QUERY_NO_HSDIR");
+      const char *query_response = (rate_limited) ? "QUERY_RATE_LIMITED" :
+                                                    "QUERY_NO_HSDIR";
+      control_event_hsv2_descriptor_failed(rend_query, NULL, query_response);
       control_event_hs_descriptor_content(rend_data_get_address(rend_query),
                                           desc_id_base32, NULL, NULL);
       return 0;
diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c
index de48af795f..0a606a9f02 100644
--- a/src/feature/rend/rendcommon.c
+++ b/src/feature/rend/rendcommon.c
@@ -15,7 +15,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/hs/hs_client.h"
@@ -171,9 +171,10 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id,
   }
   /* Convert service ID to binary. */
   if (base32_decode(service_id_binary, REND_SERVICE_ID_LEN,
-                    service_id, REND_SERVICE_ID_LEN_BASE32) < 0) {
+                    service_id, REND_SERVICE_ID_LEN_BASE32) !=
+      REND_SERVICE_ID_LEN) {
     log_warn(LD_REND, "Could not compute v2 descriptor ID: "
-                      "Illegal characters in service ID: %s",
+                      "Illegal characters or wrong length for service ID: %s",
              safe_str_client(service_id));
     return -1;
   }
@@ -785,39 +786,39 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
   switch (command) {
     case RELAY_COMMAND_ESTABLISH_INTRO:
       if (or_circ)
-        r = hs_intro_received_establish_intro(or_circ,payload,length);
+        r = hs_intro_received_establish_intro(or_circ, payload, length);
       break;
     case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
       if (or_circ)
-        r = rend_mid_establish_rendezvous(or_circ,payload,length);
+        r = rend_mid_establish_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE1:
       if (or_circ)
-        r = hs_intro_received_introduce1(or_circ,payload,length);
+        r = hs_intro_received_introduce1(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE2:
       if (origin_circ)
-        r = hs_service_receive_introduce2(origin_circ,payload,length);
+        r = hs_service_receive_introduce2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE_ACK:
       if (origin_circ)
-        r = hs_client_receive_introduce_ack(origin_circ,payload,length);
+        r = hs_client_receive_introduce_ack(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS1:
       if (or_circ)
-        r = rend_mid_rendezvous(or_circ,payload,length);
+        r = rend_mid_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS2:
       if (origin_circ)
-        r = hs_client_receive_rendezvous2(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRO_ESTABLISHED:
       if (origin_circ)
-        r = hs_service_receive_intro_established(origin_circ,payload,length);
+        r = hs_service_receive_intro_established(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       if (origin_circ)
-        r = hs_client_receive_rendezvous_acked(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous_acked(origin_circ, payload, length);
       break;
     default:
       tor_fragile_assert();
diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c
index 3ba48f8858..06471b2a7f 100644
--- a/src/feature/rend/rendmid.c
+++ b/src/feature/rend/rendmid.c
@@ -18,6 +18,7 @@
 #include "feature/rend/rendmid.h"
 #include "feature/stats/rephist.h"
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 
 #include "core/or/or_circuit_st.h"
@@ -117,6 +118,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
   /* Now, set up this circuit. */
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
   hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
+  hs_dos_setup_default_intro2_defenses(circ);
 
   log_info(LD_REND,
            "Established introduction point on circuit %u for service %s",
@@ -181,6 +183,14 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(intro_circ)) {
+    log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v2 cell due to DoS "
+                          "limitations. Sending NACK to client.");
+    goto err;
+  }
+
   log_info(LD_REND,
            "Sending introduction request for service %s "
            "from circ %u to circ %u",
@@ -237,8 +247,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
-  /* Check if we are configured to accept established rendezvous cells from
-   * client or in other words Tor2Web clients. */
+  /* Check if we are configured to defend ourselves from clients that
+   * attempt to establish rendezvous points directly to us. */
   if (channel_is_client(circ->p_chan) &&
       dos_should_refuse_single_hop_client()) {
     /* Note it down for the heartbeat log purposes. */
diff --git a/src/feature/rend/rendparse.c b/src/feature/rend/rendparse.c
index abd0feb448..a98cb3ad88 100644
--- a/src/feature/rend/rendparse.c
+++ b/src/feature/rend/rendparse.c
@@ -143,8 +143,9 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     goto err;
   }
   if (base32_decode(desc_id_out, DIGEST_LEN,
-                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
-    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
+                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Descriptor ID has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -174,8 +175,10 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
     goto err;
   }
-  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
-    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
+  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) !=
+      DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Secret ID part has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -429,8 +432,10 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed,
     /* Parse identifier. */
     tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
     if (base32_decode(info->identity_digest, DIGEST_LEN,
-                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
-      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
+                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) !=
+        DIGEST_LEN) {
+      log_warn(LD_REND,
+               "Identity digest has wrong length or illegal characters: %s",
                tok->args[0]);
       rend_intro_point_free(intro);
       goto err;
diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h
index 0cef931e90..b1ccce9b6c 100644
--- a/src/feature/rend/rendparse.h
+++ b/src/feature/rend/rendparse.h
@@ -29,4 +29,4 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
                                    size_t intro_points_encoded_size);
 int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
 
-#endif
+#endif /* !defined(TOR_REND_PARSE_H) */
diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c
index c96ecec308..e0cf06b9df 100644
--- a/src/feature/rend/rendservice.c
+++ b/src/feature/rend/rendservice.c
@@ -18,8 +18,9 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_common.h"
@@ -2126,7 +2127,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
      *
      * We only use a one-hop path on the first attempt. If the first attempt
      * fails, we use a 3-hop path for reachability / reliability.
-     * See the comment in rend_service_relauch_rendezvous() for details. */
+     * See the comment in rend_service_relaunch_rendezvous() for details. */
     if (rend_service_use_direct_connection(options, rp) && i == 0) {
       flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL;
     }
@@ -2167,7 +2168,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   cpath->rend_dh_handshake_state = dh;
   dh = NULL;
-  if (circuit_init_cpath_crypto(cpath,
+  if (cpath_init_circuit_crypto(cpath,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 1, 0)<0)
     goto err;
@@ -3016,6 +3017,10 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
 {
   origin_circuit_t *newcirc;
   cpath_build_state_t *newstate, *oldstate;
+  const char *rend_pk_digest;
+  rend_service_t *service = NULL;
+
+  int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
 
   tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
   oldstate = oldcirc->build_state;
@@ -3030,13 +3035,31 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
   log_info(LD_REND,"Reattempting rendezvous circuit to '%s'",
            safe_str(extend_info_describe(oldstate->chosen_exit)));
 
+  /* Look up the service. */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(oldcirc->rend_data, NULL);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+
+  if (!service) {
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                  rend_pk_digest, REND_SERVICE_ID_LEN);
+
+    log_warn(LD_BUG, "Internal error: Trying to relaunch a rendezvous circ "
+                     "for an unrecognized service %s.",
+                     safe_str_client(serviceid));
+    return;
+  }
+
+  if (hs_service_requires_uptime_circ(service->ports)) {
+    flags |= CIRCLAUNCH_NEED_UPTIME;
+  }
+
   /* You'd think Single Onion Services would want to retry the rendezvous
    * using a direct connection. But if it's blocked by a firewall, or the
    * service is IPv6-only, or the rend point avoiding becoming a one-hop
    * proxy, we need a 3-hop connection. */
   newcirc = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
-                            oldstate->chosen_exit,
-                            CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+                            oldstate->chosen_exit, flags);
 
   if (!newcirc) {
     log_warn(LD_REND,"Couldn't relaunch rendezvous circuit to '%s'.",
@@ -3536,7 +3559,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   hop->package_window = circuit_initial_package_window();
   hop->deliver_window = CIRCWINDOW_START;
 
-  onion_append_to_cpath(&circuit->cpath, hop);
+  cpath_extend_linked_list(&circuit->cpath, hop);
   circuit->build_state->pending_final_cpath = NULL; /* prevent double-free */
 
   /* Change the circuit purpose. */
@@ -3976,7 +3999,7 @@ remove_invalid_intro_points(rend_service_t *service,
        * accounted for when considiring uploading a descriptor. */
       intro->circuit_established = 0;
 
-      /* Node is gone or we've reached our maximum circuit creationg retry
+      /* Node is gone or we've reached our maximum circuit creation retry
        * count, clean up everything, we'll find a new one. */
       if (node == NULL ||
           intro->circuit_retries >= MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -4216,6 +4239,7 @@ rend_consider_services_intro_points(time_t now)
        * directly ourselves. */
       intro->extend_info = extend_info_from_node(node, 0);
       if (BUG(intro->extend_info == NULL)) {
+        tor_free(intro);
         break;
       }
       intro->intro_key = crypto_pk_new();
diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c
index a54b589eb6..6fb21f4f79 100644
--- a/src/feature/stats/geoip_stats.c
+++ b/src/feature/stats/geoip_stats.c
@@ -30,9 +30,9 @@
 #include "core/or/or.h"
 
 #include "ht.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/dnsserv.h"
 #include "core/or/dos.h"
 #include "lib/geoip/geoip.h"
diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h
index 272344da2f..45b206c23a 100644
--- a/src/feature/stats/predict_ports.h
+++ b/src/feature/stats/predict_ports.h
@@ -27,4 +27,4 @@ int rep_hist_circbuilding_dormant(time_t now);
 int predicted_ports_prediction_time_remaining(time_t now);
 void predicted_ports_free_all(void);
 
-#endif
+#endif /* !defined(TOR_PREDICT_PORTS_H) */
diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h
index 3accc8c610..0d72946382 100644
--- a/src/feature/stats/rephist.h
+++ b/src/feature/stats/rephist.h
@@ -103,7 +103,7 @@ typedef struct bw_array_t bw_array_t;
 STATIC uint64_t find_largest_max(bw_array_t *b);
 STATIC void commit_max(bw_array_t *b);
 STATIC void advance_obs(bw_array_t *b);
-#endif
+#endif /* defined(REPHIST_PRIVATE) */
 
 /**
  * Represents the type of a cell for padding accounting
diff --git a/src/include.am b/src/include.am
index d2f83da814..065bdc31cb 100644
--- a/src/include.am
+++ b/src/include.am
@@ -1,12 +1,16 @@
 include src/ext/include.am
 include src/lib/arch/include.am
+include src/lib/buf/include.am
 include src/lib/err/include.am
 include src/lib/cc/include.am
 include src/lib/ctime/include.am
 include src/lib/compress/include.am
+include src/lib/conf/include.am
+include src/lib/confmgt/include.am
 include src/lib/container/include.am
 include src/lib/crypt_ops/include.am
 include src/lib/defs/include.am
+include src/lib/dispatch/include.am
 include src/lib/encoding/include.am
 include src/lib/evloop/include.am
 include src/lib/fdio/include.am
@@ -23,8 +27,10 @@ include src/lib/malloc/include.am
 include src/lib/net/include.am
 include src/lib/osinfo/include.am
 include src/lib/process/include.am
+include src/lib/pubsub/include.am
 include src/lib/sandbox/include.am
 include src/lib/string/include.am
+include src/lib/subsys/include.am
 include src/lib/smartlist_core/include.am
 include src/lib/term/include.am
 include src/lib/testsupport/include.am
@@ -32,6 +38,7 @@ include src/lib/thread/include.am
 include src/lib/time/include.am
 include src/lib/tls/include.am
 include src/lib/trace/include.am
+include src/lib/version/include.am
 include src/lib/wallclock/include.am
 include src/trunnel/include.am
 
diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h
index fa82241b28..4756ca2beb 100644
--- a/src/lib/arch/bytes.h
+++ b/src/lib/arch/bytes.h
@@ -129,7 +129,7 @@ tor_ntohll(uint64_t a)
 {
   return a;
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
 static inline uint16_t
 tor_htons(uint16_t a)
 {
@@ -177,6 +177,6 @@ tor_ntohll(uint64_t a)
 {
   return tor_htonll(a);
 }
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
-#endif
+#endif /* !defined(TOR_BYTES_H) */
diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am
index f92ee9222f..c5926c6330 100644
--- a/src/lib/arch/include.am
+++ b/src/lib/arch/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/arch/bytes.h
diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include
new file mode 100644
index 0000000000..c4be73bce2
--- /dev/null
+++ b/src/lib/buf/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/buf/*.h
+lib/cc/*.h
+lib/ctime/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
+lib/log/*.h
+lib/string/*.h
+lib/time/*.h
diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c
index fe4cf7c385..ace5bdc4a4 100644
--- a/src/lib/container/buffers.c
+++ b/src/lib/buf/buffers.c
@@ -25,7 +25,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -158,7 +158,7 @@ chunk_new_with_alloc_size(size_t alloc)
 static inline chunk_t *
 chunk_grow(chunk_t *chunk, size_t sz)
 {
-  off_t offset;
+  ptrdiff_t offset;
   const size_t memlen_orig = chunk->memlen;
   const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
   const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
@@ -440,7 +440,7 @@ chunk_copy(const chunk_t *in_chunk)
 #endif
   newch->next = NULL;
   if (in_chunk->data) {
-    off_t offset = in_chunk->data - in_chunk->mem;
+    ptrdiff_t offset = in_chunk->data - in_chunk->mem;
     newch->data = newch->mem + offset;
   }
   return newch;
@@ -712,7 +712,8 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in)
 /** Internal structure: represents a position in a buffer. */
 typedef struct buf_pos_t {
   const chunk_t *chunk; /**< Which chunk are we pointing to? */
-  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  ptrdiff_t pos;/**< Which character inside the chunk's data are we pointing
+                 * to? */
   size_t chunk_pos; /**< Total length of all previous chunks. */
 } buf_pos_t;
 
@@ -728,15 +729,15 @@ buf_pos_init(const buf_t *buf, buf_pos_t *out)
 /** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
  * position of <b>out</b>, or later.  Return -1 if no instances are found;
  * otherwise returns the absolute position of the character. */
-static off_t
+static ptrdiff_t
 buf_find_pos_of_char(char ch, buf_pos_t *out)
 {
   const chunk_t *chunk;
-  int pos;
+  ptrdiff_t pos;
   tor_assert(out);
   if (out->chunk) {
     if (out->chunk->datalen) {
-      tor_assert(out->pos < (off_t)out->chunk->datalen);
+      tor_assert(out->pos < (ptrdiff_t)out->chunk->datalen);
     } else {
       tor_assert(out->pos == 0);
     }
@@ -764,7 +765,7 @@ buf_pos_inc(buf_pos_t *pos)
 {
   tor_assert(pos->pos < INT_MAX - 1);
   ++pos->pos;
-  if (pos->pos == (off_t)pos->chunk->datalen) {
+  if (pos->pos == (ptrdiff_t)pos->chunk->datalen) {
     if (!pos->chunk->next)
       return -1;
     pos->chunk_pos += pos->chunk->datalen;
@@ -838,11 +839,11 @@ buf_peek_startswith(const buf_t *buf, const char *cmd)
 
 /** Return the index within <b>buf</b> at which <b>ch</b> first appears,
  * or -1 if <b>ch</b> does not appear on buf. */
-static off_t
+static ptrdiff_t
 buf_find_offset_of_char(buf_t *buf, char ch)
 {
   chunk_t *chunk;
-  off_t offset = 0;
+  ptrdiff_t offset = 0;
   tor_assert(buf->datalen < INT_MAX);
   for (chunk = buf->head; chunk; chunk = chunk->next) {
     char *cp = memchr(chunk->data, ch, chunk->datalen);
@@ -865,7 +866,7 @@ int
 buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
 {
   size_t sz;
-  off_t offset;
+  ptrdiff_t offset;
 
   if (!buf->head)
     return 0;
diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h
index c103b93a82..c103b93a82 100644
--- a/src/lib/container/buffers.h
+++ b/src/lib/buf/buffers.h
diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am
new file mode 100644
index 0000000000..27430d1d38
--- /dev/null
+++ b/src/lib/buf/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-buf.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-buf-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_buf_a_SOURCES =			\
+	src/lib/buf/buffers.c
+
+src_lib_libtor_buf_testing_a_SOURCES = \
+	$(src_lib_libtor_buf_a_SOURCES)
+src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/buf/buffers.h
diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include
index 2b06e8519c..fa1478ce46 100644
--- a/src/lib/cc/.may_include
+++ b/src/lib/cc/.may_include
@@ -1 +1,2 @@
 orconfig.h
+lib/cc/*.h
\ No newline at end of file
diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h
index fbe6a38f1f..d9ccabb2b3 100644
--- a/src/lib/cc/compat_compiler.h
+++ b/src/lib/cc/compat_compiler.h
@@ -88,7 +88,7 @@
 #    define ENABLE_GCC_WARNING(warningopt) \
          PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
 #endif /* defined(__clang__) || GCC_VERSION >= 406 */
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 /* not gcc at all */
 # define DISABLE_GCC_WARNING(warning)
 # define ENABLE_GCC_WARNING(warning)
@@ -201,7 +201,7 @@
  * structure <b>st</b>.  Example:
  * <pre>
  *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = offsetof(struct a, bar);
+ *   ptrdiff_t bar_offset = offsetof(struct a, bar);
  *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
  *   *bar_p = 3;
  * </pre>
@@ -223,4 +223,16 @@
 /** Macro: Yields the number of elements in array x. */
 #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
 
-#endif /* !defined(TOR_COMPAT_H) */
+/**
+ * "Eat" a semicolon that somebody puts at the end of a top-level macro.
+ *
+ * Frequently, we want to declare a macro that people will use at file scope,
+ * and we want to allow people to put a semicolon after the macro.
+ *
+ * This declaration of a struct can be repeated any number of times, and takes
+ * a trailing semicolon afterwards.
+ **/
+#define EAT_SEMICOLON                                   \
+  struct dummy_semicolon_eater__
+
+#endif /* !defined(TOR_COMPAT_COMPILER_H) */
diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h
new file mode 100644
index 0000000000..bedf0b83a6
--- /dev/null
+++ b/src/lib/cc/ctassert.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ctassert.h
+ *
+ * \brief Compile-time assertions: CTASSERT(expression).
+ */
+
+#ifndef TOR_CTASSERT_H
+#define TOR_CTASSERT_H
+
+#include "lib/cc/compat_compiler.h"
+
+/**
+ * CTASSERT(expression)
+ *
+ *       Trigger a compiler error if expression is false.
+ */
+#if __STDC_VERSION__ >= 201112L
+
+/* If C11 is available, just use _Static_assert.  */
+#define CTASSERT(x) _Static_assert((x), #x)
+
+#else /* !(__STDC_VERSION__ >= 201112L) */
+
+/*
+ * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__
+ * and __LINE__, or just __LINE__, with an intermediate preprocessor
+ * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the
+ * expansions together into a unique name.
+ *
+ * We use this name as a typedef of an array type with a positive
+ * length if the assertion is true, and a negative length of the
+ * assertion is false, which is invalid and hence triggers a compiler
+ * error.
+ */
+#if defined(__COUNTER__)
+#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__)
+#elif defined(__INCLUDE_LEVEL__)
+#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__)
+#else
+/* hope it's unique enough */
+#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__)
+#endif /* defined(__COUNTER__) || ... */
+
+#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b)
+#define CTASSERT_DECL(x, a, b) \
+  typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED
+
+#endif /* __STDC_VERSION__ >= 201112L */
+
+#endif /* !defined(TOR_CTASSERT_H) */
diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am
index 2ae90f97dd..1aa722dd82 100644
--- a/src/lib/cc/include.am
+++ b/src/lib/cc/include.am
@@ -1,4 +1,6 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/cc/compat_compiler.h \
+	src/lib/cc/ctassert.h \
 	src/lib/cc/torint.h
diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h
index c9b2d329f2..94b79d30a1 100644
--- a/src/lib/cc/torint.h
+++ b/src/lib/cc/torint.h
@@ -96,9 +96,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIuSZ PRIu32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIuSZ "zu"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef _WIN32
 #  ifdef _WIN64
@@ -106,9 +106,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIdSZ PRId32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIdSZ "zd"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef SSIZE_MAX
 #if (SIZEOF_SIZE_T == 4)
@@ -125,4 +125,13 @@ typedef int32_t ssize_t;
 /** Any size_t larger than this amount is likely to be an underflow. */
 #define SIZE_T_CEILING  ((size_t)(SSIZE_MAX-16))
 
+#if SIZEOF_INT > SIZEOF_VOID_P
+#error "sizeof(int) > sizeof(void *) - Tor cannot be built on this platform!"
+#endif
+
+#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P
+#error "sizeof(unsigned int) > sizeof(void *) - Tor cannot be built on this \
+platform!"
+#endif
+
 #endif /* !defined(TOR_TORINT_H) */
diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include
index 68fe9f1c54..6cd80086e6 100644
--- a/src/lib/compress/.may_include
+++ b/src/lib/compress/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/compress/*.h
 lib/container/*.h
@@ -8,5 +9,6 @@ lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c
index 95fd73bb32..51591410a2 100644
--- a/src/lib/compress/compress.c
+++ b/src/lib/compress/compress.c
@@ -29,10 +29,12 @@
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_none.h"
+#include "lib/compress/compress_sys.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
 #include "lib/intmath/cmp.h"
 #include "lib/malloc/malloc.h"
+#include "lib/subsys/subsys.h"
 #include "lib/thread/threads.h"
 
 /** Total number of bytes allocated for compression state overhead. */
@@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state)
 }
 
 /** Initialize all compression modules. */
-void
+int
 tor_compress_init(void)
 {
   atomic_counter_init(&total_compress_allocation);
@@ -668,6 +670,8 @@ tor_compress_init(void)
   tor_zlib_init();
   tor_lzma_init();
   tor_zstd_init();
+
+  return 0;
 }
 
 /** Warn if we had any problems while setting up our compression libraries.
@@ -677,5 +681,20 @@ tor_compress_init(void)
 void
 tor_compress_log_init_warnings(void)
 {
+  // XXXX can we move this into tor_compress_init() after all?  log.c queues
+  // XXXX log messages at startup.
   tor_zstd_warn_if_version_mismatched();
 }
+
+static int
+subsys_compress_initialize(void)
+{
+  return tor_compress_init();
+}
+
+const subsys_fns_t sys_compress = {
+  .name = "compress",
+  .supported = true,
+  .level = -70,
+  .initialize = subsys_compress_initialize,
+};
diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h
index 5f16a2ab27..8cea4ead60 100644
--- a/src/lib/compress/compress.h
+++ b/src/lib/compress/compress.h
@@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state);
 
 size_t tor_compress_state_size(const tor_compress_state_t *state);
 
-void tor_compress_init(void);
+int tor_compress_init(void);
 void tor_compress_log_init_warnings(void);
 
 struct buf_t;
diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c
index 198128b261..2e704466f2 100644
--- a/src/lib/compress/compress_buf.c
+++ b/src/lib/compress/compress_buf.c
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/cc/compat_compiler.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/compress/compress.h"
 #include "lib/log/util_bug.h"
 
diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c
index 2dab37e433..915f4949ae 100644
--- a/src/lib/compress/compress_lzma.c
+++ b/src/lib/compress/compress_lzma.c
@@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress,
   tor_free(result);
   return NULL;
  /* LCOV_EXCL_STOP */
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)compress;
   (void)method;
   (void)level;
@@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state,
                lzma_error_str(retval));
       return TOR_COMPRESS_ERROR;
   }
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)state;
   (void)out;
   (void)out_len;
diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h
new file mode 100644
index 0000000000..6181072315
--- /dev/null
+++ b/src/lib/compress/compress_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_sys.h
+ * \brief Declare subsystem object for the compress module
+ **/
+
+#ifndef TOR_COMPRESS_SYS_H
+#define TOR_COMPRESS_SYS_H
+
+extern const struct subsys_fns_t sys_compress;
+
+#endif /* !defined(TOR_COMPRESS_SYS_H) */
diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c
index 45d0d4d602..9076665295 100644
--- a/src/lib/compress/compress_zstd.c
+++ b/src/lib/compress/compress_zstd.c
@@ -25,7 +25,7 @@
  * all invocations of zstd's static-only functions in a check to make sure
  * that the compile-time version matches the run-time version. */
 #define ZSTD_STATIC_LINKING_ONLY
-#endif
+#endif /* defined(ENABLE_ZSTD_ADVANCED_APIS) */
 
 #ifdef HAVE_ZSTD
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
@@ -35,7 +35,7 @@ DISABLE_GCC_WARNING(unused-const-variable)
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
 ENABLE_GCC_WARNING(unused-const-variable)
 #endif
-#endif
+#endif /* defined(HAVE_ZSTD) */
 
 /** Total number of bytes allocated for Zstandard state. */
 static atomic_counter_t total_zstd_allocation;
@@ -77,7 +77,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
                version_number / 100 % 100,
                version_number % 100);
 }
-#endif
+#endif /* defined(HAVE_ZSTD) */
 
 #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
 
@@ -93,7 +93,7 @@ tor_zstd_get_version_str(void)
                           ZSTD_versionNumber());
 
   return version_str;
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   return NULL;
 #endif /* defined(HAVE_ZSTD) */
 }
@@ -125,9 +125,9 @@ tor_zstd_can_use_static_apis(void)
   }
 #endif
   return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
-#else
+#else /* !(defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)) */
   return 0;
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) */
 }
 
 /** Internal Zstandard state for incremental compression/decompression.
@@ -237,7 +237,7 @@ tor_zstd_state_size_precalc(int compress, int preset)
 #endif
     }
   }
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) */
   return tor_zstd_state_size_precalc_fake(compress, preset);
 }
 #endif /* defined(HAVE_ZSTD) */
@@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress,
   tor_free(result);
   return NULL;
   // LCOV_EXCL_STOP
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)compress;
   (void)method;
   (void)level;
@@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state,
       return TOR_COMPRESS_OK;
   }
 
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)state;
   (void)out;
   (void)out_len;
@@ -527,7 +527,7 @@ tor_zstd_warn_if_version_mismatched(void)
              "For safety, we'll avoid using advanced zstd functionality.",
              header_version, runtime_version);
   }
-#endif
+#endif /* defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) */
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -538,4 +538,4 @@ tor_zstd_set_static_apis_disabled_for_testing(int disabled)
 {
   static_apis_disable_for_testing = disabled;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am
index 75c9032bd2..60dd447d4e 100644
--- a/src/lib/compress/include.am
+++ b/src/lib/compress/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-compress-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_compress_a_SOURCES =		\
 	src/lib/compress/compress.c		\
 	src/lib/compress/compress_buf.c		\
@@ -18,9 +19,11 @@ src_lib_libtor_compress_testing_a_SOURCES = \
 src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS	+=				\
 	src/lib/compress/compress.h		\
 	src/lib/compress/compress_lzma.h	\
 	src/lib/compress/compress_none.h	\
+	src/lib/compress/compress_sys.h		\
 	src/lib/compress/compress_zlib.h	\
 	src/lib/compress/compress_zstd.h
diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include
new file mode 100644
index 0000000000..629e2f897d
--- /dev/null
+++ b/src/lib/conf/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h
new file mode 100644
index 0000000000..68121891f1
--- /dev/null
+++ b/src/lib/conf/confmacros.h
@@ -0,0 +1,67 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file confmacros.h
+ * @brief Macro definitions for declaring configuration variables
+ **/
+
+#ifndef TOR_LIB_CONF_CONFMACROS_H
+#define TOR_LIB_CONF_CONFMACROS_H
+
+#include "orconfig.h"
+#include "lib/conf/conftesting.h"
+
+/**
+ * Used to indicate the end of an array of configuration variables.
+ **/
+#define END_OF_CONFIG_VARS                                      \
+  { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS }
+
+/**
+ * Declare a config_var_t as a member named <b>membername</b> of the structure
+ * <b>structtype</b>, whose user-visible name is <b>varname</b>, whose
+ * type corresponds to the config_type_t member CONFIG_TYPE_<b>vartype</b>,
+ * and whose initial value is <b>intval</b>.
+ *
+ * Most modules that use this macro should wrap it in a local macro that
+ * sets structtype to the local configuration type.
+ **/
+#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername,      \
+                         varflags, initval)                             \
+  { .member =                                                           \
+    { .name = varname,                                                  \
+      .type = CONFIG_TYPE_ ## vartype,                                  \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+/**
+ * As CONFIG_VAR_XTYPE, but declares a value using an extension type whose
+ * type definition is <b>vartype</b>_type_defn.
+ **/
+#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername,       \
+                        varflags, initval)                              \
+  { .member =                                                           \
+    { .name = varname,                                                \
+      .type = CONFIG_TYPE_EXTENDED,                                     \
+      .type_def = &vartype ## _type_defn,                               \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+#define CONFIG_VAR_OBSOLETE(varname)            \
+  { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE },        \
+    .flags = CFLG_GROUP_OBSOLETE                                        \
+  }
+
+#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */
diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h
new file mode 100644
index 0000000000..f01f52d59e
--- /dev/null
+++ b/src/lib/conf/conftesting.h
@@ -0,0 +1,86 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftesting.h
+ * @brief Macro and type declarations for testing
+ **/
+
+#ifndef TOR_LIB_CONF_CONFTESTING_H
+#define TOR_LIB_CONF_CONFTESTING_H
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Union used when building in test mode typechecking the members of a type
+ * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
+ * it is used. */
+typedef union {
+  char **STRING;
+  char **FILENAME;
+  int *POSINT; /* yes, this is really an int, and not an unsigned int.  For
+                * historical reasons, many configuration values are restricted
+                * to the range [0,INT_MAX], and stored in signed ints.
+                */
+  uint64_t *UINT64;
+  int *INT;
+  int *INTERVAL;
+  int *MSEC_INTERVAL;
+  uint64_t *MEMUNIT;
+  double *DOUBLE;
+  int *BOOL;
+  int *AUTOBOOL;
+  time_t *ISOTIME;
+  struct smartlist_t **CSV;
+  int *CSV_INTERVAL;
+  struct config_line_t **LINELIST;
+  struct config_line_t **LINELIST_S;
+  struct config_line_t **LINELIST_V;
+  // XXXX this doesn't belong at this level of abstraction.
+  struct routerset_t **ROUTERSET;
+} confparse_dummy_values_t;
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/* Macros to define extra members inside config_var_t fields, and at the
+ * end of a list of them.
+ */
+#ifdef TOR_UNIT_TESTS
+/* This is a somewhat magic type-checking macro for users of confparse.c.
+ * It initializes a union member "confparse_dummy_values_t.conftype" with
+ * the address of a static member "tp_dummy.member".   This
+ * will give a compiler warning unless the member field is of the correct
+ * type.
+ *
+ * (This warning is mandatory, because a type mismatch here violates the type
+ * compatibility constraint for simple assignment, and requires a diagnostic,
+ * according to the C spec.)
+ *
+ * For example, suppose you say:
+ *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
+ * Then this macro will evaluate to:
+ *     { .STRING = &or_options_t_dummy.Address }
+ * And since confparse_dummy_values_t.STRING has type "char **", that
+ * expression will create a warning unless or_options_t.Address also
+ * has type "char *".
+ */
+#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
+  { . conftype = &tp ## _dummy . member }
+#define CONF_TEST_MEMBERS(tp, conftype, member) \
+  , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member)
+#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL }
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  static tp tp ## _dummy
+
+#else /* !defined(TOR_UNIT_TESTS) */
+
+#define CONF_TEST_MEMBERS(tp, conftype, member)
+/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  struct tor_semicolon_eater
+#define DUMMY_CONF_TEST_MEMBERS
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */
diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h
new file mode 100644
index 0000000000..274065cff2
--- /dev/null
+++ b/src/lib/conf/conftypes.h
@@ -0,0 +1,202 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftypes.h
+ * @brief Types used to specify configurable options.
+ *
+ * This header defines the types that different modules will use in order to
+ * declare their configuration and state variables, and tell the configuration
+ * management code about those variables.  From the individual module's point
+ * of view, its configuration and state are simply data structures.
+ *
+ * For defining new variable types, see var_type_def_st.h.
+ *
+ * For the code that manipulates variables defined via this module, see
+ * lib/confmgt/, especially typedvar.h and (later) structvar.h.  The
+ * configuration manager is responsible for encoding, decoding, and
+ * maintaining the configuration structures used by the various modules.
+ *
+ * STATUS NOTE: This is a work in process refactoring.  It is not yet possible
+ *   for modules to define their own variables, and much of the configuration
+ *   management code is still in src/app/config/.
+ **/
+
+#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H
+#define TOR_SRC_LIB_CONF_CONFTYPES_H
+
+#include "lib/cc/torint.h"
+#ifdef TOR_UNIT_TESTS
+#include "lib/conf/conftesting.h"
+#endif
+
+#include <stddef.h>
+
+/** Enumeration of types which option values can take */
+typedef enum config_type_t {
+  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
+  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
+  CONFIG_TYPE_POSINT,       /**< A non-negative integer less than MAX_INT */
+  CONFIG_TYPE_INT,          /**< Any integer. */
+  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
+  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
+  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
+                              * units */
+  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
+  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
+  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
+  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
+                             * 1 for true, and -1 for auto  */
+  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
+  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
+                              * optional whitespace. */
+  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
+                              * optional whitespace, representing intervals in
+                              * seconds, with optional units.  We allow
+                              * multiple values here for legacy reasons, but
+                              * ignore every value after the first. */
+  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
+  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
+                             * mixed with other keywords. */
+  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
+                             * context-sensitive config lines when fetching.
+                             */
+  CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
+  /**
+   * Extended type: definition appears in the <b>type_def</b> pointer
+   * of the corresponding struct_member_t.
+   *
+   * For some types, we cannot define them as particular values of this
+   * enumeration, since those types are abstractions defined at a higher level
+   * than this module.  (For example, parsing a routerset_t is higher-level
+   * than this module.)  To handle this, we use CONFIG_TYPE_EXTENDED for those
+   * types, and give a definition for them in the struct_member_t.type_def.
+   **/
+  CONFIG_TYPE_EXTENDED,
+} config_type_t;
+
+/* Forward delcaration for var_type_def_t, for extended types. */
+struct var_type_def_t;
+
+/** Structure to specify a named, typed member within a structure. */
+typedef struct struct_member_t {
+  /** Name of the field. */
+  const char *name;
+  /**
+   * Type of the field, according to the config_type_t enumeration.
+   *
+   * For any type not otherwise listed in config_type_t, this field's value
+   * should be CONFIG_TYPE_EXTENDED.  When it is, the <b>type_def</b> pointer
+   * must be set.
+   **/
+  /*
+   * NOTE: In future refactoring, we might remove this field entirely, along
+   * with its corresponding enumeration.  In that case, we will require that
+   * type_def be set in all cases. If we do, we will also need a new mechanism
+   * to enforce consistency between configuration variable types and their
+   * corresponding structures, since our current design in
+   * lib/conf/conftesting.h won't work any more.
+   */
+  config_type_t type;
+  /**
+   * Pointer to a type definition for the type of this field. Overrides
+   * <b>type</b> if it is not NULL.  Must be set when <b>type</b> is
+   * CONFIG_TYPE_EXTENDED.
+   **/
+  const struct var_type_def_t *type_def;
+  /**
+   * Offset of this field within the structure.  Compute this with
+   * offsetof(structure, fieldname).
+   **/
+  ptrdiff_t offset;
+} struct_member_t;
+
+/**
+ * Structure to describe the location and preferred value of a "magic number"
+ * field within a structure.
+ *
+ * These 'magic numbers' are 32-bit values used to tag objects to make sure
+ * that they have the correct type.
+ */
+typedef struct struct_magic_decl_t {
+  /** The name of the structure */
+  const char *typename;
+  /** A value used to recognize instances of this structure. */
+  uint32_t magic_val;
+  /** The location within the structure at which we expect to find
+   * <b>magic_val</b>. */
+  ptrdiff_t magic_offset;
+} struct_magic_decl_t;
+
+/**
+ * Flag to indicate that an option or type is "undumpable". An
+ * undumpable option is never saved to disk.
+ *
+ * For historical reasons its name is usually is prefixed with __.
+ **/
+#define CFLG_NODUMP    (1u<<0)
+/**
+ * Flag to indicate that an option or type is "unlisted".
+ *
+ * We don't tell the controller about unlisted options when it asks for a
+ * list of them.
+ **/
+#define CFLG_NOLIST (1u<<1)
+/**
+ * Flag to indicate that an option or type is "unsettable".
+ *
+ * An unsettable option can never be set directly by name.
+ **/
+#define CFLG_NOSET (1u<<2)
+/**
+ * Flag to indicate that an option or type does not need to be copied when
+ * copying the structure that contains it.
+ *
+ * (Usually, if an option does not need to be copied, then either it contains
+ * no data, or the data that it does contain is completely contained within
+ * another option.)
+ **/
+#define CFLG_NOCOPY (1u<<3)
+/**
+ * Flag to indicate that an option or type does not need to be compared
+ * when telling the controller about the differences between two
+ * configurations.
+ *
+ * (Usually, if an option does not need to be compared, then either it
+ * contains no data, or the data that it does contain is completely contained
+ * within another option.)
+ **/
+#define CFLG_NOCMP (1u<<4)
+/**
+ * Flag to indicate that an option or type should not be replaced when setting
+ * it.
+ *
+ * For most options, setting them replaces their old value.  For some options,
+ * however, setting them appends to their old value.
+ */
+#define CFLG_NOREPLACE    (1u<<5)
+
+/**
+ * A group of flags that should be set on all obsolete options and types.
+ **/
+#define CFLG_GROUP_OBSOLETE \
+  (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST)
+
+/** A variable allowed in the configuration file or on the command line. */
+typedef struct config_var_t {
+  struct_member_t member; /** A struct member corresponding to this
+                           * variable. */
+  const char *initvalue; /**< String (or null) describing initial value. */
+  uint32_t flags; /**< One or more flags describing special handling for this
+                   * variable */
+#ifdef TOR_UNIT_TESTS
+  /** Used for compiler-magic to typecheck the corresponding field in the
+   * corresponding struct. Only used in unit test mode, at compile-time. */
+  confparse_dummy_values_t var_ptr_dummy;
+#endif
+} config_var_t;
+
+#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */
diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am
new file mode 100644
index 0000000000..cb7126184d
--- /dev/null
+++ b/src/lib/conf/include.am
@@ -0,0 +1,6 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/conf/conftesting.h              \
+	src/lib/conf/conftypes.h                \
+	src/lib/conf/confmacros.h
diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include
new file mode 100644
index 0000000000..2564133917
--- /dev/null
+++ b/src/lib/confmgt/.may_include
@@ -0,0 +1,11 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
+lib/confmgt/*.h
+lib/container/*.h
+lib/encoding/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+ext/*.h
diff --git a/src/lib/confmgt/confparse.c b/src/lib/confmgt/confparse.c
new file mode 100644
index 0000000000..08e562f654
--- /dev/null
+++ b/src/lib/confmgt/confparse.c
@@ -0,0 +1,1239 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confparse.c
+ *
+ * \brief Back-end for parsing and generating key-value files, used to
+ *   implement the torrc file format and the state file.
+ *
+ * This module is used by config.c to parse and encode torrc
+ * configuration files, and by statefile.c to parse and encode the
+ * $DATADIR/state file.
+ *
+ * To use this module, its callers provide an instance of
+ * config_format_t to describe the mappings from a set of configuration
+ * options to a number of fields in a C structure.  With this mapping,
+ * the functions here can convert back and forth between the C structure
+ * specified, and a linked list of key-value pairs.
+ */
+
+#define CONFPARSE_PRIVATE
+#include "orconfig.h"
+#include "lib/confmgt/confparse.h"
+
+#include "lib/confmgt/structvar.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include "ext/siphash.h"
+
+/**
+ * A managed_var_t is an internal wrapper around a config_var_t in
+ * a config_format_t structure.  It is used by config_mgr_t to
+ * keep track of which option goes with which structure. */
+typedef struct managed_var_t {
+  /**
+   * A pointer to the config_var_t for this option.
+   */
+  const config_var_t *cvar;
+  /**
+   * The index of the object in which this option is stored.  It is
+   * IDX_TOPLEVEL to indicate that the object is the top-level object.
+   **/
+  int object_idx;
+} managed_var_t;
+
+static void config_reset(const config_mgr_t *fmt, void *options,
+                         const managed_var_t *var, int use_defaults);
+static void config_mgr_register_fmt(config_mgr_t *mgr,
+                                    const config_format_t *fmt,
+                                    int object_idx);
+
+/** Release all storage held in a managed_var_t. */
+static void
+managed_var_free_(managed_var_t *mv)
+{
+  if (!mv)
+    return;
+  tor_free(mv);
+}
+#define managed_var_free(mv) \
+  FREE_AND_NULL(managed_var_t, managed_var_free_, (mv))
+
+struct config_suite_t {
+  /** A list of configuration objects managed by a given configuration
+   * manager. They are stored in the same order as the config_format_t
+   * objects in the manager's list of subformats. */
+  smartlist_t *configs;
+};
+
+/**
+ * Allocate a new empty config_suite_t.
+ **/
+static config_suite_t *
+config_suite_new(void)
+{
+  config_suite_t *suite = tor_malloc_zero(sizeof(config_suite_t));
+  suite->configs = smartlist_new();
+  return suite;
+}
+
+/** Release all storage held by a config_suite_t.  (Does not free
+ * any configuration objects it holds; the caller must do that first.) */
+static void
+config_suite_free_(config_suite_t *suite)
+{
+  if (!suite)
+    return;
+  smartlist_free(suite->configs);
+  tor_free(suite);
+}
+
+#define config_suite_free(suite) \
+  FREE_AND_NULL(config_suite_t, config_suite_free_, (suite))
+
+struct config_mgr_t {
+  /** The 'top-level' configuration format.  This one is used for legacy
+   * options that have not yet been assigned to different sub-modules.
+   *
+   * (NOTE: for now, this is the only config_format_t that a config_mgr_t
+   * contains.  A subsequent commit will add more. XXXX)
+   */
+  const config_format_t *toplevel;
+  /**
+   * List of second-level configuration format objects that this manager
+   * also knows about.
+   */
+  smartlist_t *subconfigs;
+  /** A smartlist of managed_var_t objects for all configuration formats. */
+  smartlist_t *all_vars;
+  /** A smartlist of config_abbrev_t objects for all configuration
+   * formats. These objects are used to track synonyms and abbreviations for
+   * different configuration options. */
+  smartlist_t *all_abbrevs;
+  /** A smartlist of config_deprecation_t for all configuration formats. */
+  smartlist_t *all_deprecations;
+  /** True if this manager has been frozen and cannot have any more formats
+   * added to it. A manager must be frozen before it can be used to construct
+   * or manipulate objects. */
+  bool frozen;
+  /** A replacement for the magic number of the toplevel object. We override
+   * that number to make it unique for this particular config_mgr_t, so that
+   * an object constructed with one mgr can't be used with another, even if
+   * those managers' contents are equal.
+   */
+  struct_magic_decl_t toplevel_magic;
+};
+
+#define IDX_TOPLEVEL (-1)
+
+/** Create a new config_mgr_t to manage a set of configuration objects to be
+ * wrapped under <b>toplevel_fmt</b>. */
+config_mgr_t *
+config_mgr_new(const config_format_t *toplevel_fmt)
+{
+  config_mgr_t *mgr = tor_malloc_zero(sizeof(config_mgr_t));
+  mgr->subconfigs = smartlist_new();
+  mgr->all_vars = smartlist_new();
+  mgr->all_abbrevs = smartlist_new();
+  mgr->all_deprecations = smartlist_new();
+
+  config_mgr_register_fmt(mgr, toplevel_fmt, IDX_TOPLEVEL);
+  mgr->toplevel = toplevel_fmt;
+
+  return mgr;
+}
+
+/** Add a config_format_t to a manager, with a specified (unique) index. */
+static void
+config_mgr_register_fmt(config_mgr_t *mgr,
+                        const config_format_t *fmt,
+                        int object_idx)
+{
+  int i;
+
+  tor_assertf(!mgr->frozen,
+              "Tried to add a format to a configuration manager after "
+              "it had been frozen.");
+
+  if (object_idx != IDX_TOPLEVEL) {
+    tor_assertf(fmt->config_suite_offset < 0,
+          "Tried to register a toplevel format in a non-toplevel position");
+  }
+  tor_assertf(fmt != mgr->toplevel &&
+              ! smartlist_contains(mgr->subconfigs, fmt),
+              "Tried to register an already-registered format.");
+
+  /* register variables */
+  for (i = 0; fmt->vars[i].member.name; ++i) {
+    managed_var_t *mv = tor_malloc_zero(sizeof(managed_var_t));
+    mv->cvar = &fmt->vars[i];
+    mv->object_idx = object_idx;
+    smartlist_add(mgr->all_vars, mv);
+  }
+
+  /* register abbrevs */
+  if (fmt->abbrevs) {
+    for (i = 0; fmt->abbrevs[i].abbreviated; ++i) {
+      smartlist_add(mgr->all_abbrevs, (void*)&fmt->abbrevs[i]);
+    }
+  }
+
+  /* register deprecations. */
+  if (fmt->deprecations) {
+    const config_deprecation_t *d;
+    for (d = fmt->deprecations; d->name; ++d) {
+      smartlist_add(mgr->all_deprecations, (void*)d);
+    }
+  }
+}
+
+/**
+ * Add a new format to this configuration object.  Asserts on failure.
+ *
+ * Returns an internal "index" value used to identify this format within
+ * all of those formats contained in <b>mgr</b>.  This index value
+ * should not generally be used outside of this module.
+ **/
+int
+config_mgr_add_format(config_mgr_t *mgr,
+                      const config_format_t *fmt)
+{
+  tor_assert(mgr);
+  int idx = smartlist_len(mgr->subconfigs);
+  config_mgr_register_fmt(mgr, fmt, idx);
+  smartlist_add(mgr->subconfigs, (void *)fmt);
+  return idx;
+}
+
+/** Return a pointer to the config_suite_t * pointer inside a
+ * configuration object; returns NULL if there is no such member. */
+static inline config_suite_t **
+config_mgr_get_suite_ptr(const config_mgr_t *mgr, void *toplevel)
+{
+  if (mgr->toplevel->config_suite_offset < 0)
+    return NULL;
+  return STRUCT_VAR_P(toplevel, mgr->toplevel->config_suite_offset);
+}
+
+/**
+ * Return a pointer to the configuration object within <b>toplevel</b> whose
+ * index is <b>idx</b>.
+ *
+ * NOTE: XXXX Eventually, there will be multiple objects supported within the
+ * toplevel object.  For example, the or_options_t will contain pointers
+ * to configuration objects for other modules.  This function gets
+ * the sub-object for a particular module.
+ */
+STATIC void *
+config_mgr_get_obj_mutable(const config_mgr_t *mgr, void *toplevel, int idx)
+{
+  tor_assert(mgr);
+  tor_assert(toplevel);
+  if (idx == IDX_TOPLEVEL)
+    return toplevel;
+
+  tor_assertf(idx >= 0 && idx < smartlist_len(mgr->subconfigs),
+              "Index %d is out of range.", idx);
+  config_suite_t **suite = config_mgr_get_suite_ptr(mgr, toplevel);
+  tor_assert(suite);
+  tor_assert(smartlist_len(mgr->subconfigs) ==
+             smartlist_len((*suite)->configs));
+
+  return smartlist_get((*suite)->configs, idx);
+}
+
+/** As config_mgr_get_obj_mutable(), but return a const pointer. */
+STATIC const void *
+config_mgr_get_obj(const config_mgr_t *mgr, const void *toplevel, int idx)
+{
+  return config_mgr_get_obj_mutable(mgr, (void*)toplevel, idx);
+}
+
+/** Sorting helper for smartlist of managed_var_t */
+static int
+managed_var_cmp(const void **a, const void **b)
+{
+  const managed_var_t *mv1 = *(const managed_var_t**)a;
+  const managed_var_t *mv2 = *(const managed_var_t**)b;
+
+  return strcasecmp(mv1->cvar->member.name, mv2->cvar->member.name);
+}
+
+/**
+ * Mark a configuration manager as "frozen", so that no more formats can be
+ * added, and so that it can be used for manipulating configuration objects.
+ **/
+void
+config_mgr_freeze(config_mgr_t *mgr)
+{
+  static uint64_t mgr_count = 0;
+
+  smartlist_sort(mgr->all_vars, managed_var_cmp);
+  memcpy(&mgr->toplevel_magic, &mgr->toplevel->magic,
+         sizeof(struct_magic_decl_t));
+  uint64_t magic_input[3] = { mgr->toplevel_magic.magic_val,
+                              (uint64_t) (uintptr_t) mgr,
+                              ++mgr_count };
+  mgr->toplevel_magic.magic_val =
+    (uint32_t)siphash24g(magic_input, sizeof(magic_input));
+  mgr->frozen = true;
+}
+
+/** Release all storage held in <b>mgr</b> */
+void
+config_mgr_free_(config_mgr_t *mgr)
+{
+  if (!mgr)
+    return;
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, managed_var_free(mv));
+  smartlist_free(mgr->all_vars);
+  smartlist_free(mgr->all_abbrevs);
+  smartlist_free(mgr->all_deprecations);
+  smartlist_free(mgr->subconfigs);
+  memset(mgr, 0, sizeof(*mgr));
+  tor_free(mgr);
+}
+
+/** Return a new smartlist_t containing a config_var_t for every variable that
+ * <b>mgr</b> knows about.  The elements of this smartlist do not need
+ * to be freed; they have the same lifespan as <b>mgr</b>. */
+smartlist_t *
+config_mgr_list_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv,
+                    smartlist_add(result, (void*) mv->cvar));
+  return result;
+}
+
+/** Return a new smartlist_t containing the names of all deprecated variables.
+ * The elements of this smartlist do not need to be freed; they have the same
+ * lifespan as <b>mgr</b>.
+ */
+smartlist_t *
+config_mgr_list_deprecated_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_deprecations, config_deprecation_t *, d,
+                    smartlist_add(result, (char*)d->name));
+  return result;
+}
+
+/** Assert that the magic fields in <b>options</b> and its subsidiary
+ * objects are all okay. */
+static void
+config_mgr_assert_magic_ok(const config_mgr_t *mgr,
+                           const void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+  tor_assert(mgr->frozen);
+  struct_check_magic(options, &mgr->toplevel_magic);
+
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, (void*)options);
+  if (suitep == NULL) {
+    tor_assert(smartlist_len(mgr->subconfigs) == 0);
+    return;
+  }
+
+  tor_assert(smartlist_len((*suitep)->configs) ==
+             smartlist_len(mgr->subconfigs));
+  SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+    void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+    tor_assert(obj);
+    struct_check_magic(obj, &fmt->magic);
+  } SMARTLIST_FOREACH_END(fmt);
+}
+
+/** Macro: assert that <b>cfg</b> has the right magic field for
+ * <b>mgr</b>. */
+#define CONFIG_CHECK(mgr, cfg) STMT_BEGIN                               \
+    config_mgr_assert_magic_ok((mgr), (cfg));                           \
+  STMT_END
+
+/** Allocate an empty configuration object of a given format type. */
+void *
+config_new(const config_mgr_t *mgr)
+{
+  tor_assert(mgr->frozen);
+  void *opts = tor_malloc_zero(mgr->toplevel->size);
+  struct_set_magic(opts, &mgr->toplevel_magic);
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, opts);
+  if (suitep) {
+    *suitep = config_suite_new();
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = tor_malloc_zero(fmt->size);
+      struct_set_magic(obj, &fmt->magic);
+      smartlist_add((*suitep)->configs, obj);
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+  CONFIG_CHECK(mgr, opts);
+  return opts;
+}
+
+/*
+ * Functions to parse config options
+ */
+
+/** If <b>option</b> is an official abbreviation for a longer option,
+ * return the longer option.  Otherwise return <b>option</b>.
+ * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
+ * apply abbreviations that work for the config file and the command line.
+ * If <b>warn_obsolete</b> is set, warn about deprecated names. */
+const char *
+config_expand_abbrev(const config_mgr_t *mgr, const char *option,
+                     int command_line, int warn_obsolete)
+{
+  SMARTLIST_FOREACH_BEGIN(mgr->all_abbrevs, const config_abbrev_t *, abbrev) {
+    /* Abbreviations are case insensitive. */
+    if (!strcasecmp(option, abbrev->abbreviated) &&
+        (command_line || !abbrev->commandline_only)) {
+      if (warn_obsolete && abbrev->warn) {
+        log_warn(LD_CONFIG,
+                 "The configuration option '%s' is deprecated; "
+                 "use '%s' instead.",
+                 abbrev->abbreviated,
+                 abbrev->full);
+      }
+      /* Keep going through the list in case we want to rewrite it more.
+       * (We could imagine recursing here, but I don't want to get the
+       * user into an infinite loop if we craft our list wrong.) */
+      option = abbrev->full;
+    }
+  } SMARTLIST_FOREACH_END(abbrev);
+  return option;
+}
+
+/** If <b>key</b> is a deprecated configuration option, return the message
+ * explaining why it is deprecated (which may be an empty string). Return NULL
+ * if it is not deprecated. The <b>key</b> field must be fully expanded. */
+const char *
+config_find_deprecation(const config_mgr_t *mgr, const char *key)
+{
+  if (BUG(mgr == NULL) || BUG(key == NULL))
+    return NULL; // LCOV_EXCL_LINE
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_deprecations, const config_deprecation_t *,
+                          d) {
+    if (!strcasecmp(d->name, key)) {
+      return d->why_deprecated ? d->why_deprecated : "";
+    }
+  } SMARTLIST_FOREACH_END(d);
+  return NULL;
+}
+
+/**
+ * Find the managed_var_t object for a variable whose name is <b>name</b>
+ * according to <b>mgr</b>.   Return that object, or NULL if none exists.
+ *
+ * If <b>allow_truncated</b> is true, then accept any variable whose
+ * name begins with <b>name</b>.
+ *
+ * If <b>idx_out</b> is not NULL, set *<b>idx_out</b> to the position of
+ * that variable within mgr-&gt;all_vars, or to -1 if the variable is
+ * not found.
+ */
+static const managed_var_t *
+config_mgr_find_var(const config_mgr_t *mgr,
+                    const char *key,
+                    bool allow_truncated, int *idx_out)
+{
+  const size_t keylen = strlen(key);
+  if (idx_out)
+    *idx_out = -1;
+
+  if (!keylen)
+    return NULL; /* if they say "--" on the command line, it's not an option */
+
+  /* First, check for an exact (case-insensitive) match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strcasecmp(mv->cvar->member.name, key)) {
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (!allow_truncated)
+    return NULL;
+
+  /* If none, check for an abbreviated match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strncasecmp(key, mv->cvar->member.name, keylen)) {
+      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
+               "Please use '%s' instead",
+               key, mv->cvar->member.name);
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  /* Okay, unrecognized option */
+  return NULL;
+}
+
+/**
+ * If <b>key</b> is a name or an abbreviation configuration option, return
+ * the corresponding canonical name for it.  Warn if the abbreviation is
+ * non-standard.  Return NULL if the option does not exist.
+ */
+const char *
+config_find_option_name(const config_mgr_t *mgr, const char *key)
+{
+  key = config_expand_abbrev(mgr, key, 0, 0);
+  const managed_var_t *mv = config_mgr_find_var(mgr, key, true, NULL);
+  if (mv)
+    return mv->cvar->member.name;
+  else
+    return NULL;
+}
+
+/** Return the number of option entries in <b>fmt</b>. */
+static int
+config_count_options(const config_mgr_t *mgr)
+{
+  return smartlist_len(mgr->all_vars);
+}
+
+/**
+ * Return true iff at least one bit from <b>flag</b> is set on <b>var</b>,
+ * either in <b>var</b>'s flags, or on the flags of its type.
+ **/
+static bool
+config_var_has_flag(const config_var_t *var, uint32_t flag)
+{
+  uint32_t have_flags = var->flags | struct_var_get_flags(&var->member);
+
+  return (have_flags & flag) != 0;
+}
+
+/**
+ * Return true if assigning a value to <b>var</b> replaces the previous
+ * value.  Return false if assigning a value to <b>var</b> appends
+ * to the previous value.
+ **/
+static bool
+config_var_is_replaced_on_set(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOREPLACE);
+}
+
+/**
+ * Return true iff <b>var</b> may be assigned by name (e.g., via the
+ * CLI, the configuration files, or the controller API).
+ **/
+bool
+config_var_is_settable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOSET);
+}
+
+/**
+ * Return true iff the controller is allowed to fetch the value of
+ * <b>var</b>.
+ **/
+static bool
+config_var_is_gettable(const config_var_t *var)
+{
+  /* Arguably, invisible or obsolete options should not be gettable.  However,
+   * they have been gettable for a long time, and making them ungettable could
+   * have compatibility effects.  For now, let's leave them alone.
+   */
+
+  // return ! config_var_has_flag(var, CVFLAG_OBSOLETE|CFGLAGS_INVISIBLE);
+  (void)var;
+  return true;
+}
+
+/**
+ * Return true iff we need to check <b>var</b> for changes when we are
+ * comparing config options for changes.
+ *
+ * A false result might mean that the variable is a derived variable, and that
+ * comparing the variable it derives from compares this one too-- or it might
+ * mean that there is no data to compare.
+ **/
+static bool
+config_var_should_list_changes(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCMP);
+}
+
+/**
+ * Return true iff we need to copy the data for <b>var</b> when we are
+ * copying a config option.
+ *
+ * A false option might mean that the variable is a derived variable, and that
+ * copying the variable it derives from copies it-- or it might mean that
+ * there is no data to copy.
+ **/
+static bool
+config_var_needs_copy(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCOPY);
+}
+
+/**
+ * Return true iff variable <b>var</b> should appear on list of variable
+ * names given to the controller or the CLI.
+ *
+ * (Note that this option is imperfectly obeyed. The
+ * --list-torrc-options command looks at the "settable" flag, whereas
+ * "GETINFO config/defaults" and "list_deprecated_*()" do not filter
+ * their results. It would be good for consistency to try to converge
+ * these behaviors in the future.)
+ **/
+bool
+config_var_is_listable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOLIST);
+}
+
+/**
+ * Return true iff variable <b>var</b> should be written out when we
+ * are writing our configuration to disk, to a controller, or via the
+ * --dump-config command.
+ *
+ * This option may be set because a variable is hidden, or because it is
+ * derived from another variable which will already be written out.
+ **/
+static bool
+config_var_is_dumpable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NODUMP);
+}
+
+/*
+ * Functions to assign config options.
+ */
+
+/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
+ * with <b>c</b>-\>value and return 0, or return -1 if bad value.
+ *
+ * Called from config_assign_line() and option_reset().
+ */
+static int
+config_assign_value(const config_mgr_t *mgr, void *options,
+                    config_line_t *c, char **msg)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, c->key, true, NULL);
+  tor_assert(var);
+  tor_assert(!strcmp(c->key, var->cvar->member.name));
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+
+  return struct_var_kvassign(object, c, msg, &var->cvar->member);
+}
+
+/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
+ * to it will replace old ones. */
+static void
+config_mark_lists_fragile(const config_mgr_t *mgr, void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    void *object = config_mgr_get_obj_mutable(mgr, options, mv->object_idx);
+    struct_var_mark_fragile(object, &mv->cvar->member);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/**
+ * Log a warning that declaring that the option called <b>what</b>
+ * is deprecated because of the reason in <b>why</b>.
+ *
+ * (Both arguments must be non-NULL.)
+ **/
+void
+warn_deprecated_option(const char *what, const char *why)
+{
+  const char *space = (why && strlen(why)) ? " " : "";
+  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
+           "be removed in a future version of Tor.%s%s (If you think this is "
+           "a mistake, please let us know!)",
+           what, space, why);
+}
+
+/** If <b>c</b> is a syntactically valid configuration line, update
+ * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
+ * key, -2 for bad value.
+ *
+ * If <b>clear_first</b> is set, clear the value first. Then if
+ * <b>use_defaults</b> is set, set the value to the default.
+ *
+ * Called from config_assign().
+ */
+static int
+config_assign_line(const config_mgr_t *mgr, void *options,
+                   config_line_t *c, unsigned flags,
+                   bitarray_t *options_seen, char **msg)
+{
+  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
+  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
+  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
+  const managed_var_t *mvar;
+
+  CONFIG_CHECK(mgr, options);
+
+  int var_index = -1;
+  mvar = config_mgr_find_var(mgr, c->key, true, &var_index);
+  if (!mvar) {
+    const config_format_t *fmt = mgr->toplevel;
+    if (fmt->extra) {
+      void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset);
+      log_info(LD_CONFIG,
+               "Found unrecognized option '%s'; saving it.", c->key);
+      config_line_append((config_line_t**)lvalue, c->key, c->value);
+      return 0;
+    } else {
+      tor_asprintf(msg,
+                "Unknown option '%s'.  Failing.", c->key);
+      return -1;
+    }
+  }
+
+  const config_var_t *cvar = mvar->cvar;
+  tor_assert(cvar);
+
+  /* Put keyword into canonical case. */
+  if (strcmp(cvar->member.name, c->key)) {
+    tor_free(c->key);
+    c->key = tor_strdup(cvar->member.name);
+  }
+
+  const char *deprecation_msg;
+  if (warn_deprecations &&
+      (deprecation_msg = config_find_deprecation(mgr, cvar->member.name))) {
+    warn_deprecated_option(cvar->member.name, deprecation_msg);
+  }
+
+  if (!strlen(c->value)) {
+    /* reset or clear it, then return */
+    if (!clear_first) {
+      if (! config_var_is_replaced_on_set(cvar) &&
+          c->command != CONFIG_LINE_CLEAR) {
+        /* We got an empty linelist from the torrc or command line.
+           As a special case, call this an error. Warn and ignore. */
+        log_warn(LD_CONFIG,
+                 "Linelist option '%s' has no value. Skipping.", c->key);
+      } else { /* not already cleared */
+        config_reset(mgr, options, mvar, use_defaults);
+      }
+    }
+    return 0;
+  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
+    // This block is unreachable, since a CLEAR line always has an
+    // empty value, and so will trigger be handled by the previous
+    // "if (!strlen(c->value))" block.
+
+    // LCOV_EXCL_START
+    tor_assert_nonfatal_unreached();
+    config_reset(mgr, options, mvar, use_defaults);
+    // LCOV_EXCL_STOP
+  }
+
+  if (options_seen && config_var_is_replaced_on_set(cvar)) {
+    /* We're tracking which options we've seen, and this option is not
+     * supposed to occur more than once. */
+    tor_assert(var_index >= 0);
+    if (bitarray_is_set(options_seen, var_index)) {
+      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
+               "value will be ignored.", cvar->member.name);
+    }
+    bitarray_set(options_seen, var_index);
+  }
+
+  if (config_assign_value(mgr, options, c, msg) < 0)
+    return -2;
+  return 0;
+}
+
+/** Restore the option named <b>key</b> in options to its default value.
+ * Called from config_assign(). */
+STATIC void
+config_reset_line(const config_mgr_t *mgr, void *options,
+                  const char *key, int use_defaults)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var)
+    return; /* give error on next pass. */
+
+  config_reset(mgr, options, var, use_defaults);
+}
+
+/** Return true iff value needs to be quoted and escaped to be used in
+ * a configuration file. */
+static int
+config_value_needs_escape(const char *value)
+{
+  if (*value == '\"')
+    return 1;
+  while (*value) {
+    switch (*value)
+    {
+    case '\r':
+    case '\n':
+    case '#':
+      /* Note: quotes and backspaces need special handling when we are using
+       * quotes, not otherwise, so they don't trigger escaping on their
+       * own. */
+      return 1;
+    default:
+      if (!TOR_ISPRINT(*value))
+        return 1;
+    }
+    ++value;
+  }
+  return 0;
+}
+
+/** Return newly allocated line or lines corresponding to <b>key</b> in the
+ * configuration <b>options</b>.  If <b>escape_val</b> is true and a
+ * value needs to be quoted before it's put in a config file, quote and
+ * escape that value. Return NULL if no such key exists. */
+config_line_t *
+config_get_assigned_option(const config_mgr_t *mgr, const void *options,
+                           const char *key, int escape_val)
+{
+  const managed_var_t *var;
+  config_line_t *result;
+
+  tor_assert(options && key);
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var) {
+    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
+    return NULL;
+  }
+  if (! config_var_is_gettable(var->cvar)) {
+    log_warn(LD_CONFIG, "Option '%s' is obsolete or unfetchable. Failing.",
+             key);
+    return NULL;
+  }
+  const void *object = config_mgr_get_obj(mgr, options, var->object_idx);
+
+  result = struct_var_kvencode(object, &var->cvar->member);
+
+  if (escape_val) {
+    config_line_t *line;
+    for (line = result; line; line = line->next) {
+      if (line->value && config_value_needs_escape(line->value)) {
+        char *newval = esc_for_log(line->value);
+        tor_free(line->value);
+        line->value = newval;
+      }
+    }
+  }
+
+  return result;
+}
+/** Iterate through the linked list of requested options <b>list</b>.
+ * For each item, convert as appropriate and assign to <b>options</b>.
+ * If an item is unrecognized, set *msg and return -1 immediately,
+ * else return 0 for success.
+ *
+ * If <b>clear_first</b>, interpret config options as replacing (not
+ * extending) their previous values. If <b>clear_first</b> is set,
+ * then <b>use_defaults</b> to decide if you set to defaults after
+ * clearing, or make the value 0 or NULL.
+ *
+ * Here are the use cases:
+ * 1. A non-empty AllowInvalid line in your torrc. Appends to current
+ *    if linelist, replaces current if csv.
+ * 2. An empty AllowInvalid line in your torrc. Should clear it.
+ * 3. "RESETCONF AllowInvalid" sets it to default.
+ * 4. "SETCONF AllowInvalid" makes it NULL.
+ * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
+ *
+ * Use_defaults   Clear_first
+ *    0                0       "append"
+ *    1                0       undefined, don't use
+ *    0                1       "set to null first"
+ *    1                1       "set to defaults first"
+ * Return 0 on success, -1 on bad key, -2 on bad value.
+ *
+ * As an additional special case, if a LINELIST config option has
+ * no value and clear_first is 0, then warn and ignore it.
+ */
+
+/*
+There are three call cases for config_assign() currently.
+
+Case one: Torrc entry
+options_init_from_torrc() calls config_assign(0, 0)
+  calls config_assign_line(0, 0).
+    if value is empty, calls config_reset(0) and returns.
+    calls config_assign_value(), appends.
+
+Case two: setconf
+options_trial_assign() calls config_assign(0, 1)
+  calls config_reset_line(0)
+    calls config_reset(0)
+      calls option_clear().
+  calls config_assign_line(0, 1).
+    if value is empty, returns.
+    calls config_assign_value(), appends.
+
+Case three: resetconf
+options_trial_assign() calls config_assign(1, 1)
+  calls config_reset_line(1)
+    calls config_reset(1)
+      calls option_clear().
+      calls config_assign_value(default)
+  calls config_assign_line(1, 1).
+    returns.
+*/
+int
+config_assign(const config_mgr_t *mgr, void *options, config_line_t *list,
+              unsigned config_assign_flags, char **msg)
+{
+  config_line_t *p;
+  bitarray_t *options_seen;
+  const int n_options = config_count_options(mgr);
+  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
+  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
+
+  CONFIG_CHECK(mgr, options);
+
+  /* pass 1: normalize keys */
+  for (p = list; p; p = p->next) {
+    const char *full = config_expand_abbrev(mgr, p->key, 0, 1);
+    if (strcmp(full,p->key)) {
+      tor_free(p->key);
+      p->key = tor_strdup(full);
+    }
+  }
+
+  /* pass 2: if we're reading from a resetting source, clear all
+   * mentioned config options, and maybe set to their defaults. */
+  if (clear_first) {
+    for (p = list; p; p = p->next)
+      config_reset_line(mgr, options, p->key, use_defaults);
+  }
+
+  options_seen = bitarray_init_zero(n_options);
+  /* pass 3: assign. */
+  while (list) {
+    int r;
+    if ((r=config_assign_line(mgr, options, list, config_assign_flags,
+                              options_seen, msg))) {
+      bitarray_free(options_seen);
+      return r;
+    }
+    list = list->next;
+  }
+  bitarray_free(options_seen);
+
+  /** Now we're done assigning a group of options to the configuration.
+   * Subsequent group assignments should _replace_ linelists, not extend
+   * them. */
+  config_mark_lists_fragile(mgr, options);
+
+  return 0;
+}
+
+/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
+ * Called from config_reset() and config_free(). */
+static void
+config_clear(const config_mgr_t *mgr, void *options, const managed_var_t *var)
+{
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+  struct_var_free(object, &var->cvar->member);
+}
+
+/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
+ * <b>use_defaults</b>, set it to its default value.
+ * Called by config_init() and option_reset_line() and option_assign_line(). */
+static void
+config_reset(const config_mgr_t *mgr, void *options,
+             const managed_var_t *var, int use_defaults)
+{
+  config_line_t *c;
+  char *msg = NULL;
+  CONFIG_CHECK(mgr, options);
+  config_clear(mgr, options, var); /* clear it first */
+
+  if (!use_defaults)
+    return; /* all done */
+
+  if (var->cvar->initvalue) {
+    c = tor_malloc_zero(sizeof(config_line_t));
+    c->key = tor_strdup(var->cvar->member.name);
+    c->value = tor_strdup(var->cvar->initvalue);
+    if (config_assign_value(mgr, options, c, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Failed to assign default: %s", msg);
+      tor_free(msg); /* if this happens it's a bug */
+      // LCOV_EXCL_STOP
+    }
+    config_free_lines(c);
+  }
+}
+
+/** Release storage held by <b>options</b>. */
+void
+config_free_(const config_mgr_t *mgr, void *options)
+{
+  if (!options)
+    return;
+
+  tor_assert(mgr);
+
+  if (mgr->toplevel->clear_fn) {
+    mgr->toplevel->clear_fn(mgr, options);
+  }
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, options);
+  if (suitep) {
+    tor_assert(smartlist_len((*suitep)->configs) ==
+               smartlist_len(mgr->subconfigs));
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+      if (fmt->clear_fn) {
+        fmt->clear_fn(mgr, obj);
+      }
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    config_clear(mgr, options, mv);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (mgr->toplevel->extra) {
+    config_line_t **linep = STRUCT_VAR_P(options,
+                                         mgr->toplevel->extra->offset);
+    config_free_lines(*linep);
+    *linep = NULL;
+  }
+
+  if (suitep) {
+    SMARTLIST_FOREACH((*suitep)->configs, void *, obj, tor_free(obj));
+    config_suite_free(*suitep);
+  }
+
+  tor_free(options);
+}
+
+/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
+ * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
+ */
+int
+config_is_same(const config_mgr_t *mgr,
+               const void *o1, const void *o2,
+               const char *name)
+{
+  CONFIG_CHECK(mgr, o1);
+  CONFIG_CHECK(mgr, o2);
+
+  const managed_var_t *var = config_mgr_find_var(mgr, name, true, NULL);
+  if (!var) {
+    return true;
+  }
+  const void *obj1 = config_mgr_get_obj(mgr, o1, var->object_idx);
+  const void *obj2 = config_mgr_get_obj(mgr, o2, var->object_idx);
+
+  return struct_var_eq(obj1, obj2, &var->cvar->member);
+}
+
+/**
+ * Return a list of the options which have changed between <b>options1</b> and
+ * <b>options2</b>. If an option has reverted to its default value, it has a
+ * value entry of NULL.
+ *
+ * <b>options1</b> and <b>options2</b> must be top-level configuration objects
+ * of the type managed by <b>mgr</b>.
+ **/
+config_line_t *
+config_get_changes(const config_mgr_t *mgr,
+                   const void *options1, const void *options2)
+{
+  config_line_t *result = NULL;
+  config_line_t **next = &result;
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_should_list_changes(mv->cvar)) {
+      /* something else will check this var, or it doesn't need checking */
+      continue;
+    }
+    const void *obj1 = config_mgr_get_obj(mgr, options1, mv->object_idx);
+    const void *obj2 = config_mgr_get_obj(mgr, options2, mv->object_idx);
+
+    if (struct_var_eq(obj1, obj2, &mv->cvar->member)) {
+      continue;
+    }
+
+    const char *varname = mv->cvar->member.name;
+    config_line_t *line =
+      config_get_assigned_option(mgr, options2, varname, 1);
+
+    if (line) {
+      *next = line;
+    } else {
+      *next = tor_malloc_zero(sizeof(config_line_t));
+      (*next)->key = tor_strdup(varname);
+    }
+    while (*next)
+      next = &(*next)->next;
+  } SMARTLIST_FOREACH_END(mv);
+
+  return result;
+}
+
+/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
+void *
+config_dup(const config_mgr_t *mgr, const void *old)
+{
+  void *newopts;
+
+  newopts = config_new(mgr);
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_needs_copy(mv->cvar)) {
+      // Something else will copy this option, or it doesn't need copying.
+      continue;
+    }
+    const void *oldobj = config_mgr_get_obj(mgr, old, mv->object_idx);
+    void *newobj = config_mgr_get_obj_mutable(mgr, newopts, mv->object_idx);
+    if (struct_var_copy(newobj, oldobj, &mv->cvar->member) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Unable to copy value for %s.",
+              mv->cvar->member.name);
+      tor_assert_unreached();
+      // LCOV_EXCL_STOP
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return newopts;
+}
+/** Set all vars in the configuration object <b>options</b> to their default
+ * values. */
+void
+config_init(const config_mgr_t *mgr, void *options)
+{
+  CONFIG_CHECK(mgr, options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!mv->cvar->initvalue)
+      continue; /* defaults to NULL or 0 */
+    config_reset(mgr, options, mv, 1);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/** Allocate and return a new string holding the written-out values of the vars
+ * in 'options'.  If 'minimal', do not write out any default-valued vars.
+ * Else, if comment_defaults, write default values as comments.
+ */
+char *
+config_dump(const config_mgr_t *mgr, const void *default_options,
+            const void *options, int minimal,
+            int comment_defaults)
+{
+  const config_format_t *fmt = mgr->toplevel;
+  smartlist_t *elements;
+  const void *defaults = default_options;
+  void *defaults_tmp = NULL;
+  config_line_t *line, *assigned;
+  char *result;
+  char *msg = NULL;
+
+  if (defaults == NULL) {
+    defaults = defaults_tmp = config_new(mgr);
+    config_init(mgr, defaults_tmp);
+  }
+
+  /* XXX use a 1 here so we don't add a new log line while dumping */
+  if (default_options == NULL) {
+    if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Failed to validate default config: %s", msg);
+      tor_free(msg);
+      tor_assert(0);
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  elements = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    int comment_option = 0;
+    /* Don't save 'hidden' control variables. */
+    if (! config_var_is_dumpable(mv->cvar))
+      continue;
+    const char *name = mv->cvar->member.name;
+    if (minimal && config_is_same(mgr, options, defaults, name))
+      continue;
+    else if (comment_defaults &&
+             config_is_same(mgr, options, defaults, name))
+      comment_option = 1;
+
+    line = assigned =
+      config_get_assigned_option(mgr, options, name, 1);
+
+    for (; line; line = line->next) {
+      if (!strcmpstart(line->key, "__")) {
+        /* This check detects "hidden" variables inside LINELIST_V structures.
+         */
+        continue;
+      }
+      smartlist_add_asprintf(elements, "%s%s %s\n",
+                   comment_option ? "# " : "",
+                   line->key, line->value);
+    }
+    config_free_lines(assigned);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (fmt->extra) {
+    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset);
+    for (; line; line = line->next) {
+      smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value);
+    }
+  }
+
+  result = smartlist_join_strings(elements, "", 0, NULL);
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+  config_free(mgr, defaults_tmp);
+  return result;
+}
+
+/**
+ * Return true if every member of <b>options</b> is in-range and well-formed.
+ * Return false otherwise.  Log errors at level <b>severity</b>.
+ */
+bool
+config_check_ok(const config_mgr_t *mgr, const void *options, int severity)
+{
+  bool all_ok = true;
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!struct_var_ok(options, &mv->cvar->member)) {
+      log_fn(severity, LD_BUG, "Invalid value for %s",
+             mv->cvar->member.name);
+      all_ok = false;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return all_ok;
+}
diff --git a/src/lib/confmgt/confparse.h b/src/lib/confmgt/confparse.h
new file mode 100644
index 0000000000..2332f69790
--- /dev/null
+++ b/src/lib/confmgt/confparse.h
@@ -0,0 +1,212 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confparse.h
+ *
+ * \brief Header for confparse.c.
+ */
+
+#ifndef TOR_CONFPARSE_H
+#define TOR_CONFPARSE_H
+
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confmacros.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * An abbreviation or alias for a configuration option.
+ **/
+typedef struct config_abbrev_t {
+  /** The option name as abbreviated.  Not case-sensitive. */
+  const char *abbreviated;
+  /** The full name of the option. Not case-sensitive. */
+  const char *full;
+  /** True if this abbreviation should only be allowed on the command line. */
+  int commandline_only;
+  /** True if we should warn whenever this abbreviation is used. */
+  int warn;
+} config_abbrev_t;
+
+/**
+ * A note that a configuration option is deprecated, with an explanation why.
+ */
+typedef struct config_deprecation_t {
+  /** The option that is deprecated. */
+  const char *name;
+  /** A user-facing string explaining why the option is deprecated. */
+  const char *why_deprecated;
+} config_deprecation_t;
+
+/**
+ * Handy macro for declaring "In the config file or on the command line, you
+ * can abbreviate <b>tok</b>s as <b>tok</b>". Used inside an array of
+ * config_abbrev_t.
+ *
+ * For example, to declare "NumCpu" as an abbreviation for "NumCPUs",
+ * you can say PLURAL(NumCpu).
+ **/
+#define PLURAL(tok) { #tok, #tok "s", 0, 0 }
+
+/**
+ * Type of a callback to validate whether a given configuration is
+ * well-formed and consistent.
+ *
+ * The configuration to validate is passed as <b>newval</b>. The previous
+ * configuration, if any, is provided in <b>oldval</b>.  The
+ * <b>default_val</b> argument receives a configuration object initialized
+ * with default values for all its fields.  The <b>from_setconf</b> argument
+ * is true iff the input comes from a SETCONF controller command.
+ *
+ * On success, return 0.  On failure, set *<b>msg_out</b> to a newly allocated
+ * error message, and return -1.
+ *
+ * REFACTORING NOTE: Currently, this callback type is only used from inside
+ * config_dump(); later in our refactoring, it will be cleaned up and used
+ * more generally.
+ */
+typedef int (*validate_fn_t)(void *oldval,
+                             void *newval,
+                             void *default_val,
+                             int from_setconf,
+                             char **msg_out);
+
+struct config_mgr_t;
+
+/**
+ * Callback to clear all non-managed fields of a configuration object.
+ *
+ * <b>obj</b> is the configuration object whose non-managed fields should be
+ * cleared.
+ *
+ * (Regular fields get cleared by config_reset(), but you might have fields
+ * in the object that do not correspond to configuration variables.  If those
+ * fields need to be cleared or freed, this is where to do it.)
+ */
+typedef void (*clear_cfg_fn_t)(const struct config_mgr_t *mgr, void *obj);
+
+/** Information on the keys, value types, key-to-struct-member mappings,
+ * variable descriptions, validation functions, and abbreviations for a
+ * configuration or storage format. */
+typedef struct config_format_t {
+  size_t size; /**< Size of the struct that everything gets parsed into. */
+  struct_magic_decl_t magic; /**< Magic number info for this struct. */
+  const config_abbrev_t *abbrevs; /**< List of abbreviations that we expand
+                             * when parsing this format. */
+  const config_deprecation_t *deprecations; /** List of deprecated options */
+  const config_var_t *vars; /**< List of variables we recognize, their default
+                             * values, and where we stick them in the
+                             * structure. */
+  validate_fn_t validate_fn; /**< Function to validate config. */
+  clear_cfg_fn_t clear_fn; /**< Function to clear the configuration. */
+  /** If present, extra denotes a LINELIST variable for unrecognized
+   * lines.  Otherwise, unrecognized lines are an error. */
+  const struct_member_t *extra;
+  /** The position of a config_suite_t pointer within the toplevel object,
+   * or -1 if there is no such pointer. */
+  ptrdiff_t config_suite_offset;
+} config_format_t;
+
+/**
+ * A collection of config_format_t objects to describe several objects
+ * that are all configured with the same configuration file.
+ *
+ * (NOTE: for now, this only handles a single config_format_t.)
+ **/
+typedef struct config_mgr_t config_mgr_t;
+
+config_mgr_t *config_mgr_new(const config_format_t *toplevel_fmt);
+void config_mgr_free_(config_mgr_t *mgr);
+int config_mgr_add_format(config_mgr_t *mgr,
+                          const config_format_t *fmt);
+void config_mgr_freeze(config_mgr_t *mgr);
+#define config_mgr_free(mgr) \
+  FREE_AND_NULL(config_mgr_t, config_mgr_free_, (mgr))
+struct smartlist_t *config_mgr_list_vars(const config_mgr_t *mgr);
+struct smartlist_t *config_mgr_list_deprecated_vars(const config_mgr_t *mgr);
+
+/** A collection of managed configuration objects. */
+typedef struct config_suite_t config_suite_t;
+
+/**
+ * Flag for config_assign: if set, then "resetting" an option changes it to
+ * its default value, as specified in the config_var_t.  Otherwise,
+ * "resetting" an option changes it to a type-dependent null value --
+ * typically 0 or NULL.
+ *
+ * (An option is "reset" when it is set to an empty value, or as described in
+ * CAL_CLEAR_FIRST).
+ **/
+#define CAL_USE_DEFAULTS      (1u<<0)
+/**
+ * Flag for config_assign: if set, then we reset every provided config
+ * option before we set it.
+ *
+ * For example, if this flag is not set, then passing a multi-line option to
+ * config_assign will cause any previous value to be extended. But if this
+ * flag is set, then a multi-line option will replace any previous value.
+ **/
+#define CAL_CLEAR_FIRST       (1u<<1)
+/**
+ * Flag for config_assign: if set, we warn about deprecated options.
+ **/
+#define CAL_WARN_DEPRECATIONS (1u<<2)
+
+void *config_new(const config_mgr_t *fmt);
+void config_free_(const config_mgr_t *fmt, void *options);
+#define config_free(mgr, options) do {                \
+    config_free_((mgr), (options));                   \
+    (options) = NULL;                                 \
+  } while (0)
+
+struct config_line_t *config_get_assigned_option(const config_mgr_t *mgr,
+                                          const void *options, const char *key,
+                                          int escape_val);
+int config_is_same(const config_mgr_t *fmt,
+                   const void *o1, const void *o2,
+                   const char *name);
+struct config_line_t *config_get_changes(const config_mgr_t *mgr,
+                                  const void *options1, const void *options2);
+void config_init(const config_mgr_t *mgr, void *options);
+void *config_dup(const config_mgr_t *mgr, const void *old);
+char *config_dump(const config_mgr_t *mgr, const void *default_options,
+                  const void *options, int minimal,
+                  int comment_defaults);
+bool config_check_ok(const config_mgr_t *mgr, const void *options,
+                     int severity);
+int config_assign(const config_mgr_t *mgr, void *options,
+                  struct config_line_t *list,
+                  unsigned flags, char **msg);
+const char *config_find_deprecation(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_find_option_name(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_expand_abbrev(const config_mgr_t *mgr,
+                                 const char *option,
+                                 int command_line, int warn_obsolete);
+void warn_deprecated_option(const char *what, const char *why);
+
+bool config_var_is_settable(const config_var_t *var);
+bool config_var_is_listable(const config_var_t *var);
+
+/* Helper macros to compare an option across two configuration objects */
+#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
+#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
+#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
+#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
+
+#ifdef CONFPARSE_PRIVATE
+STATIC void config_reset_line(const config_mgr_t *mgr, void *options,
+                              const char *key, int use_defaults);
+STATIC void *config_mgr_get_obj_mutable(const config_mgr_t *mgr,
+                                        void *toplevel, int idx);
+STATIC const void *config_mgr_get_obj(const config_mgr_t *mgr,
+                                       const void *toplevel, int idx);
+#endif /* defined(CONFPARSE_PRIVATE) */
+
+#endif /* !defined(TOR_CONFPARSE_H) */
diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am
new file mode 100644
index 0000000000..81cd868e5e
--- /dev/null
+++ b/src/lib/confmgt/include.am
@@ -0,0 +1,27 @@
+noinst_LIBRARIES += src/lib/libtor-confmgt.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_confmgt_a_SOURCES =			\
+	src/lib/confmgt/confparse.c			\
+	src/lib/confmgt/structvar.c                      \
+	src/lib/confmgt/type_defs.c                      \
+	src/lib/confmgt/typedvar.c                      \
+	src/lib/confmgt/unitparse.c
+
+src_lib_libtor_confmgt_testing_a_SOURCES = \
+	$(src_lib_libtor_confmgt_a_SOURCES)
+src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/confmgt/confparse.h			\
+	src/lib/confmgt/structvar.h			\
+	src/lib/confmgt/type_defs.h			\
+	src/lib/confmgt/typedvar.h			\
+	src/lib/confmgt/unitparse.h			\
+	src/lib/confmgt/var_type_def_st.h
diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c
new file mode 100644
index 0000000000..7a3b8c7df2
--- /dev/null
+++ b/src/lib/confmgt/structvar.c
@@ -0,0 +1,221 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.c
+ * @brief Functions to manipulate named and typed elements of
+ *    a structure.
+ *
+ * These functions represent a low-level API for accessing a member of a
+ * structure.  They use typedvar.c to work, and they are used in turn by the
+ * configuration system to examine and set fields in configuration objects
+ * used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/structvar.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/log/util_bug.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+
+/**
+ * Set the 'magic number' on <b>object</b> to correspond to decl.
+ **/
+void
+struct_set_magic(void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(object);
+  tor_assert(decl);
+  uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  *ptr = decl->magic_val;
+}
+
+/**
+ * Assert that the 'magic number' on <b>object</b> to corresponds to decl.
+ **/
+void
+struct_check_magic(const void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(object);
+  tor_assert(decl);
+
+  const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  tor_assertf(*ptr == decl->magic_val,
+              "Bad magic number on purported %s object. "
+              "Expected %"PRIu32"x but got %"PRIu32"x.",
+              decl->typename, decl->magic_val, *ptr);
+}
+
+/**
+ * Return a mutable pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+void *
+struct_get_mptr(void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Return a const pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+const void *
+struct_get_ptr(const void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Helper: given a struct_member_t, look up the type definition for its
+ * variable.
+ */
+static const var_type_def_t *
+get_type_def(const struct_member_t *member)
+{
+  if (member->type_def)
+    return member->type_def;
+
+  return lookup_type_def(member->type);
+}
+
+/**
+ * (As typed_var_free, but free and clear the member of <b>object</b> defined
+ * by <b>member</b>.)
+ **/
+void
+struct_var_free(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  typed_var_free(p, def);
+}
+
+/**
+ * (As typed_var_copy, but copy from <b>src</b> to <b>dest</b> the member
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_copy(void *dest, const void *src, const struct_member_t *member)
+{
+  void *p_dest = struct_get_mptr(dest, member);
+  const void *p_src = struct_get_ptr(src, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_copy(p_dest, p_src, def);
+}
+
+/**
+ * (As typed_var_eq, but compare the members of <b>a</b> and <b>b</b>
+ * defined by <b>member</b>.)
+ **/
+bool
+struct_var_eq(const void *a, const void *b, const struct_member_t *member)
+{
+  const void *p_a = struct_get_ptr(a, member);
+  const void *p_b = struct_get_ptr(b, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_eq(p_a, p_b, def);
+}
+
+/**
+ * (As typed_var_ok, but validate the member of <b>object</b> defined by
+ * <b>member</b>.)
+ **/
+bool
+struct_var_ok(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_ok(p, def);
+}
+
+/**
+ * (As typed_var_kvassign, but assign a value to the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_kvassign(void *object, const struct config_line_t *line,
+                    char **errmsg,
+                    const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvassign(p, line, errmsg, def);
+}
+
+/**
+ * (As typed_var_kvencode, but encode the value of the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+struct config_line_t *
+struct_var_kvencode(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvencode(member->name, p, def);
+}
+
+/**
+ * Mark the field in <b>object</b> determined by <b>member</b> -- a variable
+ * that ordinarily would be extended by assignment -- as "fragile", so that it
+ * will get replaced by the next assignment instead.
+ */
+void
+struct_var_mark_fragile(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+  return typed_var_mark_fragile(p, def);
+}
+
+/**
+ * Return the official name of this struct member.
+ **/
+const char *
+struct_var_get_name(const struct_member_t *member)
+{
+  return member->name;
+}
+
+/**
+ * Return the type name for this struct member.
+ *
+ * Do not use the output of this function to inspect a type within Tor.  It is
+ * suitable for debugging, informing the controller or user of a variable's
+ * type, etc.
+ **/
+const char *
+struct_var_get_typename(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->name : NULL;
+}
+
+/** Return all of the flags set for this struct member. */
+uint32_t
+struct_var_get_flags(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->flags : 0;
+}
diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h
new file mode 100644
index 0000000000..bcb4b58c3f
--- /dev/null
+++ b/src/lib/confmgt/structvar.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.h
+ * @brief Header for lib/confmgt/structvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H
+#define TOR_LIB_CONFMGT_STRUCTVAR_H
+
+struct struct_magic_decl_t;
+struct struct_member_t;
+struct config_line_t;
+
+#include <stdbool.h>
+#include "lib/cc/torint.h"
+
+void struct_set_magic(void *object,
+                      const struct struct_magic_decl_t *decl);
+void struct_check_magic(const void *object,
+                        const struct struct_magic_decl_t *decl);
+
+void *struct_get_mptr(void *object,
+                      const struct struct_member_t *member);
+const void *struct_get_ptr(const void *object,
+                           const struct struct_member_t *member);
+
+void struct_var_free(void *object,
+                     const struct struct_member_t *member);
+int struct_var_copy(void *dest, const void *src,
+                    const struct struct_member_t *member);
+bool struct_var_eq(const void *a, const void *b,
+                   const struct struct_member_t *member);
+bool struct_var_ok(const void *object,
+                   const struct struct_member_t *member);
+void struct_var_mark_fragile(void *object,
+                             const struct struct_member_t *member);
+
+const char *struct_var_get_name(const struct struct_member_t *member);
+const char *struct_var_get_typename(const struct struct_member_t *member);
+uint32_t struct_var_get_flags(const struct struct_member_t *member);
+
+int struct_var_kvassign(void *object, const struct config_line_t *line,
+                        char **errmsg,
+                        const struct struct_member_t *member);
+struct config_line_t *struct_var_kvencode(
+                        const void *object,
+                        const struct struct_member_t *member);
+
+#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */
diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c
new file mode 100644
index 0000000000..5066e12265
--- /dev/null
+++ b/src/lib/confmgt/type_defs.c
@@ -0,0 +1,791 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.c
+ * @brief Definitions for various low-level configuration types.
+ *
+ * This module creates a number of var_type_def_t objects, to be used by
+ * typedvar.c in manipulating variables.
+ *
+ * The types here are common types that can be implemented with Tor's
+ * low-level functionality.  To define new types, see var_type_def_st.h.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/unitparse.h"
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/conf/conftypes.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+#include <errno.h>
+
+//////
+// CONFIG_TYPE_STRING
+// CONFIG_TYPE_FILENAME
+//
+// These two types are the same for now, but they have different names.
+//
+// Warning: For this type, the default value (NULL) and "" are considered
+// different values.  That is generally risky, and best avoided for other
+// types in the future.
+//////
+
+static int
+string_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  char **p = (char**)target;
+  *p = tor_strdup(value);
+  return 0;
+}
+
+static char *
+string_encode(const void *value, const void *params)
+{
+  (void)params;
+  const char **p = (const char**)value;
+  return *p ? tor_strdup(*p) : NULL;
+}
+
+static void
+string_clear(void *value, const void *params)
+{
+  (void)params;
+  char **p = (char**)value;
+  tor_free(*p); // sets *p to NULL.
+}
+
+static const var_type_fns_t string_fns = {
+  .parse = string_parse,
+  .encode = string_encode,
+  .clear = string_clear,
+};
+
+/////
+// CONFIG_TYPE_INT
+// CONFIG_TYPE_POSINT
+//
+// These types are implemented as int, possibly with a restricted range.
+/////
+
+typedef struct int_type_params_t {
+  int minval;
+  int maxval;
+} int_parse_params_t;
+
+static const int_parse_params_t INT_PARSE_UNRESTRICTED = {
+  .minval = INT_MIN,
+  .maxval = INT_MAX,
+};
+
+static const int_parse_params_t INT_PARSE_POSINT = {
+  .minval = 0,
+  .maxval = INT_MAX,
+};
+
+static int
+int_parse(void *target, const char *value, char **errmsg, const void *params,
+          const char *key)
+{
+  (void)key;
+  const int_parse_params_t *pp;
+  if (params) {
+    pp = params;
+  } else {
+    pp = &INT_PARSE_UNRESTRICTED;
+  }
+  int *p = target;
+  int ok=0;
+  *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.",
+                 value);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+int_encode(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  char *result;
+  tor_asprintf(&result, "%d", v);
+  return result;
+}
+
+static void
+int_clear(void *value, const void *params)
+{
+  (void)params;
+  *(int*)value = 0;
+}
+
+static bool
+int_ok(const void *value, const void *params)
+{
+  const int_parse_params_t *pp = params;
+  if (pp) {
+    int v = *(int*)value;
+    return pp->minval <= v && v <= pp->maxval;
+  } else {
+    return true;
+  }
+}
+
+static const var_type_fns_t int_fns = {
+  .parse = int_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+  .ok = int_ok,
+};
+
+/////
+// CONFIG_TYPE_UINT64
+//
+// This type is an unrestricted u64.
+/////
+
+static int
+uint64_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  uint64_t *p = target;
+  int ok=0;
+  *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.",
+                 value);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+uint64_encode(const void *value, const void *params)
+{
+  (void)params;
+  uint64_t v = *(uint64_t*)value;
+  char *result;
+  tor_asprintf(&result, "%"PRIu64, v);
+  return result;
+}
+
+static void
+uint64_clear(void *value, const void *params)
+{
+  (void)params;
+  *(uint64_t*)value = 0;
+}
+
+static const var_type_fns_t uint64_fns = {
+  .parse = uint64_parse,
+  .encode = uint64_encode,
+  .clear = uint64_clear,
+};
+
+/////
+// CONFIG_TYPE_INTERVAL
+// CONFIG_TYPE_MSEC_INTERVAL
+// CONFIG_TYPE_MEMUNIT
+//
+// These types are implemented using the config_parse_units() function.
+// The intervals are stored as ints, whereas memory units are stored as
+// uint64_ts.
+/////
+
+static int
+units_parse_u64(void *target, const char *value, char **errmsg,
+                const void *params, const char *key)
+{
+  (void)key;
+  const unit_table_t *table = params;
+  tor_assert(table);
+  uint64_t *v = (uint64_t*)target;
+  int ok=1;
+  *v = config_parse_units(value, table, &ok);
+  if (!ok) {
+    *errmsg = tor_strdup("Provided value is malformed or out of bounds.");
+    return -1;
+  }
+  return 0;
+}
+
+static int
+units_parse_int(void *target, const char *value, char **errmsg,
+                const void *params, const char *key)
+{
+  (void)key;
+  const unit_table_t *table = params;
+  tor_assert(table);
+  int *v = (int*)target;
+  int ok=1;
+  uint64_t u64 = config_parse_units(value, table, &ok);
+  if (!ok) {
+    *errmsg = tor_strdup("Provided value is malformed or out of bounds.");
+    return -1;
+  }
+  if (u64 > INT_MAX) {
+    tor_asprintf(errmsg, "Provided value %s is too large", value);
+    return -1;
+  }
+  *v = (int) u64;
+  return 0;
+}
+
+static bool
+units_ok_int(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  return v >= 0;
+}
+
+static const var_type_fns_t memunit_fns = {
+  .parse = units_parse_u64,
+  .encode = uint64_encode, // doesn't use params
+  .clear = uint64_clear, // doesn't use params
+};
+
+static const var_type_fns_t interval_fns = {
+  .parse = units_parse_int,
+  .encode = int_encode, // doesn't use params
+  .clear = int_clear, // doesn't use params,
+  .ok = units_ok_int // can't use int_ok, since that expects int params.
+};
+
+/////
+// CONFIG_TYPE_DOUBLE
+//
+// This is a nice simple double.
+/////
+
+static int
+double_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  double *v = (double*)target;
+  char *endptr=NULL;
+  errno = 0;
+  *v = strtod(value, &endptr);
+  if (endptr == value || *endptr != '\0') {
+    // Either there are no converted characters, or there were some characters
+    // that didn't get converted.
+    tor_asprintf(errmsg, "Could not convert %s to a number.", escaped(value));
+    return -1;
+  }
+  if (errno == ERANGE) {
+    // strtod will set errno to ERANGE on underflow or overflow.
+    bool underflow = -.00001 < *v && *v < .00001;
+    tor_asprintf(errmsg,
+                 "%s is too %s to express as a floating-point number.",
+                 escaped(value), underflow ? "small" : "large");
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+double_encode(const void *value, const void *params)
+{
+  (void)params;
+  double v = *(double*)value;
+  char *result;
+  tor_asprintf(&result, "%f", v);
+  return result;
+}
+
+static void
+double_clear(void *value, const void *params)
+{
+  (void)params;
+  double *v = (double *)value;
+  *v = 0.0;
+}
+
+static const var_type_fns_t double_fns = {
+  .parse = double_parse,
+  .encode = double_encode,
+  .clear = double_clear,
+};
+
+/////
+// CONFIG_TYPE_BOOL
+// CONFIG_TYPE_AUTOBOOL
+//
+// These types are implemented as a case-insensitive string-to-integer
+// mapping.
+/////
+
+typedef struct enumeration_table_t {
+  const char *name;
+  int value;
+} enumeration_table_t;
+
+static int
+enum_parse(void *target, const char *value, char **errmsg,
+           const void *params, const char *key)
+{
+  (void)key;
+  const enumeration_table_t *table = params;
+  int *p = (int *)target;
+  for (; table->name; ++table) {
+    if (!strcasecmp(value, table->name)) {
+      *p = table->value;
+      return 0;
+    }
+  }
+  tor_asprintf(errmsg, "Unrecognized value %s.", value);
+  return -1;
+}
+
+static char *
+enum_encode(const void *value, const void *params)
+{
+  int v = *(const int*)value;
+  const enumeration_table_t *table = params;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return tor_strdup(table->name);
+  }
+  return NULL; // error.
+}
+
+static void
+enum_clear(void *value, const void *params)
+{
+  int *p = (int*)value;
+  const enumeration_table_t *table = params;
+  tor_assert(table->name);
+  *p = table->value;
+}
+
+static bool
+enum_ok(const void *value, const void *params)
+{
+  int v = *(const int*)value;
+  const enumeration_table_t *table = params;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return true;
+  }
+  return false;
+}
+
+static const enumeration_table_t enum_table_bool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { NULL, 0 },
+};
+
+static const enumeration_table_t enum_table_autobool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { "auto", -1 },
+  { NULL, 0 },
+};
+
+static const var_type_fns_t enum_fns = {
+  .parse = enum_parse,
+  .encode = enum_encode,
+  .clear = enum_clear,
+  .ok = enum_ok,
+};
+
+/////
+// CONFIG_TYPE_ISOTIME
+//
+// This is a time_t, encoded in ISO8601 format.
+/////
+
+static int
+time_parse(void *target, const char *value, char **errmsg,
+           const void *params, const char *key)
+{
+  (void) params;
+  (void) key;
+  time_t *p = target;
+  if (parse_iso_time(value, p) < 0) {
+    tor_asprintf(errmsg, "Invalid time %s", escaped(value));
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+time_encode(const void *value, const void *params)
+{
+  (void)params;
+  time_t v = *(const time_t *)value;
+  char *result = tor_malloc(ISO_TIME_LEN+1);
+  format_iso_time(result, v);
+  return result;
+}
+
+static void
+time_clear(void *value, const void *params)
+{
+  (void)params;
+  time_t *t = value;
+  *t = 0;
+}
+
+static const var_type_fns_t time_fns = {
+  .parse = time_parse,
+  .encode = time_encode,
+  .clear = time_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV
+//
+// This type is a comma-separated list of strings, stored in a smartlist_t.
+// An empty list may be encoded either as an empty smartlist, or as NULL.
+/////
+
+static int
+csv_parse(void *target, const char *value, char **errmsg,
+          const void *params, const char *key)
+{
+  (void)params;
+  (void)errmsg;
+  (void)key;
+  smartlist_t **sl = (smartlist_t**)target;
+  *sl = smartlist_new();
+  smartlist_split_string(*sl, value, ",",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  return 0;
+}
+
+static char *
+csv_encode(const void *value, const void *params)
+{
+  (void)params;
+  const smartlist_t *sl = *(const smartlist_t **)value;
+  if (! sl)
+    return tor_strdup("");
+
+  return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
+}
+
+static void
+csv_clear(void *value, const void *params)
+{
+  (void)params;
+  smartlist_t **sl = (smartlist_t**)value;
+  if (!*sl)
+    return;
+  SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp));
+  smartlist_free(*sl); // clears pointer.
+}
+
+static const var_type_fns_t csv_fns = {
+  .parse = csv_parse,
+  .encode = csv_encode,
+  .clear = csv_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV_INTERVAL
+//
+// This type used to be a list of time intervals, used to determine a download
+// schedule.  Now, only the first interval counts: everything after the first
+// comma is discarded.
+/////
+
+static int
+legacy_csv_interval_parse(void *target, const char *value, char **errmsg,
+                          const void *params, const char *key)
+{
+  (void)params;
+  /* We used to have entire smartlists here.  But now that all of our
+   * download schedules use exponential backoff, only the first part
+   * matters. */
+  const char *comma = strchr(value, ',');
+  const char *val = value;
+  char *tmp = NULL;
+  if (comma) {
+    tmp = tor_strndup(val, comma - val);
+    val = tmp;
+  }
+
+  int rv = units_parse_int(target, val, errmsg, &time_units, key);
+  tor_free(tmp);
+  return rv;
+}
+
+static const var_type_fns_t legacy_csv_interval_fns = {
+  .parse = legacy_csv_interval_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+};
+
+/////
+// CONFIG_TYPE_LINELIST
+// CONFIG_TYPE_LINELIST_S
+// CONFIG_TYPE_LINELIST_V
+//
+// A linelist is a raw config_line_t list.  Order is preserved.
+//
+// The LINELIST type is used for homogeneous lists, where all the lines
+// have the same key.
+//
+// The LINELIST_S and LINELIST_V types are used for the case where multiple
+// lines of different keys are kept in a single list, to preserve their
+// relative order.  The unified list is stored as a "virtual" variable whose
+// type is LINELIST_V; the individual sublists are treated as variables of
+// type LINELIST_S.
+//
+// A linelist may be fragile or non-fragile. Assigning a line to a fragile
+// linelist replaces the list with the line.  If the line has the "APPEND"
+// command set on it, or if the list is non-fragile, the line is appended.
+// Either way, the new list is non-fragile.
+/////
+
+static int
+linelist_kv_parse(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  config_line_t **lines = target;
+
+  if (*lines && (*lines)->fragile) {
+    if (line->command == CONFIG_LINE_APPEND) {
+      (*lines)->fragile = 0;
+    } else {
+      config_free_lines(*lines); // sets it to NULL
+    }
+  }
+
+  config_line_append(lines, line->key, line->value);
+  return 0;
+}
+
+static int
+linelist_kv_virt_noparse(void *target, const struct config_line_t *line,
+                         char **errmsg, const void *params)
+{
+  (void)target;
+  (void)line;
+  (void)params;
+  *errmsg = tor_strdup("Cannot assign directly to virtual option.");
+  return -1;
+}
+
+static struct config_line_t *
+linelist_kv_encode(const char *key, const void *value,
+                   const void *params)
+{
+  (void)key;
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup(lines);
+}
+
+static struct config_line_t *
+linelist_s_kv_encode(const char *key, const void *value,
+                     const void *params)
+{
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup_and_filter(lines, key);
+}
+
+static void
+linelist_clear(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **lines = target;
+  config_free_lines(*lines); // sets it to NULL
+}
+
+static bool
+linelist_eq(const void *a, const void *b, const void *params)
+{
+  (void)params;
+  const config_line_t *lines_a = *(const config_line_t **)a;
+  const config_line_t *lines_b = *(const config_line_t **)b;
+  return config_lines_eq(lines_a, lines_b);
+}
+
+static int
+linelist_copy(void *target, const void *value, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  const config_line_t *val = *(const config_line_t **)value;
+  config_free_lines(*ptr);
+  *ptr = config_lines_dup(val);
+  return 0;
+}
+
+static void
+linelist_mark_fragile(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  if (*ptr)
+    (*ptr)->fragile = 1;
+}
+
+static const var_type_fns_t linelist_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_v_fns = {
+  .kv_parse = linelist_kv_virt_noparse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_s_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_s_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+};
+
+/////
+// CONFIG_TYPE_ROUTERSET
+//
+// XXXX This type is not implemented here, since routerset_t is not available
+// XXXX to this module.
+/////
+
+/////
+// CONFIG_TYPE_OBSOLETE
+//
+// Used to indicate an obsolete option.
+//
+// XXXX This is not a type, and should be handled at a higher level of
+// XXXX abstraction.
+/////
+
+static int
+ignore_parse(void *target, const char *value, char **errmsg,
+             const void *params, const char *key)
+{
+  (void)target;
+  (void)value;
+  (void)errmsg;
+  (void)params;
+  // XXXX move this to a higher level, once such a level exists.
+  log_warn(LD_GENERAL, "Skipping obsolete configuration option%s%s%s",
+           key && *key ? " \"" : "",
+           key && *key ? key : "",
+           key && *key ? "\"." : ".");
+  return 0;
+}
+
+static char *
+ignore_encode(const void *value, const void *params)
+{
+  (void)value;
+  (void)params;
+  return NULL;
+}
+
+static const var_type_fns_t ignore_fns = {
+  .parse = ignore_parse,
+  .encode = ignore_encode,
+};
+
+/**
+ * Table mapping conf_type_t values to var_type_def_t objects.
+ **/
+static const var_type_def_t type_definitions_table[] = {
+  [CONFIG_TYPE_STRING] = { .name="String", .fns=&string_fns },
+  [CONFIG_TYPE_FILENAME] = { .name="Filename", .fns=&string_fns },
+  [CONFIG_TYPE_INT] = { .name="SignedInteger", .fns=&int_fns,
+                        .params=&INT_PARSE_UNRESTRICTED },
+  [CONFIG_TYPE_POSINT] = { .name="Integer", .fns=&int_fns,
+                           .params=&INT_PARSE_POSINT },
+  [CONFIG_TYPE_UINT64] = { .name="Integer", .fns=&uint64_fns, },
+  [CONFIG_TYPE_MEMUNIT] = { .name="DataSize", .fns=&memunit_fns,
+                            .params=&memory_units },
+  [CONFIG_TYPE_INTERVAL] = { .name="TimeInterval", .fns=&interval_fns,
+                             .params=&time_units },
+  [CONFIG_TYPE_MSEC_INTERVAL] = { .name="TimeMsecInterval",
+                                  .fns=&interval_fns,
+                                  .params=&time_msec_units },
+  [CONFIG_TYPE_DOUBLE] = { .name="Float", .fns=&double_fns, },
+  [CONFIG_TYPE_BOOL] = { .name="Boolean", .fns=&enum_fns,
+                         .params=&enum_table_bool },
+  [CONFIG_TYPE_AUTOBOOL] = { .name="Boolean+Auto", .fns=&enum_fns,
+                             .params=&enum_table_autobool },
+  [CONFIG_TYPE_ISOTIME] = { .name="Time", .fns=&time_fns, },
+  [CONFIG_TYPE_CSV] = { .name="CommaList", .fns=&csv_fns, },
+  [CONFIG_TYPE_CSV_INTERVAL] = { .name="TimeInterval",
+                                 .fns=&legacy_csv_interval_fns, },
+  [CONFIG_TYPE_LINELIST] = { .name="LineList", .fns=&linelist_fns,
+                             .flags=CFLG_NOREPLACE },
+  /*
+   * A "linelist_s" is a derived view of a linelist_v: inspecting
+   * it gets part of a linelist_v, and setting it adds to the linelist_v.
+   */
+  [CONFIG_TYPE_LINELIST_S] = { .name="Dependent", .fns=&linelist_s_fns,
+                               .flags=CFLG_NOREPLACE|
+                               /* The operations we disable here are
+                                * handled by the linelist_v. */
+                               CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP },
+  [CONFIG_TYPE_LINELIST_V] = { .name="Virtual", .fns=&linelist_v_fns,
+                               .flags=CFLG_NOREPLACE|CFLG_NOSET },
+  [CONFIG_TYPE_OBSOLETE] = {
+         .name="Obsolete", .fns=&ignore_fns,
+         .flags=CFLG_GROUP_OBSOLETE,
+  }
+};
+
+/**
+ * Return a pointer to the var_type_def_t object for the given
+ * config_type_t value, or NULL if no such type definition exists.
+ **/
+const var_type_def_t *
+lookup_type_def(config_type_t type)
+{
+  int t = type;
+  tor_assert(t >= 0);
+  if (t >= (int)ARRAY_LENGTH(type_definitions_table))
+    return NULL;
+  return &type_definitions_table[t];
+}
diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h
new file mode 100644
index 0000000000..ecf040529e
--- /dev/null
+++ b/src/lib/confmgt/type_defs.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.h
+ * @brief Header for lib/confmgt/type_defs.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H
+#define TOR_LIB_CONFMGT_TYPE_DEFS_H
+
+const struct var_type_def_t *lookup_type_def(config_type_t type);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */
diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c
new file mode 100644
index 0000000000..ce11a69379
--- /dev/null
+++ b/src/lib/confmgt/typedvar.c
@@ -0,0 +1,228 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.c
+ * @brief Functions for accessing a pointer as an object of a given type.
+ *
+ * These functions represent a low-level API for accessing a typed variable.
+ * They are used in the configuration system to examine and set fields in
+ * configuration objects used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/util_string.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+
+/**
+ * Try to parse a string in <b>value</b> that encodes an object of the type
+ * defined by <b>def</b>. If not NULL, <b>key</b> is the name of the option,
+ * which may be used for logging.
+ *
+ * On success, adjust the lvalue pointed to by <b>target</b> to hold that
+ * value, and return 0.  On failure, set *<b>errmsg</b> to a newly allocated
+ * string holding an error message, and return -1.
+ **/
+int
+typed_var_assign(void *target, const char *value, char **errmsg,
+                 const var_type_def_t *def, const char *key)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  // clear old value if needed.
+  typed_var_free(target, def);
+
+  tor_assert(def->fns->parse);
+  return def->fns->parse(target, value, errmsg, def->params, key);
+}
+
+/**
+ * Try to parse a single line from the head of<b>line</b> that encodes an
+ * object of the type defined in <b>def</b>. On success and failure, behave as
+ * typed_var_assign().
+ *
+ * All types for which keys are significant should use this function.
+ *
+ * Note that although multiple lines may be provided in <b>line</b>,
+ * only the first one is handled by this function.
+ **/
+int
+typed_var_kvassign(void *target, const config_line_t *line,
+                      char **errmsg, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+
+  if (def->fns->kv_parse) {
+    // We do _not_ free the old value here, since linelist options
+    // sometimes have append semantics.
+    return def->fns->kv_parse(target, line, errmsg, def->params);
+  }
+
+  return typed_var_assign(target, line->value, errmsg, def, line->key);
+}
+
+/**
+ * Release storage held by a variable in <b>target</b> of type defined by
+ * <b>def</b>, and set <b>target</b> to a reasonable default.
+ **/
+void
+typed_var_free(void *target, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return; // LCOV_EXCL_LINE
+  if (def->fns->clear) {
+    def->fns->clear(target, def->params);
+  }
+}
+
+/**
+ * Encode a value of type <b>def</b> pointed to by <b>value</b>, and return
+ * its result in a newly allocated string.  The string may need to be escaped.
+ *
+ * Returns NULL if this option has a NULL value, or on internal error.
+ **/
+char *
+typed_var_encode(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  tor_assert(def->fns->encode);
+  return def->fns->encode(value, def->params);
+}
+
+/**
+ * As typed_var_encode(), but returns a newly allocated config_line_t
+ * object.  The provided <b>key</b> is used as the key of the lines, unless
+ * the type is one (line a linelist) that encodes its own keys.
+ *
+ * This function may return a list of multiple lines.
+ *
+ * Returns NULL if there are no lines to encode, or on internal error.
+ */
+config_line_t *
+typed_var_kvencode(const char *key, const void *value,
+                      const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  if (def->fns->kv_encode) {
+    return def->fns->kv_encode(key, value, def->params);
+  }
+  char *encoded_value = typed_var_encode(value, def);
+  if (!encoded_value)
+    return NULL;
+
+  config_line_t *result = tor_malloc_zero(sizeof(config_line_t));
+  result->key = tor_strdup(key);
+  result->value = encoded_value;
+  return result;
+}
+
+/**
+ * Set <b>dest</b> to contain the same value as <b>src</b>. Both types
+ * must be as defined by <b>def</b>.
+ *
+ * Return 0 on success, and -1 on failure.
+ **/
+int
+typed_var_copy(void *dest, const void *src, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  if (def->fns->copy) {
+    // If we have been provided a copy fuction, use it.
+    return def->fns->copy(dest, src, def);
+  }
+
+  // Otherwise, encode 'src' and parse the result into 'def'.
+  char *enc = typed_var_encode(src, def);
+  if (!enc) {
+    typed_var_free(dest, def);
+    return 0;
+  }
+  char *err = NULL;
+  int rv = typed_var_assign(dest, enc, &err, def, NULL);
+  if (BUG(rv < 0)) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s",
+             escaped(enc), def->name, err?err:"");
+    // LCOV_EXCL_STOP
+  }
+  tor_free(err);
+  tor_free(enc);
+  return rv;
+}
+
+/**
+ * Return true if <b>a</b> and <b>b</b> are semantically equivalent.
+ * Both types must be as defined by <b>def</b>.
+ **/
+bool
+typed_var_eq(const void *a, const void *b, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->eq) {
+    // Use a provided eq function if we got one.
+    return def->fns->eq(a, b, def->params);
+  }
+
+  // Otherwise, encode the values and compare them.
+  char *enc_a = typed_var_encode(a, def);
+  char *enc_b = typed_var_encode(b, def);
+  bool eq = !strcmp_opt(enc_a,enc_b);
+  tor_free(enc_a);
+  tor_free(enc_b);
+  return eq;
+}
+
+/**
+ * Check whether <b>value</b> encodes a valid value according to the
+ * type definition in <b>def</b>.
+ */
+bool
+typed_var_ok(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->ok)
+    return def->fns->ok(value, def->params);
+
+  return true;
+}
+
+/**
+ * Mark <b>value</b> -- a variable that ordinarily would be extended by
+ * assignment -- as "fragile", so that it will get replaced by the next
+ * assignment instead.
+ **/
+void
+typed_var_mark_fragile(void *value, const var_type_def_t *def)
+{
+  if (BUG(!def)) {
+    return; // LCOV_EXCL_LINE
+  }
+
+  if (def->fns->mark_fragile)
+    def->fns->mark_fragile(value, def->params);
+}
diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h
new file mode 100644
index 0000000000..4382613833
--- /dev/null
+++ b/src/lib/confmgt/typedvar.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.h
+ * @brief Header for lib/confmgt/typedvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H
+#define TOR_LIB_CONFMGT_TYPEDVAR_H
+
+#include <stdbool.h>
+
+enum config_type_t;
+struct config_line_t;
+
+typedef struct var_type_fns_t var_type_fns_t;
+typedef struct var_type_def_t var_type_def_t;
+
+int typed_var_assign(void *target, const char *value, char **errmsg,
+                     const var_type_def_t *def, const char *key);
+void typed_var_free(void *target, const var_type_def_t *def);
+char *typed_var_encode(const void *value, const var_type_def_t *def);
+int typed_var_copy(void *dest, const void *src, const var_type_def_t *def);
+bool typed_var_eq(const void *a, const void *b, const var_type_def_t *def);
+bool typed_var_ok(const void *value, const var_type_def_t *def);
+
+int typed_var_kvassign(void *target, const struct config_line_t *line,
+                           char **errmsg, const var_type_def_t *def);
+struct config_line_t *typed_var_kvencode(const char *key, const void *value,
+                                            const var_type_def_t *def);
+
+void typed_var_mark_fragile(void *value, const var_type_def_t *def);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */
diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c
new file mode 100644
index 0000000000..c3ed8285a4
--- /dev/null
+++ b/src/lib/confmgt/unitparse.c
@@ -0,0 +1,206 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.c
+ * @brief Functions for parsing values with units from a configuration file.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/util_string.h"
+
+#include <string.h>
+
+/** Table to map the names of memory units to the number of bytes they
+ * contain. */
+const struct unit_table_t memory_units[] = {
+  { "",          1 },
+  { "b",         1<< 0 },
+  { "byte",      1<< 0 },
+  { "bytes",     1<< 0 },
+  { "kb",        1<<10 },
+  { "kbyte",     1<<10 },
+  { "kbytes",    1<<10 },
+  { "kilobyte",  1<<10 },
+  { "kilobytes", 1<<10 },
+  { "kilobits",  1<<7  },
+  { "kilobit",   1<<7  },
+  { "kbits",     1<<7  },
+  { "kbit",      1<<7  },
+  { "m",         1<<20 },
+  { "mb",        1<<20 },
+  { "mbyte",     1<<20 },
+  { "mbytes",    1<<20 },
+  { "megabyte",  1<<20 },
+  { "megabytes", 1<<20 },
+  { "megabits",  1<<17 },
+  { "megabit",   1<<17 },
+  { "mbits",     1<<17 },
+  { "mbit",      1<<17 },
+  { "gb",        1<<30 },
+  { "gbyte",     1<<30 },
+  { "gbytes",    1<<30 },
+  { "gigabyte",  1<<30 },
+  { "gigabytes", 1<<30 },
+  { "gigabits",  1<<27 },
+  { "gigabit",   1<<27 },
+  { "gbits",     1<<27 },
+  { "gbit",      1<<27 },
+  { "tb",        UINT64_C(1)<<40 },
+  { "tbyte",     UINT64_C(1)<<40 },
+  { "tbytes",    UINT64_C(1)<<40 },
+  { "terabyte",  UINT64_C(1)<<40 },
+  { "terabytes", UINT64_C(1)<<40 },
+  { "terabits",  UINT64_C(1)<<37 },
+  { "terabit",   UINT64_C(1)<<37 },
+  { "tbits",     UINT64_C(1)<<37 },
+  { "tbit",      UINT64_C(1)<<37 },
+  { NULL, 0 },
+};
+
+/** Table to map the names of time units to the number of seconds they
+ * contain. */
+const struct unit_table_t time_units[] = {
+  { "",         1 },
+  { "second",   1 },
+  { "seconds",  1 },
+  { "minute",   60 },
+  { "minutes",  60 },
+  { "hour",     60*60 },
+  { "hours",    60*60 },
+  { "day",      24*60*60 },
+  { "days",     24*60*60 },
+  { "week",     7*24*60*60 },
+  { "weeks",    7*24*60*60 },
+  { "month",    2629728, }, /* about 30.437 days */
+  { "months",   2629728, },
+  { NULL, 0 },
+};
+
+/** Table to map the names of time units to the number of milliseconds
+ * they contain. */
+const struct unit_table_t time_msec_units[] = {
+  { "",         1 },
+  { "msec",     1 },
+  { "millisecond", 1 },
+  { "milliseconds", 1 },
+  { "second",   1000 },
+  { "seconds",  1000 },
+  { "minute",   60*1000 },
+  { "minutes",  60*1000 },
+  { "hour",     60*60*1000 },
+  { "hours",    60*60*1000 },
+  { "day",      24*60*60*1000 },
+  { "days",     24*60*60*1000 },
+  { "week",     7*24*60*60*1000 },
+  { "weeks",    7*24*60*60*1000 },
+  { NULL, 0 },
+};
+
+/** Parse a string <b>val</b> containing a number, zero or more
+ * spaces, and an optional unit string.  If the unit appears in the
+ * table <b>u</b>, then multiply the number by the unit multiplier.
+ * On success, set *<b>ok</b> to 1 and return this product.
+ * Otherwise, set *<b>ok</b> to 0.
+ */
+uint64_t
+config_parse_units(const char *val, const unit_table_t *u, int *ok)
+{
+  uint64_t v = 0;
+  double d = 0;
+  int use_float = 0;
+  char *cp;
+
+  tor_assert(ok);
+
+  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
+  if (!*ok || (cp && *cp == '.')) {
+    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
+    if (!*ok)
+      goto done;
+    use_float = 1;
+  }
+
+  if (BUG(!cp)) {
+    // cp should always be non-NULL if the parse operation succeeds.
+
+    // LCOV_EXCL_START
+    *ok = 1;
+    v = use_float ? ((uint64_t)d) :  v;
+    goto done;
+    // LCOV_EXCL_STOP
+  }
+
+  cp = (char*) eat_whitespace(cp);
+
+  for ( ;u->unit;++u) {
+    if (!strcasecmp(u->unit, cp)) {
+      if (use_float)
+        v = (uint64_t)(u->multiplier * d);
+      else
+        v *= u->multiplier;
+      *ok = 1;
+      goto done;
+    }
+  }
+  log_warn(LD_CONFIG, "Unknown unit '%s'.", cp);
+  *ok = 0;
+ done:
+
+  if (*ok)
+    return v;
+  else
+    return 0;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
+ * and return the number of bytes specified.  Otherwise, set
+ * *<b>ok</b> to false and return 0. */
+uint64_t
+config_parse_memunit(const char *s, int *ok)
+{
+  uint64_t u = config_parse_units(s, memory_units, ok);
+  return u;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * time in milliseconds.  On success, set *<b>ok</b> to true and return
+ * the number of milliseconds in the provided interval.  Otherwise, set
+ * *<b>ok</b> to 0 and return -1. */
+int
+config_parse_msec_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_msec_units, ok);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of time.
+ * On success, set *<b>ok</b> to true and return the number of seconds in
+ * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
+ */
+int
+config_parse_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_units, ok);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h
new file mode 100644
index 0000000000..216361a7d4
--- /dev/null
+++ b/src/lib/confmgt/unitparse.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.h
+ * @brief Header for lib/confmgt/unitparse.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_UNITPARSE_H
+#define TOR_LIB_CONFMGT_UNITPARSE_H
+
+#include <lib/cc/torint.h>
+
+/** Mapping from a unit name to a multiplier for converting that unit into a
+ * base unit.  Used by config_parse_unit. */
+typedef struct unit_table_t {
+  const char *unit; /**< The name of the unit */
+  uint64_t multiplier; /**< How many of the base unit appear in this unit */
+} unit_table_t;
+
+extern const unit_table_t memory_units[];
+extern const unit_table_t time_units[];
+extern const struct unit_table_t time_msec_units[];
+
+uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok);
+
+uint64_t config_parse_memunit(const char *s, int *ok);
+int config_parse_msec_interval(const char *s, int *ok);
+int config_parse_interval(const char *s, int *ok);
+
+#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */
diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h
new file mode 100644
index 0000000000..aa9ded39e9
--- /dev/null
+++ b/src/lib/confmgt/var_type_def_st.h
@@ -0,0 +1,170 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file var_type_def_st.h
+ * @brief Structure declarations for typedvar type definitions.
+ *
+ * This structure is used for defining new variable types.  If you are not
+ * defining a new variable type for use by the configuration management
+ * system, you don't need this structure.
+ *
+ * For defining new variables, see the types in conftypes.h.
+ *
+ * For data-driven access to configuration variables, see the other members of
+ * lib/confmgt/.
+ *
+ * STATUS NOTE: It is not yet possible to actually define new variables
+ *   outside of config.c, and many of the types that will eventually be used
+ *   to do so are not yet moved. This will change as more of #29211 is
+ *   completed.
+ **/
+
+#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+
+#include <stdbool.h>
+
+struct config_line_t;
+
+/**
+ * A structure full of functions pointers to implement a variable type.
+ *
+ * Every type MUST implement parse or kv_parse and encode or kv_encode;
+ * the other functions pointers MAY be NULL.
+ *
+ * All functions here take a <b>params</b> argument, whose value
+ * is determined by the type definition.  Two types may have the
+ * same functions, but differ only in parameters.
+ *
+ * Implementation considerations: If "" encodes a valid value for a type, try
+ * to make sure that it encodes the same thing as the default value for the
+ * type (that is, the value that is set by config_clear() or memset(0)). If
+ * this is not the case, you need to make extra certain that your parse/encode
+ * implementations preserve the NULL/"" distinction.
+ **/
+struct var_type_fns_t {
+  /**
+   * Try to parse a string in <b>value</b> that encodes an object of this
+   * type.  On success, adjust the lvalue pointed to by <b>target</b> to hold
+   * that value, and return 0.  On failure, set *<b>errmsg</b> to a newly
+   * allocated string holding an error message, and return -1.
+   *
+   * If not NULL, <b>key</b> is the name of the option, which may be used for
+   * logging.
+   **/
+  int (*parse)(void *target, const char *value, char **errmsg,
+               const void *params, const char *key);
+  /**
+   * Try to parse a single line from the head of<b>line</b> that encodes
+   * an object of this type.  On success and failure, behave as in the parse()
+   * function.
+   *
+   * If this function is absent, it is implemented in terms of parse().
+   *
+   * All types for which keys are significant should use this method. For
+   * example, a "linelist" type records the actual keys that are given
+   * for each line, and so should use this method.
+   *
+   * Note that although multiple lines may be provided in <b>line</b>,
+   * only the first one should be handled by this function.
+   **/
+  int (*kv_parse)(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params);
+  /**
+   * Encode a value pointed to by <b>value</b> and return its result
+   * in a newly allocated string.  The string may need to be escaped.
+   *
+   * If this function is absent, it is implemented in terms of kv_encode().
+   *
+   * Returns NULL if this option has a NULL value, or on internal error.
+   *
+   * Requirement: all strings generated by encode() should produce a
+   * semantically equivalent value when given to parse().
+   **/
+  char *(*encode)(const void *value, const void *params);
+  /**
+   * As encode(), but returns a newly allocated config_line_t object.  The
+   * provided <b>key</b> is used as the key of the lines, unless the type is
+   * one that encodes its own keys.
+   *
+   * Unlike kv_parse(), this function will return a list of multiple lines,
+   * if <b>value</b> is such that it must be encoded by multiple lines.
+   *
+   * Returns NULL if there are no lines to encode, or on internal error.
+   *
+   * If this function is absent, it is implemented in terms of encode().
+   **/
+  struct config_line_t *(*kv_encode)(const char *key, const void *value,
+                                     const void *params);
+  /**
+   * Free all storage held in <b>arg</b>, and set <b>arg</b> to a default
+   * value -- usually zero or NULL.
+   *
+   * If this function is absent, the default implementation does nothing.
+   **/
+  void (*clear)(void *arg, const void *params);
+  /**
+   * Return true if <b>a</b> and <b>b</b> hold the same value, and false
+   * otherwise.
+   *
+   * If this function is absent, it is implemented by encoding both a and
+   * b and comparing their encoded strings for equality.
+   **/
+  bool (*eq)(const void *a, const void *b, const void *params);
+  /**
+   * Try to copy the value from <b>value</b> into <b>target</b>.
+   * On success return 0; on failure return -1.
+   *
+   * If this function is absent, it is implemented by encoding the value
+   * into a string, and then parsing it into the target.
+   **/
+  int (*copy)(void *target, const void *value, const void *params);
+  /**
+   * Check whether <b>value</b> holds a valid value according to the
+   * rules of this type; return true if it does and false if it doesn't.
+   *
+   * The default implementation for this function assumes that all
+   * values are valid.
+   **/
+  bool (*ok)(const void *value, const void *params);
+  /**
+   * Mark a value of this variable as "fragile", so that future attempts to
+   * assign to this variable will replace rather than extending it.
+   *
+   * The default implementation for this function does nothing.
+   *
+   * Only meaningful for types with is_cumulative set.
+   **/
+  void (*mark_fragile)(void *value, const void *params);
+};
+
+/**
+ * A structure describing a type that can be manipulated with the typedvar_*
+ * functions.
+ **/
+struct var_type_def_t {
+  /**
+   * The name of this type. Should not include spaces. Used for
+   * debugging, log messages, and the controller API. */
+  const char *name;
+  /**
+   * A function table for this type.
+   */
+  const struct var_type_fns_t *fns;
+  /**
+   * A pointer to a value that should be passed as the 'params' argument when
+   * calling the functions in this type's function table.
+   */
+  const void *params;
+  /**
+   * A bitwise OR of one or more VTFLAG_* values, describing properties
+   * for all values of this type.
+   **/
+  uint32_t flags;
+};
+
+#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */
diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include
index 90de5eda40..81507527d3 100644
--- a/src/lib/container/.may_include
+++ b/src/lib/container/.may_include
@@ -7,12 +7,9 @@ lib/malloc/*.h
 lib/err/*.h
 lib/smartlist_core/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
 lib/intmath/*.h
 lib/log/*.h
 
-# XXXX I am unsure about this one. It's only here for buffers.c
-lib/time/*.h
-
-ht.h
-siphash.h
+ext/ht.h
+ext/siphash.h
diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h
index 910d5fea65..45992796af 100644
--- a/src/lib/container/bitarray.h
+++ b/src/lib/container/bitarray.h
@@ -83,4 +83,4 @@ bitarray_is_set(bitarray_t *b, int bit)
   return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_BITARRAY_H) */
diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c
index 9aa9b1ee56..8c61db81d6 100644
--- a/src/lib/container/bloomfilt.c
+++ b/src/lib/container/bloomfilt.c
@@ -14,7 +14,7 @@
 #include "lib/container/bloomfilt.h"
 #include "lib/intmath/bits.h"
 #include "lib/log/util_bug.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** How many bloom-filter bits we set per address. This is twice the
  * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit
diff --git a/src/lib/container/include.am b/src/lib/container/include.am
index e6492098b5..00d7b8e587 100644
--- a/src/lib/container/include.am
+++ b/src/lib/container/include.am
@@ -5,10 +5,11 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-container-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_container_a_SOURCES =			\
 	src/lib/container/bloomfilt.c			\
-	src/lib/container/buffers.c			\
 	src/lib/container/map.c				\
+	src/lib/container/namemap.c			\
 	src/lib/container/order.c			\
 	src/lib/container/smartlist.c
 
@@ -17,11 +18,13 @@ src_lib_libtor_container_testing_a_SOURCES = \
 src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/container/bitarray.h			\
 	src/lib/container/bloomfilt.h			\
-	src/lib/container/buffers.h			\
 	src/lib/container/handles.h			\
 	src/lib/container/map.h				\
+	src/lib/container/namemap.h			\
+	src/lib/container/namemap_st.h			\
 	src/lib/container/order.h			\
 	src/lib/container/smartlist.h
diff --git a/src/lib/container/map.c b/src/lib/container/map.c
index d213ad50bf..fde33d6ace 100644
--- a/src/lib/container/map.c
+++ b/src/lib/container/map.c
@@ -21,7 +21,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "ht.h"
+#include "ext/ht.h"
 
 /** Helper: Declare an entry type and a map type to implement a mapping using
  * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
diff --git a/src/lib/container/map.h b/src/lib/container/map.h
index a2d1b01d12..9da1d3072c 100644
--- a/src/lib/container/map.h
+++ b/src/lib/container/map.h
@@ -15,7 +15,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "lib/cc/torint.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
   typedef struct maptype maptype;                                       \
@@ -258,4 +258,4 @@ void* strmap_remove_lc(strmap_t *map, const char *key);
     return digestmap_iter_done((digestmap_iter_t*)iter);                \
   }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_MAP_H) */
diff --git a/src/lib/container/namemap.c b/src/lib/container/namemap.c
new file mode 100644
index 0000000000..a90057b32c
--- /dev/null
+++ b/src/lib/container/namemap.c
@@ -0,0 +1,184 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/container/smartlist.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "ext/siphash.h"
+
+#include <string.h>
+
+/** Helper for namemap hashtable implementation: compare two entries. */
+static inline int
+mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b)
+{
+  return !strcmp(a->name, b->name);
+}
+
+/** Helper for namemap hashtable implementation: hash an entry. */
+static inline unsigned
+mapped_name_hash(const mapped_name_t *a)
+{
+  return (unsigned) siphash24g(a->name, strlen(a->name));
+}
+
+HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq)
+HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq, 0.6, tor_reallocarray_, tor_free_)
+
+/** Set up an uninitialized <b>map</b>. */
+void
+namemap_init(namemap_t *map)
+{
+  memset(map, 0, sizeof(*map));
+  HT_INIT(namemap_ht, &map->ht);
+  map->names = smartlist_new();
+}
+
+/** Return the name that <b>map</b> associates with a given <b>id</b>, or
+ * NULL if there is no such name. */
+const char *
+namemap_get_name(const namemap_t *map, unsigned id)
+{
+  if (map->names && id < (unsigned)smartlist_len(map->names)) {
+    mapped_name_t *name = smartlist_get(map->names, (int)id);
+    return name->name;
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * Return the name that <b>map</b> associates with a given <b>id</b>, or a
+ * pointer to a statically allocated string describing the value of <b>id</b>
+ * if no such name exists.
+ **/
+const char *
+namemap_fmt_name(const namemap_t *map, unsigned id)
+{
+  static char buf[32];
+
+  const char *name = namemap_get_name(map, id);
+  if (name)
+    return name;
+
+  tor_snprintf(buf, sizeof(buf), "{%u}", id);
+
+  return buf;
+}
+
+/**
+ * Helper: As namemap_get_id(), but requires that <b>name</b> is
+ * <b>namelen</b> charaters long, and that <b>namelen</b> is no more than
+ * MAX_NAMEMAP_NAME_LEN.
+ */
+static unsigned
+namemap_get_id_unchecked(const namemap_t *map,
+                         const char *name,
+                         size_t namelen)
+{
+  union {
+    mapped_name_t n;
+    char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1];
+  } u;
+  memcpy(u.n.name, name, namelen);
+  u.n.name[namelen] = 0;
+  const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n);
+  if (found) {
+    tor_assert(map->names);
+    tor_assert(smartlist_get(map->names, found->intval) == found);
+    return found->intval;
+  }
+
+  return NAMEMAP_ERR;
+}
+
+/**
+ * Return the identifier currently associated by <b>map</b> with the name
+ * <b>name</b>, or NAMEMAP_ERR if no such identifier exists.
+ **/
+unsigned
+namemap_get_id(const namemap_t *map,
+               const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  return namemap_get_id_unchecked(map, name, namelen);
+}
+
+/**
+ * Return the identifier associated by <b>map</b> with the name
+ * <b>name</b>, allocating a new identifier in <b>map</b> if none exists.
+ *
+ * Return NAMEMAP_ERR if <b>name</b> is too long, or if there are no more
+ * identifiers we can allocate.
+ **/
+unsigned
+namemap_get_or_create_id(namemap_t *map,
+                         const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    map->names = smartlist_new();
+
+  unsigned found = namemap_get_id_unchecked(map, name, namelen);
+  if (found != NAMEMAP_ERR)
+    return found;
+
+  unsigned new_id = (unsigned)smartlist_len(map->names);
+  if (new_id == NAMEMAP_ERR)
+    return NAMEMAP_ERR; /* Can't allocate any more. */
+
+  mapped_name_t *insert = tor_malloc_zero(
+                       offsetof(mapped_name_t, name) + namelen + 1);
+  memcpy(insert->name, name, namelen+1);
+  insert->intval = new_id;
+
+  HT_INSERT(namemap_ht, &map->ht, insert);
+  smartlist_add(map->names, insert);
+
+  return new_id;
+}
+
+/** Return the number of entries in 'names' */
+size_t
+namemap_get_size(const namemap_t *map)
+{
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    return 0;
+
+  return smartlist_len(map->names);
+}
+
+/**
+ * Release all storage held in <b>map</b>.
+ */
+void
+namemap_clear(namemap_t *map)
+{
+  if (!map)
+    return;
+
+  HT_CLEAR(namemap_ht, &map->ht);
+  if (map->names) {
+    SMARTLIST_FOREACH(map->names, mapped_name_t *, n,
+                      tor_free(n));
+    smartlist_free(map->names);
+  }
+  memset(map, 0, sizeof(*map));
+}
diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h
new file mode 100644
index 0000000000..b96bc13f3a
--- /dev/null
+++ b/src/lib/container/namemap.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NAMEMAP_H
+#define TOR_NAMEMAP_H
+
+/**
+ * \file namemap.h
+ *
+ * \brief Header for namemap.c
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+#include <stddef.h>
+
+typedef struct namemap_t namemap_t;
+
+/** Returned in place of an identifier when an error occurs. */
+#define NAMEMAP_ERR UINT_MAX
+
+void namemap_init(namemap_t *map);
+const char *namemap_get_name(const namemap_t *map, unsigned id);
+const char *namemap_fmt_name(const namemap_t *map, unsigned id);
+unsigned namemap_get_id(const namemap_t *map,
+                        const char *name);
+unsigned namemap_get_or_create_id(namemap_t *map,
+                                  const char *name);
+size_t namemap_get_size(const namemap_t *map);
+void namemap_clear(namemap_t *map);
+
+#endif /* !defined(TOR_NAMEMAP_H) */
diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h
new file mode 100644
index 0000000000..5008fd5855
--- /dev/null
+++ b/src/lib/container/namemap_st.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NAMEMAP_ST_H
+#define NAMEMAP_ST_H
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+struct smartlist_t;
+
+/** Longest allowed name that's allowed in a namemap_t. */
+#define MAX_NAMEMAP_NAME_LEN 128
+
+/** An entry inside a namemap_t. Maps a string to a numeric identifier. */
+typedef struct mapped_name_t {
+  HT_ENTRY(mapped_name_t) node;
+  unsigned intval;
+  char name[FLEXIBLE_ARRAY_MEMBER];
+} mapped_name_t;
+
+/** A structure that allocates small numeric identifiers for names and maps
+ * back and forth between them.  */
+struct namemap_t {
+  HT_HEAD(namemap_ht, mapped_name_t) ht;
+  struct smartlist_t *names;
+};
+
+/** Macro to initialize a namemap. */
+#define NAMEMAP_INIT() { HT_INITIALIZER(), NULL }
+
+#endif /* !defined(NAMEMAP_ST_H) */
diff --git a/src/lib/container/order.h b/src/lib/container/order.h
index a176d6d8a6..3f2fd054a0 100644
--- a/src/lib/container/order.h
+++ b/src/lib/container/order.h
@@ -57,4 +57,4 @@ third_quartile_uint32(uint32_t *array, int n_elements)
   return find_nth_uint32(array, n_elements, (n_elements*3)/4);
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_ORDER_H) */
diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c
index 3ab2797d68..2b71c11287 100644
--- a/src/lib/container/smartlist.c
+++ b/src/lib/container/smartlist.c
@@ -678,7 +678,7 @@ smartlist_sort_pointers(smartlist_t *sl)
 static inline void
 smartlist_heapify(smartlist_t *sl,
                   int (*compare)(const void *a, const void *b),
-                  int idx_field_offset,
+                  ptrdiff_t idx_field_offset,
                   int idx)
 {
   while (1) {
@@ -725,7 +725,7 @@ smartlist_heapify(smartlist_t *sl,
 void
 smartlist_pqueue_add(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset,
+                     ptrdiff_t idx_field_offset,
                      void *item)
 {
   int idx;
@@ -754,7 +754,7 @@ smartlist_pqueue_add(smartlist_t *sl,
 void *
 smartlist_pqueue_pop(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset)
+                     ptrdiff_t idx_field_offset)
 {
   void *top;
   tor_assert(sl->num_used);
@@ -778,7 +778,7 @@ smartlist_pqueue_pop(smartlist_t *sl,
 void
 smartlist_pqueue_remove(smartlist_t *sl,
                         int (*compare)(const void *a, const void *b),
-                        int idx_field_offset,
+                        ptrdiff_t idx_field_offset,
                         void *item)
 {
   int idx = IDX_OF_ITEM(item);
@@ -802,7 +802,7 @@ smartlist_pqueue_remove(smartlist_t *sl,
 void
 smartlist_pqueue_assert_ok(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset)
+                           ptrdiff_t idx_field_offset)
 {
   int i;
   for (i = sl->num_used - 1; i >= 0; --i) {
diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h
index 77682db03e..25638e4b22 100644
--- a/src/lib/container/smartlist.h
+++ b/src/lib/container/smartlist.h
@@ -13,6 +13,7 @@
  **/
 
 #include <stdarg.h>
+#include <stddef.h>
 
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -72,18 +73,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
 
 void smartlist_pqueue_add(smartlist_t *sl,
                           int (*compare)(const void *a, const void *b),
-                          int idx_field_offset,
+                          ptrdiff_t idx_field_offset,
                           void *item);
 void *smartlist_pqueue_pop(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset);
+                           ptrdiff_t idx_field_offset);
 void smartlist_pqueue_remove(smartlist_t *sl,
                              int (*compare)(const void *a, const void *b),
-                             int idx_field_offset,
+                             ptrdiff_t idx_field_offset,
                              void *item);
 void smartlist_pqueue_assert_ok(smartlist_t *sl,
                                 int (*compare)(const void *a, const void *b),
-                                int idx_field_offset);
+                                ptrdiff_t idx_field_offset);
 
 char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
                              size_t *len_out) ATTR_MALLOC;
@@ -165,4 +166,4 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
   }                                             \
   STMT_END
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_H) */
diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include
index a0fa4ec05c..0739699686 100644
--- a/src/lib/crypt_ops/.may_include
+++ b/src/lib/crypt_ops/.may_include
@@ -12,7 +12,8 @@ lib/malloc/*.h
 lib/intmath/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/thread/*.h
 lib/log/*.h
 
@@ -21,4 +22,4 @@ trunnel/pwbox.h
 keccak-tiny/*.h
 ed25519/*.h
 
-siphash.h
+ext/siphash.h
diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c
index 2f985d4512..64564892ad 100644
--- a/src/lib/crypt_ops/aes_openssl.c
+++ b/src/lib/crypt_ops/aes_openssl.c
@@ -148,7 +148,7 @@ evaluate_ctr_for_aes(void)
 {
   return 0;
 }
-#else /* !(defined(USE_EVP_AES_CTR)) */
+#else /* !defined(USE_EVP_AES_CTR) */
 
 /*======================================================================*/
 /* Interface to AES code, and counter implementation */
diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h
index 9c10386c34..61ca51315f 100644
--- a/src/lib/crypt_ops/compat_openssl.h
+++ b/src/lib/crypt_ops/compat_openssl.h
@@ -45,7 +45,7 @@
    ((st) == SSL3_ST_SW_SRVR_HELLO_B))
 #define OSSL_HANDSHAKE_STATE int
 #define CONST_IF_OPENSSL_1_1_API
-#else /* !(!defined(OPENSSL_1_1_API)) */
+#else /* defined(OPENSSL_1_1_API) */
 #define STATE_IS_SW_SERVER_HELLO(st) \
   ((st) == TLS_ST_SW_SRVR_HELLO)
 #define CONST_IF_OPENSSL_1_1_API const
diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h
index cc4fbf7a41..88d63c1df2 100644
--- a/src/lib/crypt_ops/crypto_cipher.h
+++ b/src/lib/crypt_ops/crypto_cipher.h
@@ -54,4 +54,4 @@ int crypto_cipher_decrypt_with_iv(const char *key,
                                   char *to, size_t tolen,
                                   const char *from, size_t fromlen);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_CIPHER_H) */
diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h
index 061a7a3505..cd23169cd5 100644
--- a/src/lib/crypt_ops/crypto_curve25519.h
+++ b/src/lib/crypt_ops/crypto_curve25519.h
@@ -76,8 +76,8 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret);
 
 int curve25519_public_from_base64(curve25519_public_key_t *pkey,
                                   const char *input);
-int curve25519_public_to_base64(char *output,
-                                const curve25519_public_key_t *pkey);
+void curve25519_public_to_base64(char *output,
+                                 const curve25519_public_key_t *pkey);
 
 void curve25519_set_impl_params(int use_ed);
 void curve25519_init(void);
diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c
index 8c6388fd5d..8ae97373e8 100644
--- a/src/lib/crypt_ops/crypto_dh_openssl.c
+++ b/src/lib/crypt_ops/crypto_dh_openssl.c
@@ -34,7 +34,7 @@ static int tor_check_dh_key(int severity, const BIGNUM *bn);
 struct crypto_dh_t {
   DH *dh; /**< The openssl DH object */
 };
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g);
 
@@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     goto out;
   if (!DH_set0_pqg(dh, dh_p, NULL, dh_g))
     goto out;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (!(dh->p = BN_dup(p)))
     goto out;
   if (!(dh->g = BN_dup(g)))
@@ -100,7 +100,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     DH_free(dh);
   return ret;
 }
-#endif
+#endif /* 0 */
 
 /**
  * Helper: convert <b>hex<b> to a bignum, and return it.  Assert that the
@@ -202,7 +202,7 @@ crypto_dh_new(int dh_type)
     tor_free(res); // sets res to NULL.
   return res;
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 /** Create and return a new openssl DH from a given prime and generator. */
 static DH *
@@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g)
 
   if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS))
     goto err;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   res_dh->p = dh_p;
   res_dh->g = dh_g;
   res_dh->length = DH_PRIVATE_KEY_BITS;
@@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh)
              "the-universe chances really do happen.  Treating as a failure.");
     return -1;
   }
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) {
     /* LCOV_EXCL_START
      * If this happens, then openssl's DH implementation is busted. */
@@ -461,7 +461,7 @@ crypto_dh_free_(crypto_dh_t *dh)
   DH_free(dh->dh);
   tor_free(dh);
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 void
 crypto_dh_free_all_openssl(void)
diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c
index de81b87b7e..ba226f8756 100644
--- a/src/lib/crypt_ops/crypto_digest.c
+++ b/src/lib/crypt_ops/crypto_digest.c
@@ -23,171 +23,6 @@
 
 #include "lib/arch/bytes.h"
 
-#ifdef ENABLE_NSS
-DISABLE_GCC_WARNING(strict-prototypes)
-#include <pk11pub.h>
-ENABLE_GCC_WARNING(strict-prototypes)
-#else
-
-#include "lib/crypt_ops/crypto_openssl_mgt.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-
-#include <openssl/hmac.h>
-#include <openssl/sha.h>
-
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
-
-#ifdef ENABLE_NSS
-/**
- * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
- * On failure, return SEC_OID_UNKNOWN. */
-static SECOidTag
-digest_alg_to_nss_oid(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: return SEC_OID_SHA1;
-    case DIGEST_SHA256: return SEC_OID_SHA256;
-    case DIGEST_SHA512: return SEC_OID_SHA512;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return SEC_OID_UNKNOWN;
-  }
-}
-
-/* Helper: get an unkeyed digest via pk11wrap */
-static int
-digest_nss_internal(SECOidTag alg,
-                    char *digest, unsigned len_out,
-                    const char *msg, size_t msg_len)
-{
-  if (alg == SEC_OID_UNKNOWN)
-    return -1;
-  tor_assert(msg_len <= UINT_MAX);
-
-  int rv = -1;
-  SECStatus s;
-  PK11Context *ctx = PK11_CreateDigestContext(alg);
-  if (!ctx)
-    return -1;
-
-  s = PK11_DigestBegin(ctx);
-  if (s != SECSuccess)
-    goto done;
-
-  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-
-  unsigned int len = 0;
-  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
-  if (s != SECSuccess)
-    goto done;
-
-  rv = 0;
- done:
-  PK11_DestroyContext(ctx, PR_TRUE);
-  return rv;
-}
-
-/** True iff alg is implemented in our crypto library, and we want to use that
- * implementation */
-static bool
-library_supports_digest(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return true;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return false;
-  }
-}
-#endif
-
-/* Crypto digest functions */
-
-/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
- * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, -1 on failure.
- */
-MOCK_IMPL(int,
-crypto_digest,(char *digest, const char *m, size_t len))
-{
-  tor_assert(m);
-  tor_assert(digest);
-#ifdef ENABLE_NSS
-  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
-#else
-  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
-    return -1;
-  }
-#endif
-  return 0;
-}
-
-/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest256(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA256) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
-#else
-    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
-#endif
-  } else {
-    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
-/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest512(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA512) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
-#else
-    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-           != NULL);
-#endif
-  } else {
-    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
 /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
  * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
  * success, -1 on failure. */
@@ -267,485 +102,6 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg)
   }
 }
 
-/** Intermediate information about the digest of a stream of data. */
-struct crypto_digest_t {
-  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
-   /** State for the digest we're using.  Only one member of the
-    * union is usable, depending on the value of <b>algorithm</b>. Note also
-    * that space for other members might not even be allocated!
-    */
-  union {
-#ifdef ENABLE_NSS
-    PK11Context *ctx;
-#else
-    SHA_CTX sha1; /**< state for SHA1 */
-    SHA256_CTX sha2; /**< state for SHA256 */
-    SHA512_CTX sha512; /**< state for SHA512 */
-#endif
-    keccak_state sha3; /**< state for SHA3-[256,512] */
-  } d;
-};
-
-#ifdef TOR_UNIT_TESTS
-
-digest_algorithm_t
-crypto_digest_get_algorithm(crypto_digest_t *digest)
-{
-  tor_assert(digest);
-
-  return digest->algorithm;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/**
- * Return the number of bytes we need to malloc in order to get a
- * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
- * when we free one.
- */
-static size_t
-crypto_digest_alloc_bytes(digest_algorithm_t alg)
-{
-  /* Helper: returns the number of bytes in the 'f' field of 'st' */
-#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
-  /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
-                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
-  switch (alg) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.ctx);
-#else
-    case DIGEST_SHA1:
-      return END_OF_FIELD(d.sha1);
-    case DIGEST_SHA256:
-      return END_OF_FIELD(d.sha2);
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.sha512);
-#endif
-    case DIGEST_SHA3_256:
-    case DIGEST_SHA3_512:
-      return END_OF_FIELD(d.sha3);
-    default:
-      tor_assert(0); // LCOV_EXCL_LINE
-      return 0;      // LCOV_EXCL_LINE
-  }
-#undef END_OF_FIELD
-#undef STRUCT_FIELD_SIZE
-}
-
-/**
- * Internal function: create and return a new digest object for 'algorithm'.
- * Does not typecheck the algorithm.
- */
-static crypto_digest_t *
-crypto_digest_new_internal(digest_algorithm_t algorithm)
-{
-  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
-  r->algorithm = algorithm;
-
-  switch (algorithm)
-    {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
-      if (BUG(!r->d.ctx)) {
-        tor_free(r);
-        return NULL;
-      }
-      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
-        crypto_digest_free(r);
-        return NULL;
-      }
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Init(&r->d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Init(&r->d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Init(&r->d.sha512);
-      break;
-#endif
-    case DIGEST_SHA3_256:
-      keccak_digest_init(&r->d.sha3, 256);
-      break;
-    case DIGEST_SHA3_512:
-      keccak_digest_init(&r->d.sha3, 512);
-      break;
-    default:
-      tor_assert_unreached();
-    }
-
-  return r;
-}
-
-/** Allocate and return a new digest object to compute SHA1 digests.
- */
-crypto_digest_t *
-crypto_digest_new(void)
-{
-  return crypto_digest_new_internal(DIGEST_SHA1);
-}
-
-/** Allocate and return a new digest object to compute 256-bit digests
- * using <b>algorithm</b>.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
- * C_RUST_COUPLED: `crypto::digest::Sha256::default`
- */
-crypto_digest_t *
-crypto_digest256_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Allocate and return a new digest object to compute 512-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest512_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Deallocate a digest object.
- */
-void
-crypto_digest_free_(crypto_digest_t *digest)
-{
-  if (!digest)
-    return;
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
-  }
-#endif
-  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memwipe(digest, 0, bytes);
-  tor_free(digest);
-}
-
-/** Add <b>len</b> bytes from <b>data</b> to the digest object.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
- * C_RUST_COUPLED: `crypto::digest::Sha256::process`
- */
-void
-crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                        size_t len)
-{
-  tor_assert(digest);
-  tor_assert(data);
-  /* Using the SHA*_*() calls directly means we don't support doing
-   * SHA in hardware. But so far the delay of getting the question
-   * to the hardware, and hearing the answer, is likely higher than
-   * just doing it ourselves. Hashes are fast.
-   */
-  switch (digest->algorithm) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      tor_assert(len <= UINT_MAX);
-      SECStatus s = PK11_DigestOp(digest->d.ctx,
-                                  (const unsigned char *)data,
-                                  (unsigned int)len);
-      tor_assert(s == SECSuccess);
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Update(&digest->d.sha1, (void*)data, len);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Update(&digest->d.sha2, (void*)data, len);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Update(&digest->d.sha512, (void*)data, len);
-      break;
-#endif
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
-      break;
-    default:
-      /* LCOV_EXCL_START */
-      tor_fragile_assert();
-      break;
-      /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Compute the hash of the data that has been passed to the digest
- * object; write the first out_len bytes of the result to <b>out</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
- * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
- */
-void
-crypto_digest_get_digest(crypto_digest_t *digest,
-                         char *out, size_t out_len)
-{
-  unsigned char r[DIGEST512_LEN];
-  tor_assert(digest);
-  tor_assert(out);
-  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
-
-  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
-   * short output buffers by truncating appropriately. */
-  if (digest->algorithm == DIGEST_SHA3_256 ||
-      digest->algorithm == DIGEST_SHA3_512) {
-    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
-    return;
-  }
-
-#ifdef ENABLE_NSS
-  /* Copy into a temporary buffer since DigestFinal (alters) the context */
-  unsigned char buf[1024];
-  unsigned int saved_len = 0;
-  unsigned rlen;
-  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
-                                               buf, sizeof(buf),
-                                               &saved_len);
-  tor_assert(saved);
-  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
-  tor_assert(s == SECSuccess);
-  tor_assert(rlen >= out_len);
-  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
-  tor_assert(s == SECSuccess);
-  if (saved != buf) {
-    PORT_ZFree(saved, saved_len);
-  }
-#else
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t tmpenv;
-  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
-  memcpy(&tmpenv, digest, alloc_bytes);
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Final(r, &tmpenv.d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Final(r, &tmpenv.d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Final(r, &tmpenv.d.sha512);
-      break;
-//LCOV_EXCL_START
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-    default:
-      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
-      /* This is fatal, because it should never happen. */
-      tor_assert_unreached();
-      break;
-//LCOV_EXCL_STOP
-  }
-#endif
-  memcpy(out, r, out_len);
-  memwipe(r, 0, sizeof(r));
-}
-
-/** Allocate and return a new digest object with the same state as
- * <b>digest</b>
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
- * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
- */
-crypto_digest_t *
-crypto_digest_dup(const crypto_digest_t *digest)
-{
-  tor_assert(digest);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    result->d.ctx = PK11_CloneContext(digest->d.ctx);
-  }
-#endif
-  return result;
-}
-
-/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object.
- */
-void
-crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
-                         const crypto_digest_t *digest)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  tor_assert(bytes <= sizeof(checkpoint->mem));
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    unsigned char *allocated;
-    allocated = PK11_SaveContextAlloc(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      sizeof(checkpoint->mem),
-                                      &checkpoint->bytes_used);
-    /* No allocation is allowed here. */
-    tor_assert(allocated == checkpoint->mem);
-    return;
-  }
-#endif
-  memcpy(checkpoint->mem, digest, bytes);
-}
-
-/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
- * state was previously stored with crypto_digest_checkpoint() */
-void
-crypto_digest_restore(crypto_digest_t *digest,
-                      const crypto_digest_checkpoint_t *checkpoint)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    SECStatus s = PK11_RestoreContext(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      checkpoint->bytes_used);
-    tor_assert(s == SECSuccess);
-    return;
-  }
-#endif
-  memcpy(digest, checkpoint->mem, bytes);
-}
-
-/** Replace the state of the digest object <b>into</b> with the state
- * of the digest object <b>from</b>.  Requires that 'into' and 'from'
- * have the same digest type.
- */
-void
-crypto_digest_assign(crypto_digest_t *into,
-                     const crypto_digest_t *from)
-{
-  tor_assert(into);
-  tor_assert(from);
-  tor_assert(into->algorithm == from->algorithm);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(from->algorithm)) {
-    PK11_DestroyContext(into->d.ctx, PR_TRUE);
-    into->d.ctx = PK11_CloneContext(from->d.ctx);
-    return;
-  }
-#endif
-  memcpy(into,from,alloc_bytes);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of those strings,
- * plus the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist(char *digest_out, size_t len_out,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of: the
- * optional string <b>prepend</b>, those strings,
- * and the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>len_out</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                        const char *prepend,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_t *d = crypto_digest_new_internal(alg);
-  if (prepend)
-    crypto_digest_add_bytes(d, prepend, strlen(prepend));
-  SMARTLIST_FOREACH(lst, const char *, cp,
-                    crypto_digest_add_bytes(d, cp, strlen(cp)));
-  if (append)
-    crypto_digest_add_bytes(d, append, strlen(append));
-  crypto_digest_get_digest(d, digest_out, len_out);
-  crypto_digest_free(d);
-}
-
-/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
- * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
- * result in <b>hmac_out</b>. Asserts on failure.
- */
-void
-crypto_hmac_sha256(char *hmac_out,
-                   const char *key, size_t key_len,
-                   const char *msg, size_t msg_len)
-{
-  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
-  tor_assert(key_len < INT_MAX);
-  tor_assert(msg_len < INT_MAX);
-  tor_assert(hmac_out);
-#ifdef ENABLE_NSS
-  PK11SlotInfo *slot = NULL;
-  PK11SymKey *symKey = NULL;
-  PK11Context *hmac = NULL;
-
-  int ok = 0;
-  SECStatus s;
-  SECItem keyItem, paramItem;
-  keyItem.data = (unsigned char *)key;
-  keyItem.len = (unsigned)key_len;
-  paramItem.type = siBuffer;
-  paramItem.data = NULL;
-  paramItem.len = 0;
-
-  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
-  if (!slot)
-    goto done;
-  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
-                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
-  if (!symKey)
-    goto done;
-
-  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
-                                    &paramItem);
-  if (!hmac)
-    goto done;
-  s = PK11_DigestBegin(hmac);
-  if (s != SECSuccess)
-    goto done;
-  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-  unsigned int len=0;
-  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
-  if (s != SECSuccess || len != DIGEST256_LEN)
-    goto done;
-  ok = 1;
-
- done:
-  if (hmac)
-    PK11_DestroyContext(hmac, PR_TRUE);
-  if (symKey)
-    PK11_FreeSymKey(symKey);
-  if (slot)
-    PK11_FreeSlot(slot);
-
-  tor_assert(ok);
-#else
-  unsigned char *rv = NULL;
-  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
-            (unsigned char*)hmac_out, NULL);
-  tor_assert(rv);
-#endif
-}
-
 /** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
  * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
  * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
@@ -779,7 +135,23 @@ crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
 
 /** Internal state for a eXtendable-Output Function (XOF). */
 struct crypto_xof_t {
+#ifdef OPENSSL_HAS_SHAKE3_EVP
+  /* XXXX We can't enable this yet, because OpenSSL's
+   * DigestFinalXOF function can't be called repeatedly on the same
+   * XOF.
+   *
+   * We could in theory use the undocumented SHA3_absorb and SHA3_squeeze
+   * functions, but let's not mess with undocumented OpenSSL internals any
+   * more than we have to.
+   *
+   * We could also revise our XOF code so that it only allows a single
+   * squeeze operation; we don't require streaming squeeze operations
+   * outside the tests yet.
+   */
+  EVP_MD_CTX *ctx;
+#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */
   keccak_state s;
+#endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */
 };
 
 /** Allocate a new XOF object backed by SHAKE-256.  The security level
@@ -792,7 +164,14 @@ crypto_xof_new(void)
 {
   crypto_xof_t *xof;
   xof = tor_malloc(sizeof(crypto_xof_t));
+#ifdef OPENSSL_HAS_SHAKE256
+  xof->ctx = EVP_MD_CTX_new();
+  tor_assert(xof->ctx);
+  int r = EVP_DigestInit(xof->ctx, EVP_shake256());
+  tor_assert(r == 1);
+#else /* !defined(OPENSSL_HAS_SHAKE256) */
   keccak_xof_init(&xof->s, 256);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
   return xof;
 }
 
@@ -803,8 +182,13 @@ crypto_xof_new(void)
 void
 crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestUpdate(xof->ctx, data, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_absorb(&xof->s, data, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Squeeze bytes out of a XOF object.  Calling this routine will render
@@ -813,8 +197,13 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 void
 crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestFinalXOF(xof->ctx, out, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_squeeze(&xof->s, out, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Cleanse and deallocate a XOF object. */
@@ -823,6 +212,34 @@ crypto_xof_free_(crypto_xof_t *xof)
 {
   if (!xof)
     return;
+#ifdef OPENSSL_HAS_SHAKE256
+  if (xof->ctx)
+    EVP_MD_CTX_free(xof->ctx);
+#endif
   memwipe(xof, 0, sizeof(crypto_xof_t));
   tor_free(xof);
 }
+
+/** Compute the XOF (SHAKE256) of a <b>input_len</b> bytes at <b>input</b>,
+ * putting <b>output_len</b> bytes at <b>output</b>. */
+void
+crypto_xof(uint8_t *output, size_t output_len,
+           const uint8_t *input, size_t input_len)
+{
+#ifdef OPENSSL_HAS_SHA3
+  EVP_MD_CTX *ctx = EVP_MD_CTX_new();
+  tor_assert(ctx);
+  int r = EVP_DigestInit(ctx, EVP_shake256());
+  tor_assert(r == 1);
+  r = EVP_DigestUpdate(ctx, input, input_len);
+  tor_assert(r == 1);
+  r = EVP_DigestFinalXOF(ctx, output, output_len);
+  tor_assert(r == 1);
+  EVP_MD_CTX_free(ctx);
+#else /* !defined(OPENSSL_HAS_SHA3) */
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, input, input_len);
+  crypto_xof_squeeze_bytes(xof, output, output_len);
+  crypto_xof_free(xof);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+}
diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h
index 47e60ce617..5869db7800 100644
--- a/src/lib/crypt_ops/crypto_digest.h
+++ b/src/lib/crypt_ops/crypto_digest.h
@@ -124,6 +124,8 @@ void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
 void crypto_xof_free_(crypto_xof_t *xof);
 #define crypto_xof_free(xof) \
   FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
+void crypto_xof(uint8_t *output, size_t output_len,
+                const uint8_t *input, size_t input_len);
 
 #ifdef TOR_UNIT_TESTS
 digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
diff --git a/src/lib/crypt_ops/crypto_digest_nss.c b/src/lib/crypt_ops/crypto_digest_nss.c
new file mode 100644
index 0000000000..afa8f6d5ef
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_nss.c
@@ -0,0 +1,559 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_nss.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (NSS specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+DISABLE_GCC_WARNING(strict-prototypes)
+#include <pk11pub.h>
+ENABLE_GCC_WARNING(strict-prototypes)
+
+/**
+ * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
+ * On failure, return SEC_OID_UNKNOWN. */
+static SECOidTag
+digest_alg_to_nss_oid(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: return SEC_OID_SHA1;
+    case DIGEST_SHA256: return SEC_OID_SHA256;
+    case DIGEST_SHA512: return SEC_OID_SHA512;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return SEC_OID_UNKNOWN;
+  }
+}
+
+/* Helper: get an unkeyed digest via pk11wrap */
+static int
+digest_nss_internal(SECOidTag alg,
+                    char *digest, unsigned len_out,
+                    const char *msg, size_t msg_len)
+{
+  if (alg == SEC_OID_UNKNOWN)
+    return -1;
+  tor_assert(msg_len <= UINT_MAX);
+
+  int rv = -1;
+  SECStatus s;
+  PK11Context *ctx = PK11_CreateDigestContext(alg);
+  if (!ctx)
+    return -1;
+
+  s = PK11_DigestBegin(ctx);
+  if (s != SECSuccess)
+    goto done;
+
+  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+
+  unsigned int len = 0;
+  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
+  if (s != SECSuccess)
+    goto done;
+
+  rv = 0;
+ done:
+  PK11_DestroyContext(ctx, PR_TRUE);
+  return rv;
+}
+
+/** True iff alg is implemented in our crypto library, and we want to use that
+ * implementation */
+static bool
+library_supports_digest(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return true;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return false;
+  }
+}
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
+  } else {
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
+  } else {
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    PK11Context *ctx;
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.ctx);
+    case DIGEST_SHA3_256:
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
+      if (BUG(!r->d.ctx)) {
+        tor_free(r);
+        return NULL;
+      }
+      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+  if (library_supports_digest(digest->algorithm)) {
+    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
+  }
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      tor_assert(len <= UINT_MAX);
+      SECStatus s = PK11_DigestOp(digest->d.ctx,
+                                  (const unsigned char *)data,
+                                  (unsigned int)len);
+      tor_assert(s == SECSuccess);
+      break;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+  }
+
+  /* Copy into a temporary buffer since DigestFinal (alters) the context */
+  unsigned char buf[1024];
+  unsigned int saved_len = 0;
+  unsigned rlen;
+  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
+                                               buf, sizeof(buf),
+                                               &saved_len);
+  tor_assert(saved);
+  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
+  tor_assert(s == SECSuccess);
+  tor_assert(rlen >= out_len);
+  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
+  tor_assert(s == SECSuccess);
+
+  if (saved != buf) {
+    PORT_ZFree(saved, saved_len);
+  }
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+  if (library_supports_digest(digest->algorithm)) {
+    result->d.ctx = PK11_CloneContext(digest->d.ctx);
+  }
+
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  if (library_supports_digest(digest->algorithm)) {
+    unsigned char *allocated;
+    allocated = PK11_SaveContextAlloc(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      sizeof(checkpoint->mem),
+                                      &checkpoint->bytes_used);
+    /* No allocation is allowed here. */
+    tor_assert(allocated == checkpoint->mem);
+    return;
+  }
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  if (library_supports_digest(digest->algorithm)) {
+    SECStatus s = PK11_RestoreContext(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      checkpoint->bytes_used);
+    tor_assert(s == SECSuccess);
+    return;
+  }
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+  if (library_supports_digest(from->algorithm)) {
+    PK11_DestroyContext(into->d.ctx, PR_TRUE);
+    into->d.ctx = PK11_CloneContext(from->d.ctx);
+    return;
+  }
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+
+  PK11SlotInfo *slot = NULL;
+  PK11SymKey *symKey = NULL;
+  PK11Context *hmac = NULL;
+
+  int ok = 0;
+  SECStatus s;
+  SECItem keyItem, paramItem;
+  keyItem.data = (unsigned char *)key;
+  keyItem.len = (unsigned)key_len;
+  paramItem.type = siBuffer;
+  paramItem.data = NULL;
+  paramItem.len = 0;
+
+  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
+  if (!slot)
+    goto done;
+  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
+                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
+  if (!symKey)
+    goto done;
+
+  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
+                                    &paramItem);
+  if (!hmac)
+    goto done;
+  s = PK11_DigestBegin(hmac);
+  if (s != SECSuccess)
+    goto done;
+  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+  unsigned int len=0;
+  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
+  if (s != SECSuccess || len != DIGEST256_LEN)
+    goto done;
+  ok = 1;
+
+ done:
+  if (hmac)
+    PK11_DestroyContext(hmac, PR_TRUE);
+  if (symKey)
+    PK11_FreeSymKey(symKey);
+  if (slot)
+    PK11_FreeSlot(slot);
+
+  tor_assert(ok);
+}
diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c
new file mode 100644
index 0000000000..50a8ff4d27
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_openssl.c
@@ -0,0 +1,522 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_openssl.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (OpenSSL specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+
+DISABLE_GCC_WARNING(redundant-decls)
+
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+ENABLE_GCC_WARNING(redundant-decls)
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
+    return -1;
+  }
+  return 0;
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST256_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL);
+#else
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
+           != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST512_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL);
+#else
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    SHA_CTX sha1; /**< state for SHA1 */
+    SHA256_CTX sha2; /**< state for SHA256 */
+    SHA512_CTX sha512; /**< state for SHA512 */
+#ifdef OPENSSL_HAS_SHA3
+    EVP_MD_CTX *md;
+#else
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+#endif
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1:
+      return END_OF_FIELD(d.sha1);
+    case DIGEST_SHA256:
+      return END_OF_FIELD(d.sha2);
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.sha512);
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.md);
+#else
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1:
+      SHA1_Init(&r->d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Init(&r->d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Init(&r->d.sha512);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_512:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    if (digest->d.md) {
+      EVP_MD_CTX_free(digest->d.md);
+    }
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Update(&digest->d.sha1, (void*)data, len);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Update(&digest->d.sha2, (void*)data, len);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Update(&digest->d.sha512, (void*)data, len);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: {
+      int r = EVP_DigestUpdate(digest->d.md, data, len);
+      tor_assert(r);
+  }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned dlen = (unsigned)
+      crypto_digest_algorithm_get_length(digest->algorithm);
+    EVP_MD_CTX *tmp = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(tmp, digest->d.md);
+    memset(r, 0xff, sizeof(r));
+    int res = EVP_DigestFinal(tmp, r, &dlen);
+    EVP_MD_CTX_free(tmp);
+    tor_assert(res == 1);
+    goto done;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    /* Tiny-Keccak handles copying into a temporary ctx, and also can handle
+     * short output buffers by truncating appropriately. */
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t tmpenv;
+  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
+  memcpy(&tmpenv, digest, alloc_bytes);
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Final(r, &tmpenv.d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Final(r, &tmpenv.d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Final(r, &tmpenv.d.sha512);
+      break;
+//LCOV_EXCL_START
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+    default:
+      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
+      /* This is fatal, because it should never happen. */
+      tor_assert_unreached();
+      break;
+//LCOV_EXCL_STOP
+  }
+#ifdef OPENSSL_HAS_SHA3
+ done:
+#endif
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    result->d.md = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(result->d.md, digest->d.md);
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (from->algorithm == DIGEST_SHA3_256 ||
+      from->algorithm == DIGEST_SHA3_512) {
+    EVP_MD_CTX_copy(into->d.md, from->d.md);
+    return;
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+  unsigned char *rv = NULL;
+  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
+            (unsigned char*)hmac_out, NULL);
+  tor_assert(rv);
+}
+
diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c
index 0a442bb739..c28111a5a5 100644
--- a/src/lib/crypt_ops/crypto_ed25519.c
+++ b/src/lib/crypt_ops/crypto_ed25519.c
@@ -226,7 +226,7 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong)
 int
 ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey)
 {
-  return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
+  return safe_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
 }
 
 /* Return a heap-allocated array that contains <b>msg</b> prefixed by the
diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c
index 84f73e5272..118cd79045 100644
--- a/src/lib/crypt_ops/crypto_format.c
+++ b/src/lib/crypt_ops/crypto_format.c
@@ -104,7 +104,7 @@ crypto_read_tagged_contents_from_file(const char *fname,
   prefix[32] = 0;
   /* Check type, extract tag. */
   if (strcmpstart(prefix, "== ") || strcmpend(prefix, " ==") ||
-      ! tor_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
+      ! fast_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
     saved_errno = EINVAL;
     goto end;
   }
@@ -131,20 +131,27 @@ crypto_read_tagged_contents_from_file(const char *fname,
   return r;
 }
 
-/** Encode <b>pkey</b> as a base64-encoded string, without trailing "="
+/** Encode <b>pkey</b> as a base64-encoded string, including trailing "="
  * characters, in the buffer <b>output</b>, which must have at least
- * CURVE25519_BASE64_PADDED_LEN+1 bytes available.  Return 0 on success, -1 on
- * failure. */
-int
+ * CURVE25519_BASE64_PADDED_LEN+1 bytes available.
+ * Can not fail.
+ *
+ * Careful! CURVE25519_BASE64_PADDED_LEN is one byte longer than
+ * ED25519_BASE64_LEN.
+ */
+void
 curve25519_public_to_base64(char *output,
                             const curve25519_public_key_t *pkey)
 {
   char buf[128];
-  base64_encode(buf, sizeof(buf),
-                (const char*)pkey->public_key, CURVE25519_PUBKEY_LEN, 0);
-  buf[CURVE25519_BASE64_PADDED_LEN] = '\0';
+  int n = base64_encode(buf, sizeof(buf),
+                        (const char*)pkey->public_key,
+                        CURVE25519_PUBKEY_LEN, 0);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode(). */
+  tor_assert(n == CURVE25519_BASE64_PADDED_LEN);
+  tor_assert(buf[CURVE25519_BASE64_PADDED_LEN] == '\0');
   memcpy(output, buf, CURVE25519_BASE64_PADDED_LEN+1);
-  return 0;
 }
 
 /** Try to decode a base64-encoded curve25519 public key from <b>input</b>
@@ -181,8 +188,7 @@ ed25519_fmt(const ed25519_public_key_t *pkey)
     if (ed25519_public_key_is_zero(pkey)) {
       strlcpy(formatted, "<unset>", sizeof(formatted));
     } else {
-      int r = ed25519_public_to_base64(formatted, pkey);
-      tor_assert(!r);
+      ed25519_public_to_base64(formatted, pkey);
     }
   } else {
     strlcpy(formatted, "<null>", sizeof(formatted));
@@ -202,28 +208,35 @@ ed25519_public_from_base64(ed25519_public_key_t *pkey,
 
 /** Encode the public key <b>pkey</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_BASE64_LEN bytes of encoded key,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
+ *
+ * Careful! ED25519_BASE64_LEN is one byte shorter than
+ * CURVE25519_BASE64_PADDED_LEN.
  */
-int
+void
 ed25519_public_to_base64(char *output,
                          const ed25519_public_key_t *pkey)
 {
-  return digest256_to_base64(output, (const char *)pkey->pubkey);
+  digest256_to_base64(output, (const char *)pkey->pubkey);
 }
 
 /** Encode the signature <b>sig</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_SIG_BASE64_LEN bytes of encoded signature,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
  */
-int
+void
 ed25519_signature_to_base64(char *output,
                             const ed25519_signature_t *sig)
 {
   char buf[256];
   int n = base64_encode_nopad(buf, sizeof(buf), sig->sig, ED25519_SIG_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
   tor_assert(n == ED25519_SIG_BASE64_LEN);
+  tor_assert(buf[ED25519_SIG_BASE64_LEN] == '\0');
   memcpy(output, buf, ED25519_SIG_BASE64_LEN+1);
-  return 0;
 }
 
 /** Try to decode the string <b>input</b> into an ed25519 signature. On
@@ -233,16 +246,11 @@ int
 ed25519_signature_from_base64(ed25519_signature_t *sig,
                               const char *input)
 {
-
   if (strlen(input) != ED25519_SIG_BASE64_LEN)
     return -1;
-  char buf[ED25519_SIG_BASE64_LEN+3];
-  memcpy(buf, input, ED25519_SIG_BASE64_LEN);
-  buf[ED25519_SIG_BASE64_LEN+0] = '=';
-  buf[ED25519_SIG_BASE64_LEN+1] = '=';
-  buf[ED25519_SIG_BASE64_LEN+2] = 0;
   char decoded[128];
-  int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf));
+  int n = base64_decode(decoded, sizeof(decoded), input,
+                        ED25519_SIG_BASE64_LEN);
   if (n < 0 || n != ED25519_SIG_LEN)
     return -1;
   memcpy(sig->sig, decoded, ED25519_SIG_LEN);
@@ -250,24 +258,26 @@ ed25519_signature_from_base64(ed25519_signature_t *sig,
   return 0;
 }
 
-/** Base64 encode DIGEST_LINE bytes from <b>digest</b>, remove the trailing =
+/** Base64 encode DIGEST_LEN bytes from <b>digest</b>, remove the trailing =
  * characters, and store the nul-terminated result in the first
- * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.  */
-/* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST_LEN, 0);
-  buf[BASE64_DIGEST_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST_LEN);
+  tor_assert(buf[BASE64_DIGEST_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest_from_base64(char *digest, const char *d64)
 {
@@ -279,22 +289,24 @@ digest_from_base64(char *digest, const char *d64)
 
 /** Base64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the
  * trailing = characters, and store the nul-terminated result in the first
- * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */
- /* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest256_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN, 0);
-  buf[BASE64_DIGEST256_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST256_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST256_LEN);
+  tor_assert(buf[BASE64_DIGEST256_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST256_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST256_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest256_from_base64(char *digest, const char *d64)
 {
diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h
index fe852e6a61..b4b3aa189c 100644
--- a/src/lib/crypt_ops/crypto_format.h
+++ b/src/lib/crypt_ops/crypto_format.h
@@ -33,18 +33,18 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname,
 
 int ed25519_public_from_base64(struct ed25519_public_key_t *pkey,
                                const char *input);
-int ed25519_public_to_base64(char *output,
-                             const struct ed25519_public_key_t *pkey);
+void ed25519_public_to_base64(char *output,
+                              const struct ed25519_public_key_t *pkey);
 const char *ed25519_fmt(const struct ed25519_public_key_t *pkey);
 
 int ed25519_signature_from_base64(struct ed25519_signature_t *sig,
                                   const char *input);
-int ed25519_signature_to_base64(char *output,
-                                const struct ed25519_signature_t *sig);
+void ed25519_signature_to_base64(char *output,
+                                 const struct ed25519_signature_t *sig);
 
-int digest_to_base64(char *d64, const char *digest);
+void digest_to_base64(char *d64, const char *digest);
 int digest_from_base64(char *digest, const char *d64);
-int digest256_to_base64(char *d64, const char *digest);
+void digest256_to_base64(char *d64, const char *digest);
 int digest256_from_base64(char *digest, const char *d64);
 
 #endif /* !defined(TOR_CRYPTO_FORMAT_H) */
diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c
index fd2e701651..e0d241d4ea 100644
--- a/src/lib/crypt_ops/crypto_hkdf.c
+++ b/src/lib/crypt_ops/crypto_hkdf.c
@@ -25,7 +25,7 @@
 #include <openssl/kdf.h>
 #define HAVE_OPENSSL_HKDF 1
 #endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <string.h>
 
@@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl(
   return 0;
 }
 
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
 
 /**
  * Perform RFC5869 HKDF computation using our own legacy implementation.
@@ -166,7 +166,7 @@ crypto_expand_key_material_rfc5869_sha256_legacy(
   memwipe(mac, 0, sizeof(mac));
   return 0;
 }
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 
 /** Expand some secret key material according to RFC5869, using SHA256 as the
  * underlying hash.  The <b>key_in_len</b> bytes at <b>key_in</b> are the
@@ -191,11 +191,11 @@ crypto_expand_key_material_rfc5869_sha256(
                                              salt_in_len, info_in,
                                              info_in_len,
                                              key_out, key_out_len);
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
   return crypto_expand_key_material_rfc5869_sha256_legacy(key_in,
                                                key_in_len, salt_in,
                                                salt_in_len, info_in,
                                                info_in_len,
                                                key_out, key_out_len);
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 }
diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c
index 329c264af6..a16bf4e11a 100644
--- a/src/lib/crypt_ops/crypto_init.c
+++ b/src/lib/crypt_ops/crypto_init.c
@@ -12,6 +12,8 @@
 
 #include "orconfig.h"
 
+#define CRYPTO_PRIVATE
+
 #include "lib/crypt_ops/crypto_init.h"
 
 #include "lib/crypt_ops/crypto_curve25519.h"
@@ -20,8 +22,11 @@
 #include "lib/crypt_ops/crypto_openssl_mgt.h"
 #include "lib/crypt_ops/crypto_nss_mgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_sys.h"
+
+#include "lib/subsys/subsys.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** Boolean: has our crypto library been initialized? (early phase) */
 static int crypto_early_initialized_ = 0;
@@ -66,6 +71,8 @@ crypto_early_init(void)
     if (crypto_init_siphash_key() < 0)
       return -1;
 
+    crypto_rand_fast_init();
+
     curve25519_init();
     ed25519_init();
   }
@@ -92,7 +99,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
     (void)useAccel;
     (void)accelName;
     (void)accelDir;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
     if (crypto_nss_late_init() < 0)
       return -1;
@@ -108,6 +115,7 @@ crypto_thread_cleanup(void)
 #ifdef ENABLE_OPENSSL
   crypto_openssl_thread_cleanup();
 #endif
+  destroy_thread_fast_rng();
 }
 
 /**
@@ -126,6 +134,8 @@ crypto_global_cleanup(void)
   crypto_nss_global_cleanup();
 #endif
 
+  crypto_rand_fast_shutdown();
+
   crypto_early_initialized_ = 0;
   crypto_global_initialized_ = 0;
   have_seeded_siphash = 0;
@@ -142,6 +152,12 @@ crypto_prefork(void)
 #ifdef ENABLE_NSS
   crypto_nss_prefork();
 #endif
+  /* It is not safe to share a fast_rng object across a fork boundary unless
+   * we actually have zero-on-fork support in map_anon.c.  If we have
+   * drop-on-fork support, we will crash; if we have neither, we will yield
+   * a copy of the parent process's rng, which is scary and insecure.
+   */
+  destroy_thread_fast_rng();
 }
 
 /** Run operations that the crypto library requires to be happy again
@@ -202,3 +218,47 @@ tor_is_using_nss(void)
   return 0;
 #endif
 }
+
+static int
+subsys_crypto_initialize(void)
+{
+  if (crypto_early_init() < 0)
+    return -1;
+  crypto_dh_init();
+  return 0;
+}
+
+static void
+subsys_crypto_shutdown(void)
+{
+  crypto_global_cleanup();
+}
+
+static void
+subsys_crypto_prefork(void)
+{
+  crypto_prefork();
+}
+
+static void
+subsys_crypto_postfork(void)
+{
+  crypto_postfork();
+}
+
+static void
+subsys_crypto_thread_cleanup(void)
+{
+  crypto_thread_cleanup();
+}
+
+const struct subsys_fns_t sys_crypto = {
+  .name = "crypto",
+  .supported = true,
+  .level = -60,
+  .initialize = subsys_crypto_initialize,
+  .shutdown = subsys_crypto_shutdown,
+  .prefork = subsys_crypto_prefork,
+  .postfork = subsys_crypto_postfork,
+  .thread_cleanup = subsys_crypto_thread_cleanup,
+};
diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h
index 540d08eb56..8de3eb03ed 100644
--- a/src/lib/crypt_ops/crypto_init.h
+++ b/src/lib/crypt_ops/crypto_init.h
@@ -33,4 +33,4 @@ const char *crypto_get_header_version_string(void);
 
 int tor_is_using_nss(void);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_INIT_H) */
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h
index 72fd2a1229..4cfa9b42a4 100644
--- a/src/lib/crypt_ops/crypto_nss_mgt.h
+++ b/src/lib/crypt_ops/crypto_nss_mgt.h
@@ -29,6 +29,6 @@ void crypto_nss_global_cleanup(void);
 
 void crypto_nss_prefork(void);
 void crypto_nss_postfork(void);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
-#endif /* !defined(TOR_CRYPTO_NSS_H) */
+#endif /* !defined(TOR_CRYPTO_NSS_MGT_H) */
diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c
index 2186d2a939..ed832d852e 100644
--- a/src/lib/crypt_ops/crypto_ope.c
+++ b/src/lib/crypt_ops/crypto_ope.c
@@ -57,9 +57,9 @@ ope_val_from_le(ope_val_t x)
     ((x) >> 8) |
     (((x)&0xff) << 8);
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
 #define ope_val_from_le(x) (x)
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
 /**
  * Return a new AES256-CTR stream cipher object for <b>ope</b>, ready to yield
diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h
index 610d956335..9778dfe0f0 100644
--- a/src/lib/crypt_ops/crypto_ope.h
+++ b/src/lib/crypt_ops/crypto_ope.h
@@ -41,6 +41,6 @@ struct aes_cnt_cipher;
 STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope,
                                               uint32_t initial_idx);
 STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n);
-#endif
+#endif /* defined(CRYPTO_OPE_PRIVATE) */
 
-#endif
+#endif /* !defined(CRYPTO_OPE_H) */
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c
index c97815f9a4..f51309219a 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.c
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.c
@@ -176,6 +176,10 @@ crypto_openssl_free_all(void)
   tor_free(crypto_openssl_version_str);
   tor_free(crypto_openssl_header_version_str);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
 #ifndef NEW_THREAD_API
   if (n_openssl_mutexes_) {
     int n = n_openssl_mutexes_;
@@ -200,10 +204,10 @@ crypto_openssl_early_init(void)
                      OPENSSL_INIT_LOAD_CRYPTO_STRINGS |
                      OPENSSL_INIT_ADD_ALL_CIPHERS |
                      OPENSSL_INIT_ADD_ALL_DIGESTS, NULL);
-#else
+#else /* !defined(OPENSSL_1_1_API) */
     ERR_load_crypto_strings();
     OpenSSL_add_all_algorithms();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
     setup_openssl_threading();
 
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h
index a3dd03aa04..111a2d12ed 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.h
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.h
@@ -84,6 +84,6 @@ int crypto_openssl_late_init(int useAccel, const char *accelName,
 void crypto_openssl_thread_cleanup(void);
 void crypto_openssl_global_cleanup(void);
 
-#endif /* ENABLE_OPENSSL */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #endif /* !defined(TOR_CRYPTO_OPENSSL_H) */
diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c
index 915fe0870d..afbafbfa35 100644
--- a/src/lib/crypt_ops/crypto_rand.c
+++ b/src/lib/crypt_ops/crypto_rand.c
@@ -11,7 +11,6 @@
  * number generators, and working with randomness.
  **/
 
-#ifndef CRYPTO_RAND_PRIVATE
 #define CRYPTO_RAND_PRIVATE
 
 #include "lib/crypt_ops/crypto_rand.h"
@@ -37,6 +36,7 @@
 
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/crypto_digest.h"
+#include "lib/ctime/di_ops.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
@@ -47,7 +47,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/rand.h>
 #include <openssl/sha.h>
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 #include <pk11pub.h>
@@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
   (void)out;
   (void)out_len;
   return -1;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   static const char *filenames[] = {
     "/dev/srandom", "/dev/urandom", "/dev/random", NULL
   };
@@ -315,7 +315,7 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
       }
     }
 
-    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
+    if ((out_len < sanity_min_size) || !safe_mem_is_zero((char*)out, out_len))
       return 0;
   }
 
@@ -419,7 +419,7 @@ crypto_seed_openssl_rng(void)
   else
     return -1;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 /**
@@ -442,7 +442,7 @@ crypto_seed_nss_rng(void)
 
   return load_entropy_ok ? 0 : -1;
 }
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 /**
  * Seed the RNG for any and all crypto libraries that we're using with bytes
@@ -520,121 +520,24 @@ crypto_rand_unmocked(char *to, size_t n)
 
 #undef BUFLEN
   }
-#else
+#else /* !defined(ENABLE_NSS) */
   int r = RAND_bytes((unsigned char*)to, (int)n);
   /* We consider a PRNG failure non-survivable. Let's assert so that we get a
    * stack trace about where it happened.
    */
   tor_assert(r >= 0);
-#endif
+#endif /* defined(ENABLE_NSS) */
 }
 
 /**
- * Return a pseudorandom integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
- * INT_MAX+1, inclusive.
+ * Draw an unsigned 32-bit integer uniformly at random.
  */
-int
-crypto_rand_int(unsigned int max)
-{
-  unsigned int val;
-  unsigned int cutoff;
-  tor_assert(max <= ((unsigned int)INT_MAX)+1);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT_MAX - (UINT_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom integer, chosen uniformly from the values i such
- * that min <= i < max.
- *
- * <b>min</b> MUST be in range [0, <b>max</b>).
- * <b>max</b> MUST be in range (min, INT_MAX].
- **/
-int
-crypto_rand_int_range(unsigned int min, unsigned int max)
-{
-  tor_assert(min < max);
-  tor_assert(max <= INT_MAX);
-
-  /* The overflow is avoided here because crypto_rand_int() returns a value
-   * between 0 and (max - min) inclusive. */
-  return min + crypto_rand_int(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports uint64_t.
- **/
-uint64_t
-crypto_rand_uint64_range(uint64_t min, uint64_t max)
+uint32_t
+crypto_rand_u32(void)
 {
-  tor_assert(min < max);
-  return min + crypto_rand_uint64(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports time_t.
- **/
-time_t
-crypto_rand_time_range(time_t min, time_t max)
-{
-  tor_assert(min < max);
-  return min + (time_t)crypto_rand_uint64(max - min);
-}
-
-/**
- * Return a pseudorandom 64-bit integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.
- **/
-uint64_t
-crypto_rand_uint64(uint64_t max)
-{
-  uint64_t val;
-  uint64_t cutoff;
-  tor_assert(max < UINT64_MAX);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT64_MAX - (UINT64_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom double d, chosen uniformly from the range
- * 0.0 <= d < 1.0.
- **/
-double
-crypto_rand_double(void)
-{
-  /* We just use an unsigned int here; we don't really care about getting
-   * more than 32 bits of resolution */
-  unsigned int u;
-  crypto_rand((char*)&u, sizeof(u));
-#if SIZEOF_INT == 4
-#define UINT_MAX_AS_DOUBLE 4294967296.0
-#elif SIZEOF_INT == 8
-#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
-#else
-#error SIZEOF_INT is neither 4 nor 8
-#endif /* SIZEOF_INT == 4 || ... */
-  return ((double)u) / UINT_MAX_AS_DOUBLE;
+  uint32_t rand;
+  crypto_rand((void*)&rand, sizeof(rand));
+  return rand;
 }
 
 /**
@@ -724,8 +627,6 @@ crypto_force_rand_ssleay(void)
     RAND_set_rand_method(default_method);
     return 1;
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   return 0;
 }
-
-#endif /* !defined(CRYPTO_RAND_PRIVATE) */
diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h
index 86fa20faa3..a019287aa9 100644
--- a/src/lib/crypt_ops/crypto_rand.h
+++ b/src/lib/crypt_ops/crypto_rand.h
@@ -16,6 +16,7 @@
 #include "lib/cc/compat_compiler.h"
 #include "lib/cc/torint.h"
 #include "lib/testsupport/testsupport.h"
+#include "lib/malloc/malloc.h"
 
 /* random numbers */
 int crypto_seed_rng(void) ATTR_WUR;
@@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n);
 void crypto_strongest_rand(uint8_t *out, size_t out_len);
 MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len));
 int crypto_rand_int(unsigned int max);
+unsigned crypto_rand_uint(unsigned limit);
 int crypto_rand_int_range(unsigned int min, unsigned int max);
 uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
 time_t crypto_rand_time_range(time_t min, time_t max);
+uint32_t crypto_rand_u32(void);
 uint64_t crypto_rand_uint64(uint64_t max);
 double crypto_rand_double(void);
 struct tor_weak_rng_t;
@@ -40,6 +43,61 @@ void *smartlist_choose(const struct smartlist_t *sl);
 void smartlist_shuffle(struct smartlist_t *sl);
 int crypto_force_rand_ssleay(void);
 
+/**
+ * A fast PRNG, for use when the PRNG provided by our crypto library isn't
+ * fast enough.  This one _should_ be cryptographically strong, but
+ * has seen less auditing than the PRNGs in OpenSSL and NSS.  Use with
+ * caution.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+typedef struct crypto_fast_rng_t crypto_fast_rng_t;
+/**
+ * Number of bytes used to seed a crypto_rand_fast_t.
+ **/
+crypto_fast_rng_t *crypto_fast_rng_new(void);
+#define CRYPTO_FAST_RNG_SEED_LEN 48
+crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed);
+void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n);
+void crypto_fast_rng_free_(crypto_fast_rng_t *);
+#define crypto_fast_rng_free(c)                                 \
+  FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c))
+
+unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit);
+uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit);
+uint32_t crypto_fast_rng_get_u32(crypto_fast_rng_t *rng);
+uint64_t crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                                      uint64_t min, uint64_t max);
+double crypto_fast_rng_get_double(crypto_fast_rng_t *rng);
+
+/**
+ * Using the fast_rng <b>rng</b>, yield true with probability
+ * 1/<b>n</b>. Otherwise yield false.
+ *
+ * <b>n</b> must not be zero.
+ **/
+#define crypto_fast_rng_one_in_n(rng, n)        \
+  (0 == (crypto_fast_rng_get_uint((rng), (n))))
+
+crypto_fast_rng_t *get_thread_fast_rng(void);
+
+#ifdef CRYPTO_PRIVATE
+/* These are only used from crypto_init.c */
+void destroy_thread_fast_rng(void);
+void crypto_rand_fast_init(void);
+void crypto_rand_fast_shutdown(void);
+#endif /* defined(CRYPTO_PRIVATE) */
+
+#if defined(TOR_UNIT_TESTS)
+/* Used for white-box testing */
+size_t crypto_fast_rng_get_bytes_used_per_stream(void);
+/* For deterministic prng implementations */
+void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng);
+/* To override the prng for testing. */
+crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng);
+#endif /* defined(TOR_UNIT_TESTS) */
+
 #ifdef CRYPTO_RAND_PRIVATE
 
 STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c
new file mode 100644
index 0000000000..e6ceb42ccb
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_fast.c
@@ -0,0 +1,439 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand_fast.c
+ *
+ * \brief A fast strong PRNG for use when our underlying cryptographic
+ *   library's PRNG isn't fast enough.
+ **/
+
+/* This library is currently implemented to use the same implementation
+ * technique as libottery, using AES-CTR-256 as our underlying stream cipher.
+ * It's backtracking-resistant immediately, and prediction-resistant after
+ * a while.
+ *
+ * Here's how it works:
+ *
+ * We generate pseudorandom bytes using AES-CTR-256.  We generate BUFLEN bytes
+ * at a time.  When we do this, we keep the first SEED_LEN bytes as the key
+ * and the IV for our next invocation of AES_CTR, and yield the remaining
+ * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG.  As we yield
+ * bytes to the user, we clear them from the buffer.
+ *
+ * After we have refilled the buffer RESEED_AFTER times, we mix in an
+ * additional SEED_LEN bytes from our strong PRNG into the seed.
+ *
+ * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN
+ * bytes from the PRNG and use them with our stream cipher to fill the user's
+ * request.
+ */
+
+#define CRYPTO_RAND_FAST_PRIVATE
+#define CRYPTO_PRIVATE
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/intmath/cmp.h"
+#include "lib/cc/ctassert.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/thread/threads.h"
+
+#include "lib/log/util_bug.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <string.h>
+
+#ifdef NOINHERIT_CAN_FAIL
+#define CHECK_PID
+#endif
+
+#ifdef CHECK_PID
+#define PID_FIELD_LEN sizeof(pid_t)
+#else
+#define PID_FIELD_LEN 0
+#endif
+
+/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter.
+ */
+#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN)
+
+/* The amount of space that we mmap for a crypto_fast_rng_t.
+ */
+#define MAPLEN 4096
+
+/* The number of random bytes that we can yield to the user after each
+ * time we fill a crypto_fast_rng_t's buffer.
+ */
+#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN - PID_FIELD_LEN)
+
+/* The number of buffer refills after which we should fetch more
+ * entropy from crypto_strongest_rand().
+ */
+#define RESEED_AFTER 16
+
+/* The length of the stream cipher key we will use for the PRNG, in bytes.
+ */
+#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN)
+/* The length of the stream cipher key we will use for the PRNG, in bits.
+ */
+#define KEY_BITS (KEY_LEN * 8)
+
+/* Make sure that we have a key length we can actually use with AES. */
+CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256);
+
+struct crypto_fast_rng_t {
+  /** How many more fills does this buffer have before we should mix
+   * in the output of crypto_strongest_rand()?
+   *
+   * This value may be negative if unit tests are enabled.  If so, it
+   * indicates that we should never mix in extra data from
+   * crypto_strongest_rand().
+   */
+  int16_t n_till_reseed;
+  /** How many bytes are remaining in cbuf.bytes? */
+  uint16_t bytes_left;
+#ifdef CHECK_PID
+  /** Which process owns this fast_rng?  If this value is zero, we do not
+   * need to test the owner. */
+  pid_t owner;
+#endif
+  struct cbuf {
+    /** The seed (key and IV) that we will use the next time that we refill
+     * cbuf. */
+    uint8_t seed[SEED_LEN];
+    /**
+     * Bytes that we are yielding to the user.  The next byte to be
+     * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this
+     * array are set to zero.
+     */
+    uint8_t bytes[BUFLEN];
+  } buf;
+};
+
+/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf.
+ */
+CTASSERT(sizeof(struct cbuf) == BUFLEN+SEED_LEN);
+/* We're trying to fit all of the RNG state into a nice mmapable chunk.
+ */
+CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN);
+
+/**
+ * Initialize and return a new fast PRNG, using a strong random seed.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new(void)
+{
+  uint8_t seed[SEED_LEN];
+  crypto_strongest_rand(seed, sizeof(seed));
+  crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed);
+  memwipe(seed, 0, sizeof(seed));
+  return result;
+}
+
+/**
+ * Initialize and return a new fast PRNG, using a seed value specified
+ * in <b>seed</b>.  This value must be CRYPTO_FAST_RNG_SEED_LEN bytes
+ * long.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, you should probably look at get_thread_fast_rng().  Alternatively,
+ * use crypto_rand(), wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new_from_seed(const uint8_t *seed)
+{
+  unsigned inherit = INHERIT_RES_KEEP;
+  /* We try to allocate this object as securely as we can, to avoid
+   * having it get dumped, swapped, or shared after fork.
+   */
+  crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result),
+                                ANONMAP_PRIVATE | ANONMAP_NOINHERIT,
+                                &inherit);
+  memcpy(result->buf.seed, seed, SEED_LEN);
+  /* Causes an immediate refill once the user asks for data. */
+  result->bytes_left = 0;
+  result->n_till_reseed = RESEED_AFTER;
+#ifdef CHECK_PID
+  if (inherit == INHERIT_RES_KEEP) {
+    /* This value will neither be dropped nor zeroed after fork, so we need to
+     * check our pid to make sure we are not sharing it across a fork.  This
+     * can be expensive if the pid value isn't cached, sadly.
+     */
+    result->owner = getpid();
+  }
+#elif defined(_WIN32)
+  /* Windows can't fork(), so there's no need to noinherit. */
+#else
+  /* We decided above that noinherit would always do _something_. Assert here
+   * that we were correct. */
+  tor_assertf(inherit != INHERIT_RES_KEEP,
+              "We failed to create a non-inheritable memory region, even "
+              "though we believed such a failure to be impossible! This is "
+              "probably a bug in Tor support for your platform; please report "
+              "it.");
+#endif /* defined(CHECK_PID) || ... */
+  return result;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more
+ * entropy.
+ */
+void
+crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed = -1;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Helper: create a crypto_cipher_t object from SEED_LEN bytes of
+ * input.  The first KEY_LEN bytes are used as the stream cipher's key,
+ * and the remaining CIPHER_IV_LEN bytes are used as its IV.
+ **/
+static inline crypto_cipher_t *
+cipher_from_seed(const uint8_t *seed)
+{
+  return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS);
+}
+
+/**
+ * Helper: mix additional entropy into <b>rng</b> by using our XOF to mix the
+ * old value for the seed with some additional bytes from
+ * crypto_strongest_rand().
+ **/
+static void
+crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng)
+{
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN);
+  {
+    uint8_t seedbuf[SEED_LEN];
+    crypto_strongest_rand(seedbuf, SEED_LEN);
+    crypto_xof_add_bytes(xof, seedbuf, SEED_LEN);
+    memwipe(seedbuf, 0, SEED_LEN);
+  }
+  crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN);
+  crypto_xof_free(xof);
+}
+
+/**
+ * Helper: refill the seed bytes and output buffer of <b>rng</b>, using
+ * the input seed bytes as input (key and IV) for the stream cipher.
+ *
+ * If the n_till_reseed counter has reached zero, mix more random bytes into
+ * the seed before refilling the buffer.
+ **/
+static void
+crypto_fast_rng_refill(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed--;
+  if (rng->n_till_reseed == 0) {
+    /* It's time to reseed the RNG. */
+    crypto_fast_rng_add_entopy(rng);
+    rng->n_till_reseed = RESEED_AFTER;
+  } else if (rng->n_till_reseed < 0) {
+#ifdef TOR_UNIT_TESTS
+    /* Reseeding is disabled for testing; never do it on this prng. */
+    rng->n_till_reseed = -1;
+#else
+    /* If testing is disabled, this shouldn't be able to become negative. */
+    tor_assert_unreached();
+#endif /* defined(TOR_UNIT_TESTS) */
+  }
+  /* Now fill rng->buf with output from our stream cipher, initialized from
+   * that seed value. */
+  crypto_cipher_t *c = cipher_from_seed(rng->buf.seed);
+  memset(&rng->buf, 0, sizeof(rng->buf));
+  crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf));
+  crypto_cipher_free(c);
+
+  rng->bytes_left = sizeof(rng->buf.bytes);
+}
+
+/**
+ * Release all storage held by <b>rng</b>.
+ **/
+void
+crypto_fast_rng_free_(crypto_fast_rng_t *rng)
+{
+  if (!rng)
+    return;
+  memwipe(rng, 0, sizeof(*rng));
+  tor_munmap_anonymous(rng, sizeof(*rng));
+}
+
+/**
+ * Helper: extract bytes from the PRNG, refilling it as necessary. Does not
+ * optimize the case when the user has asked for a huge output.
+ **/
+static void
+crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out,
+                              const size_t n)
+{
+#ifdef CHECK_PID
+  if (rng->owner) {
+    /* Note that we only need to do this check when we have owner set: that
+     * is, when our attempt to block inheriting failed, and the result was
+     * INHERIT_RES_KEEP.
+     *
+     * If the result was INHERIT_RES_DROP, then any attempt to access the rng
+     * memory after forking will crash.
+     *
+     * If the result was INHERIT_RES_ZERO, then forking will set the bytes_left
+     * and n_till_reseed fields to zero.  This function will call
+     * crypto_fast_rng_refill(), which will in turn reseed the PRNG.
+     *
+     * So we only need to do this test in the case when mmap_anonymous()
+     * returned INHERIT_KEEP.  We avoid doing it needlessly, since getpid() is
+     * often a system call, and that can be slow.
+     */
+    tor_assert(rng->owner == getpid());
+  }
+#endif /* defined(CHECK_PID) */
+
+  size_t bytes_to_yield = n;
+
+  while (bytes_to_yield) {
+    if (rng->bytes_left == 0)
+      crypto_fast_rng_refill(rng);
+
+    const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield);
+
+    tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left);
+    uint8_t *copy_from = rng->buf.bytes +
+      (sizeof(rng->buf.bytes) - rng->bytes_left);
+    memcpy(out, copy_from, to_copy);
+    memset(copy_from, 0, to_copy);
+
+    out += to_copy;
+    bytes_to_yield -= to_copy;
+    rng->bytes_left -= to_copy;
+  }
+}
+
+/**
+ * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>.
+ **/
+void
+crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n)
+{
+  if (PREDICT_UNLIKELY(n > BUFLEN)) {
+    /* The user has asked for a lot of output; generate it from a stream
+     * cipher seeded by the PRNG rather than by pulling it out of the PRNG
+     * directly.
+     */
+    uint8_t seed[SEED_LEN];
+    crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN);
+    crypto_cipher_t *c = cipher_from_seed(seed);
+    memset(out, 0, n);
+    crypto_cipher_crypt_inplace(c, (char*)out, n);
+    crypto_cipher_free(c);
+    memwipe(seed, 0, sizeof(seed));
+    return;
+  }
+
+  crypto_fast_rng_getbytes_impl(rng, out, n);
+}
+
+#if defined(TOR_UNIT_TESTS)
+/** for white-box testing: return the number of bytes that are returned from
+ * the user for each invocation of the stream cipher in this RNG. */
+size_t
+crypto_fast_rng_get_bytes_used_per_stream(void)
+{
+  return BUFLEN;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Thread-local instance for our fast RNG.
+ **/
+static tor_threadlocal_t thread_rng;
+
+/**
+ * Return a per-thread fast RNG, initializing it if necessary.
+ *
+ * You do not need to free this yourself.
+ *
+ * It is NOT safe to share this value across threads.
+ **/
+crypto_fast_rng_t *
+get_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+
+  if (PREDICT_UNLIKELY(rng == NULL)) {
+    rng = crypto_fast_rng_new();
+    tor_threadlocal_set(&thread_rng, rng);
+  }
+
+  return rng;
+}
+
+/**
+ * Used when a thread is exiting: free the per-thread fast RNG if needed.
+ * Invoked from the crypto subsystem's thread-cleanup code.
+ **/
+void
+destroy_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+  if (!rng)
+    return;
+  crypto_fast_rng_free(rng);
+  tor_threadlocal_set(&thread_rng, NULL);
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Replace the current thread's rng with <b>rng</b>. For use by the
+ * unit tests only.  Returns the previous thread rng.
+ **/
+crypto_fast_rng_t *
+crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng)
+{
+  crypto_fast_rng_t *old_rng =  tor_threadlocal_get(&thread_rng);
+  tor_threadlocal_set(&thread_rng, rng);
+  return old_rng;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * initialization code.
+ **/
+void
+crypto_rand_fast_init(void)
+{
+  tor_threadlocal_init(&thread_rng);
+}
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * shutdown code.
+ **/
+void
+crypto_rand_fast_shutdown(void)
+{
+  destroy_thread_fast_rng();
+  tor_threadlocal_destroy(&thread_rng);
+}
diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c
new file mode 100644
index 0000000000..ffbfa2d56c
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_numeric.c
@@ -0,0 +1,194 @@
+/**
+ * \file crypto_rand_numeric.c
+ *
+ * \brief Functions for retrieving uniformly distributed numbers
+ *   from our PRNGs.
+ **/
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/log/util_bug.h"
+
+/**
+ * Implementation macro: yields code that returns a uniform unbiased
+ * random number between 0 and limit.  "type" is the type of the number to
+ * return; "maxval" is the largest possible value of "type"; and "fill_stmt"
+ * is a code snippet that fills an object named "val" with random bits.
+ **/
+#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt)         \
+  do {                                                                  \
+    type val;                                                           \
+    type cutoff;                                                        \
+    tor_assert((limit) > 0);                                            \
+                                                                        \
+    /* We ignore any values that are >= 'cutoff,' to avoid biasing */   \
+    /* the distribution with clipping at the upper end of the type's */ \
+    /* range. */                                                        \
+    cutoff = (maxval) - ((maxval)%(limit));                             \
+    while (1) {                                                         \
+      fill_stmt;                                                        \
+      if (val < cutoff)                                                 \
+        return val % (limit);                                           \
+    }                                                                   \
+  } while (0)
+
+/**
+ * Return a pseudorandom integer chosen uniformly from the values between 0
+ * and <b>limit</b>-1 inclusive. limit must be strictly between 0 and
+ * UINT_MAX. */
+unsigned
+crypto_rand_uint(unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
+ * INT_MAX+1, inclusive.
+ */
+int
+crypto_rand_int(unsigned int max)
+{
+  /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying
+   * to return a signed type. Instead we make sure that the range is
+   * reasonable for a nonnegative int, use crypto_rand_uint(), and cast.
+   */
+  tor_assert(max <= ((unsigned int)INT_MAX)+1);
+
+  return (int)crypto_rand_uint(max);
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values i such
+ * that min <= i < max.
+ *
+ * <b>min</b> MUST be in range [0, <b>max</b>).
+ * <b>max</b> MUST be in range (min, INT_MAX].
+ **/
+int
+crypto_rand_int_range(unsigned int min, unsigned int max)
+{
+  tor_assert(min < max);
+  tor_assert(max <= INT_MAX);
+
+  /* The overflow is avoided here because crypto_rand_int() returns a value
+   * between 0 and (max - min) inclusive. */
+  return min + crypto_rand_int(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports uint64_t.
+ **/
+uint64_t
+crypto_rand_uint64_range(uint64_t min, uint64_t max)
+{
+  tor_assert(min < max);
+  return min + crypto_rand_uint64(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports time_t.
+ **/
+time_t
+crypto_rand_time_range(time_t min, time_t max)
+{
+  tor_assert(min < max);
+  return min + (time_t)crypto_rand_uint64(max - min);
+}
+
+/**
+ * Return a pseudorandom 64-bit integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.
+ **/
+uint64_t
+crypto_rand_uint64(uint64_t max)
+{
+  tor_assert(max < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+#if SIZEOF_INT == 4
+#define UINT_MAX_AS_DOUBLE 4294967296.0
+#elif SIZEOF_INT == 8
+#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
+#else
+#error SIZEOF_INT is neither 4 nor 8
+#endif /* SIZEOF_INT == 4 || ... */
+
+/**
+ * Return a pseudorandom double d, chosen uniformly from the range
+ * 0.0 <= d < 1.0.
+ **/
+double
+crypto_rand_double(void)
+{
+  /* We just use an unsigned int here; we don't really care about getting
+   * more than 32 bits of resolution */
+  unsigned int u;
+  crypto_rand((char*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
+/**
+ * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t
+ */
+unsigned
+crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit)
+{
+  tor_assert(limit < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_u32, but extract the result from a crypto_fast_rng_t.
+ */
+uint32_t
+crypto_fast_rng_get_u32(crypto_fast_rng_t *rng)
+{
+  uint32_t val;
+  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val));
+  return val;
+}
+
+/**
+ * As crypto_rand_uint64_range(), but extract the result from a
+ * crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                             uint64_t min, uint64_t max)
+{
+  /* Handle corrupted input */
+  if (BUG(min >= max)) {
+    return min;
+  }
+
+  return min + crypto_fast_rng_get_uint64(rng, max - min);
+}
+
+/**
+ * As crypto_rand_get_double() but extract the result from a crypto_fast_rng_t.
+ */
+double
+crypto_fast_rng_get_double(crypto_fast_rng_t *rng)
+{
+  unsigned int u;
+  crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c
index 8fd8a8aa7b..1289a82e62 100644
--- a/src/lib/crypt_ops/crypto_rsa.c
+++ b/src/lib/crypt_ops/crypto_rsa.c
@@ -59,7 +59,7 @@ crypto_get_rsa_padding(int padding)
     default: tor_assert(0); return -1; // LCOV_EXCL_LINE
     }
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Compare the public-key components of a and b.  Return non-zero iff
  * a==b.  A NULL key is considered to be distinct from all non-NULL
diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h
index 6d9cc8d30e..01e6d10906 100644
--- a/src/lib/crypt_ops/crypto_rsa.h
+++ b/src/lib/crypt_ops/crypto_rsa.h
@@ -122,7 +122,7 @@ struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env);
 crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa);
 MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,(
                                  crypto_pk_t *env,int private));
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 struct SECKEYPublicKeyStr;
@@ -132,7 +132,7 @@ const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey(
                                            const crypto_pk_t *key);
 const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey(
                                            const crypto_pk_t *key);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src);
 void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src);
@@ -143,6 +143,6 @@ struct SECItemStr;
 STATIC int secitem_uint_cmp(const struct SECItemStr *a,
                             const struct SECItemStr *b);
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_CRYPTO_RSA_H) */
diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c
index 7abf6716f0..fd8fda486e 100644
--- a/src/lib/crypt_ops/crypto_rsa_nss.c
+++ b/src/lib/crypt_ops/crypto_rsa_nss.c
@@ -156,7 +156,7 @@ crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private))
   tor_free(buf);
   return result;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Allocate and return storage for a public key.  The key itself will not yet
  * be set.
diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c
index 17eae24cc2..41d936d25b 100644
--- a/src/lib/crypt_ops/crypto_rsa_openssl.c
+++ b/src/lib/crypt_ops/crypto_rsa_openssl.c
@@ -54,7 +54,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k)
   const BIGNUM *p, *q;
   RSA_get0_factors(k->key, &p, &q);
   return p != NULL; /* XXX/yawning: Should we check q? */
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   return k && k->key && k->key->p;
 #endif /* defined(OPENSSL_1_1_API) */
 }
@@ -288,7 +288,7 @@ crypto_pk_num_bits(crypto_pk_t *env)
   tor_assert(n != NULL);
 
   return RSA_bits(env->key);
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   tor_assert(env->key->n);
   return BN_num_bits(env->key->n);
 #endif /* defined(OPENSSL_1_1_API) */
diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c
index 42276597d4..361db18927 100644
--- a/src/lib/crypt_ops/crypto_s2k.c
+++ b/src/lib/crypt_ops/crypto_s2k.c
@@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv < 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
       SECItem passItem = { .type = siBuffer,
                            .data = (unsigned char *) secret,
                            .len = (int)secret_len };
@@ -325,7 +325,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (alg)
         SECOID_DestroyAlgorithmID(alg, PR_TRUE);
       return rv;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
     }
 
     case S2K_TYPE_SCRYPT: {
@@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv != 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else /* !(defined(HAVE_SCRYPT)) */
+#else /* !defined(HAVE_SCRYPT) */
       return S2K_NO_SCRYPT_SUPPORT;
 #endif /* defined(HAVE_SCRYPT) */
     }
diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h
new file mode 100644
index 0000000000..894243b175
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_crypto.h
+ * \brief Declare subsystem object for the crypto module.
+ **/
+
+#ifndef TOR_CRYPTO_SYS_H
+#define TOR_CRYPTO_SYS_H
+
+extern const struct subsys_fns_t sys_crypto;
+
+#endif /* !defined(TOR_CRYPTO_SYS_H) */
diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c
index 67a1a9eb92..5e3f4a87a1 100644
--- a/src/lib/crypt_ops/crypto_util.c
+++ b/src/lib/crypt_ops/crypto_util.c
@@ -30,7 +30,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/err.h>
 #include <openssl/crypto.h>
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c
index 0dba64d595..c931b58369 100644
--- a/src/lib/crypt_ops/digestset.c
+++ b/src/lib/crypt_ops/digestset.c
@@ -11,7 +11,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/digestset.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /* Wrap our hash function to have the signature that the bloom filter
  * needs. */
diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h
index 91d53a0542..7d6d687342 100644
--- a/src/lib/crypt_ops/digestset.h
+++ b/src/lib/crypt_ops/digestset.h
@@ -26,4 +26,4 @@ void digestset_add(digestset_t *set, const char *addr);
 int digestset_probably_contains(const digestset_t *set,
                                 const char *addr);
 
-#endif
+#endif /* !defined(TOR_DIGESTSET_H) */
diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am
index 1022096fdc..1f58a33d38 100644
--- a/src/lib/crypt_ops/include.am
+++ b/src/lib/crypt_ops/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_cipher.c		\
 	src/lib/crypt_ops/crypto_curve25519.c		\
@@ -17,6 +18,8 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_ope.c          	\
 	src/lib/crypt_ops/crypto_pwbox.c		\
 	src/lib/crypt_ops/crypto_rand.c			\
+	src/lib/crypt_ops/crypto_rand_fast.c		\
+	src/lib/crypt_ops/crypto_rand_numeric.c		\
 	src/lib/crypt_ops/crypto_rsa.c			\
 	src/lib/crypt_ops/crypto_s2k.c			\
 	src/lib/crypt_ops/crypto_util.c                 \
@@ -25,12 +28,14 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 if USE_NSS
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_nss.c			\
+	src/lib/crypt_ops/crypto_digest_nss.c		\
 	src/lib/crypt_ops/crypto_dh_nss.c		\
 	src/lib/crypt_ops/crypto_nss_mgt.c		\
 	src/lib/crypt_ops/crypto_rsa_nss.c
 else
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_openssl.c			\
+	src/lib/crypt_ops/crypto_digest_openssl.c	\
 	src/lib/crypt_ops/crypto_rsa_openssl.c
 endif
 
@@ -48,6 +53,7 @@ src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_crypt_ops_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/crypt_ops/aes.h				\
 	src/lib/crypt_ops/compat_openssl.h		\
@@ -66,5 +72,6 @@ noinst_HEADERS +=					\
 	src/lib/crypt_ops/crypto_rand.h			\
 	src/lib/crypt_ops/crypto_rsa.h			\
 	src/lib/crypt_ops/crypto_s2k.h			\
+	src/lib/crypt_ops/crypto_sys.h			\
 	src/lib/crypt_ops/crypto_util.h                 \
 	src/lib/crypt_ops/digestset.h
diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am
index b46c43ba0c..83942ca4e0 100644
--- a/src/lib/ctime/include.am
+++ b/src/lib/ctime/include.am
@@ -11,6 +11,7 @@ else
 mulodi4_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_ctime_a_SOURCES =			\
 	$(mulodi4_source) 				\
 	src/ext/csiphash.c   				\
@@ -21,5 +22,6 @@ src_lib_libtor_ctime_testing_a_SOURCES = \
 src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@
 src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/ctime/di_ops.h
diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h
index a2ffbc51c2..b0d1eba0c5 100644
--- a/src/lib/defs/dh_sizes.h
+++ b/src/lib/defs/dh_sizes.h
@@ -19,4 +19,4 @@
 /** Length of our legacy DH keys. */
 #define DH1024_KEY_LEN (1024/8)
 
-#endif
+#endif /* !defined(TOR_DH_SIZES_H) */
diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h
index 525e5209d6..a0dd97a74d 100644
--- a/src/lib/defs/digest_sizes.h
+++ b/src/lib/defs/digest_sizes.h
@@ -24,4 +24,4 @@
 /** Length of the output of our 64-bit optimized message digests (SHA512). */
 #define DIGEST512_LEN 64
 
-#endif
+#endif /* !defined(TOR_DIGEST_SIZES_H) */
diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am
index 48ee7f29fc..84ee403771 100644
--- a/src/lib/defs/include.am
+++ b/src/lib/defs/include.am
@@ -1,5 +1,8 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += 			\
 	src/lib/defs/dh_sizes.h 	\
 	src/lib/defs/digest_sizes.h	\
+	src/lib/defs/logging_types.h	\
+	src/lib/defs/time.h      	\
 	src/lib/defs/x25519_sizes.h
diff --git a/src/lib/defs/logging_types.h b/src/lib/defs/logging_types.h
new file mode 100644
index 0000000000..d3eacde464
--- /dev/null
+++ b/src/lib/defs/logging_types.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file logging_types.h
+ *
+ * \brief Global definition for types used by logging systems.
+ **/
+
+#ifndef TOR_LOGGING_TYPES_H
+#define TOR_LOGGING_TYPES_H
+
+/* We define this here so that it can be used both by backtrace.h and
+ * log.h.
+ */
+
+/** Mask of zero or more log domains, OR'd together. */
+typedef uint64_t log_domain_mask_t;
+
+#endif /* !defined(TOR_LOGGING_TYPES_H) */
diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h
new file mode 100644
index 0000000000..459afbf42d
--- /dev/null
+++ b/src/lib/defs/time.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TIME_DEFS_H
+#define TOR_TIME_DEFS_H
+
+/**
+ * \file time.h
+ *
+ * \brief Definitions for timing-related constants.
+ **/
+
+/** How many microseconds per second */
+#define TOR_USEC_PER_SEC (1000000)
+/** How many nanoseconds per microsecond */
+#define TOR_NSEC_PER_USEC (1000)
+/* How many nanoseconds per millisecond */
+#define TOR_NSEC_PER_MSEC (1000*1000)
+
+#endif /* !defined(TOR_TIME_DEFS_H) */
diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h
index 8933a8866b..6431f0a2dd 100644
--- a/src/lib/defs/x25519_sizes.h
+++ b/src/lib/defs/x25519_sizes.h
@@ -33,4 +33,4 @@
 #define ED25519_BASE64_LEN 43
 #define ED25519_SIG_BASE64_LEN 86
 
-#endif
+#endif /* !defined(TOR_X25519_SIZES_H) */
diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include
new file mode 100644
index 0000000000..884f4c0dbc
--- /dev/null
+++ b/src/lib/dispatch/.may_include
@@ -0,0 +1,11 @@
+orconfig.h
+
+ext/tor_queue.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
\ No newline at end of file
diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h
new file mode 100644
index 0000000000..a9e655409a
--- /dev/null
+++ b/src/lib/dispatch/dispatch.h
@@ -0,0 +1,114 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_H
+#define TOR_DISPATCH_H
+
+#include "lib/dispatch/msgtypes.h"
+
+/**
+ * \file dispatch.h
+ * \brief Low-level APIs for message-passing system.
+ *
+ * This module implements message dispatch based on a set of short integer
+ * identifiers.  For a higher-level interface, see pubsub.h.
+ *
+ * Each message is represented as a generic msg_t object, and is discriminated
+ * by its message_id_t.  Messages are delivered by a dispatch_t object, which
+ * delivers each message to its recipients by a configured "channel".
+ *
+ * A "channel" is a means of delivering messages.  Every message_id_t must
+ * be associated with exactly one channel, identified by channel_id_t.
+ * When a channel receives messages, a callback is invoked to either process
+ * the messages immediately, or to cause them to be processed later.
+ *
+ * Every message_id_t has zero or more associated receiver functions set up in
+ * the dispatch_t object.  Once the dispatch_t object is created, receivers
+ * can be enabled or disabled [TODO], but not added or removed.
+ *
+ * Every message_id_t has an associated datatype, identified by a
+ * msg_type_id_t.  These datatypes can be associated with functions to
+ * (for example) free them, or format them for debugging.
+ *
+ * To setup a dispatch_t object, first create a dispatch_cfg_t object, and
+ * configure messages with their types, channels, and receivers.  Then, use
+ * dispatch_new() with that dispatch_cfg_t to create the dispatch_t object.
+ *
+ * (We use a two-phase contruction procedure here to enable better static
+ * reasoning about publish/subscribe relationships.)
+ *
+ * Once you have a dispatch_t, you can queue messages on it with
+ * dispatch_send*(), and cause those messages to be delivered with
+ * dispatch_flush().
+ **/
+
+/**
+ * A "dispatcher" is the highest-level object; it handles making sure that
+ * messages are received and delivered properly.  Only the mainloop
+ * should handle this type directly.
+ */
+typedef struct dispatch_t dispatch_t;
+
+struct dispatch_cfg_t;
+
+dispatch_t *dispatch_new(const struct dispatch_cfg_t *cfg);
+
+/**
+ * Free a dispatcher.  Tor does this at exit.
+ */
+#define dispatch_free(d) \
+  FREE_AND_NULL(dispatch_t, dispatch_free_, (d))
+
+void dispatch_free_(dispatch_t *);
+
+int dispatch_send(dispatch_t *d,
+                  subsys_id_t sender,
+                  channel_id_t channel,
+                  message_id_t msg,
+                  msg_type_id_t type,
+                  msg_aux_data_t auxdata);
+
+int dispatch_send_msg(dispatch_t *d, msg_t *m);
+
+int dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m);
+
+/* Flush up to <b>max_msgs</b> currently pending messages from the
+ * dispatcher.  Messages that are not pending when this function are
+ * called, are not flushed by this call.  Return 0 on success, -1 on
+ * unrecoverable error.
+ */
+int dispatch_flush(dispatch_t *, channel_id_t chan, int max_msgs);
+
+/**
+ * Function callback type used to alert some other module when a channel's
+ * queue changes from empty to nonempty.
+ *
+ * Ex 1: To cause messages to be processed immediately on-stack, this callback
+ * should invoke dispatch_flush() directly.
+ *
+ * Ex 2: To cause messages to be processed very soon, from the event queue,
+ * this callback should schedule an event callback to run dispatch_flush().
+ *
+ * Ex 3: To cause messages to be processed periodically, this function should
+ * do nothing, and a periodic event should invoke dispatch_flush().
+ **/
+typedef void (*dispatch_alertfn_t)(struct dispatch_t *,
+                                   channel_id_t, void *);
+
+int dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                          dispatch_alertfn_t fn, void *userdata);
+
+#define dispatch_free_msg(d,msg)                                \
+  STMT_BEGIN {                                                  \
+    msg_t **msg_tmp_ptr__ = &(msg);                             \
+    dispatch_free_msg_((d), *msg_tmp_ptr__);                    \
+    *msg_tmp_ptr__= NULL;                                       \
+  } STMT_END
+void dispatch_free_msg_(const dispatch_t *d, msg_t *msg);
+
+char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg);
+
+#endif /* !defined(TOR_DISPATCH_H) */
diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c
new file mode 100644
index 0000000000..b3a72ec22f
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.c
@@ -0,0 +1,141 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_cfg.c
+ * \brief Create and configure a dispatch_cfg_t.
+ *
+ * A dispatch_cfg_t object is used to configure a set of messages and
+ * associated information before creating a dispatch_t.
+ */
+
+#define DISPATCH_PRIVATE
+
+#include "orconfig.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/malloc/malloc.h"
+
+/**
+ * Create and return a new dispatch_cfg_t.
+ **/
+dispatch_cfg_t *
+dcfg_new(void)
+{
+  dispatch_cfg_t *cfg = tor_malloc(sizeof(dispatch_cfg_t));
+  cfg->type_by_msg = smartlist_new();
+  cfg->chan_by_msg = smartlist_new();
+  cfg->fns_by_type = smartlist_new();
+  cfg->recv_by_msg = smartlist_new();
+  return cfg;
+}
+
+/**
+ * Associate a message with a datatype.  Return 0 on success, -1 if a
+ * different type was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                  msg_type_id_t type)
+{
+  smartlist_grow(cfg->type_by_msg, msg+1);
+  msg_type_id_t *oldval = smartlist_get(cfg->type_by_msg, msg);
+  if (oldval != NULL && *oldval != type) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->type_by_msg, msg, tor_memdup(&type, sizeof(type)));
+  return 0;
+}
+
+/**
+ * Associate a message with a channel.  Return 0 on success, -1 if a
+ * different channel was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                  channel_id_t chan)
+{
+  smartlist_grow(cfg->chan_by_msg, msg+1);
+  channel_id_t *oldval = smartlist_get(cfg->chan_by_msg, msg);
+  if (oldval != NULL && *oldval != chan) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->chan_by_msg, msg, tor_memdup(&chan, sizeof(chan)));
+  return 0;
+}
+
+/**
+ * Associate a set of functions with a datatype. Return 0 on success, -1 if
+ * different functions were previously associated with the type.
+ **/
+int
+dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                  const dispatch_typefns_t *fns)
+{
+  smartlist_grow(cfg->fns_by_type, type+1);
+  dispatch_typefns_t *oldfns = smartlist_get(cfg->fns_by_type, type);
+  if (oldfns && (oldfns->free_fn != fns->free_fn ||
+                 oldfns->fmt_fn != fns->fmt_fn))
+    return -1;
+  if (!oldfns)
+    smartlist_set(cfg->fns_by_type, type, tor_memdup(fns, sizeof(*fns)));
+  return 0;
+}
+
+/**
+ * Associate a receiver with a message ID.  Multiple receivers may be
+ * associated with a single messasge ID.
+ *
+ * Return 0 on success, on failure.
+ **/
+int
+dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+              subsys_id_t sys, recv_fn_t fn)
+{
+  smartlist_grow(cfg->recv_by_msg, msg+1);
+  smartlist_t *receivers = smartlist_get(cfg->recv_by_msg, msg);
+  if (!receivers) {
+    receivers = smartlist_new();
+    smartlist_set(cfg->recv_by_msg, msg, receivers);
+  }
+
+  dispatch_rcv_t *rcv = tor_malloc(sizeof(dispatch_rcv_t));
+  rcv->sys = sys;
+  rcv->enabled = true;
+  rcv->fn = fn;
+  smartlist_add(receivers, (void*)rcv);
+  return 0;
+}
+
+/** Helper: release all storage held by <b>cfg</b>. */
+void
+dcfg_free_(dispatch_cfg_t *cfg)
+{
+  if (!cfg)
+    return;
+
+  SMARTLIST_FOREACH(cfg->type_by_msg, msg_type_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->chan_by_msg, channel_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->fns_by_type, dispatch_typefns_t *, f, tor_free(f));
+  smartlist_free(cfg->type_by_msg);
+  smartlist_free(cfg->chan_by_msg);
+  smartlist_free(cfg->fns_by_type);
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, receivers) {
+    if (!receivers)
+      continue;
+    SMARTLIST_FOREACH(receivers, dispatch_rcv_t *, rcv, tor_free(rcv));
+    smartlist_free(receivers);
+  } SMARTLIST_FOREACH_END(receivers);
+  smartlist_free(cfg->recv_by_msg);
+
+  tor_free(cfg);
+}
diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h
new file mode 100644
index 0000000000..348dce8d40
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_CFG_H
+#define TOR_DISPATCH_CFG_H
+
+#include "lib/dispatch/msgtypes.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * A "dispatch_cfg" is the configuration used to set up a dispatcher.
+ * It is created and accessed with a set of dcfg_* functions, and then
+ * used with dispatcher_new() to make the dispatcher.
+ */
+typedef struct dispatch_cfg_t dispatch_cfg_t;
+
+dispatch_cfg_t *dcfg_new(void);
+
+int dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                      msg_type_id_t type);
+
+int dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                      channel_id_t chan);
+
+int dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                      const dispatch_typefns_t *fns);
+
+int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+                  subsys_id_t sys, recv_fn_t fn);
+
+/** Free a dispatch_cfg_t. */
+#define dcfg_free(cfg) \
+  FREE_AND_NULL(dispatch_cfg_t, dcfg_free_, (cfg))
+
+void dcfg_free_(dispatch_cfg_t *cfg);
+
+#ifdef DISPATCH_NEW_PRIVATE
+struct smartlist_t;
+STATIC int max_in_u16_sl(const struct smartlist_t *sl, int dflt);
+#endif
+
+#endif /* !defined(TOR_DISPATCH_CFG_H) */
diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h
new file mode 100644
index 0000000000..57b6f0347f
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg_st.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_CFG_ST_H
+#define TOR_DISPATCH_CFG_ST_H
+
+struct smartlist_t;
+
+/* Information needed to create a dispatcher, but in a less efficient, more
+ * mutable format. */
+struct dispatch_cfg_t {
+  /** A list of msg_type_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *type_by_msg;
+  /** A list of channel_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *chan_by_msg;
+  /** A list of dispatch_rcv_t, indexed by msg_type_id_t. */
+  struct smartlist_t *fns_by_type;
+  /** A list of dispatch_typefns_t, indexed by msg_t. */
+  struct smartlist_t *recv_by_msg;
+};
+
+#endif /* !defined(TOR_DISPATCH_CFG_ST_H) */
diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c
new file mode 100644
index 0000000000..da54f9b437
--- /dev/null
+++ b/src/lib/dispatch/dispatch_core.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_core.c
+ * \brief Core module for sending and receiving messages.
+ */
+
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_naming.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Use <b>d</b> to drop all storage held for <b>msg</b>.
+ *
+ * (We need the dispatcher so we know how to free the auxiliary data.)
+ **/
+void
+dispatch_free_msg_(const dispatch_t *d, msg_t *msg)
+{
+  if (!msg)
+    return;
+
+  d->typefns[msg->type].free_fn(msg->aux_data__);
+  tor_free(msg);
+}
+
+/**
+ * Format the auxiliary data held by msg.
+ **/
+char *
+dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg)
+{
+  if (!msg)
+    return NULL;
+
+  return d->typefns[msg->type].fmt_fn(msg->aux_data__);
+}
+
+/**
+ * Release all storage held by <b>d</b>.
+ **/
+void
+dispatch_free_(dispatch_t *d)
+{
+  if (d == NULL)
+    return;
+
+  size_t n_queues = d->n_queues;
+  for (size_t i = 0; i < n_queues; ++i) {
+    msg_t *m, *mtmp;
+    TOR_SIMPLEQ_FOREACH_SAFE(m, &d->queues[i].queue, next, mtmp) {
+      dispatch_free_msg(d, m);
+    }
+  }
+
+  size_t n_msgs = d->n_msgs;
+
+  for (size_t i = 0; i < n_msgs; ++i) {
+    tor_free(d->table[i]);
+  }
+  tor_free(d->table);
+  tor_free(d->typefns);
+  tor_free(d->queues);
+
+  // This is the only time we will treat d->cfg as non-const.
+  //dispatch_cfg_free_((dispatch_items_t *) d->cfg);
+
+  tor_free(d);
+}
+
+/**
+ * Tell the dispatcher to call <b>fn</b> with <b>userdata</b> whenever
+ * <b>chan</b> becomes nonempty.  Return 0 on success, -1 on error.
+ **/
+int
+dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                      dispatch_alertfn_t fn, void *userdata)
+{
+  if (BUG(chan >= d->n_queues))
+    return -1;
+
+  dqueue_t *q = &d->queues[chan];
+  q->alert_fn = fn;
+  q->alert_fn_arg = userdata;
+  return 0;
+}
+
+/**
+ * Send a message on the appropriate channel notifying that channel if
+ * necessary.
+ *
+ * This function takes ownership of the auxiliary data; it can't be static or
+ * stack-allocated, and the caller is not allowed to use it afterwards.
+ *
+ * This function does not check the various vields of the message object for
+ * consistency.
+ **/
+int
+dispatch_send(dispatch_t *d,
+              subsys_id_t sender,
+              channel_id_t channel,
+              message_id_t msg,
+              msg_type_id_t type,
+              msg_aux_data_t auxdata)
+{
+  if (!d->table[msg]) {
+    /* Fast path: nobody wants this data. */
+
+    d->typefns[type].free_fn(auxdata);
+    return 0;
+  }
+
+  msg_t *m = tor_malloc(sizeof(msg_t));
+
+  m->sender = sender;
+  m->channel = channel;
+  m->msg = msg;
+  m->type = type;
+  memcpy(&m->aux_data__, &auxdata, sizeof(msg_aux_data_t));
+
+  return dispatch_send_msg(d, m);
+}
+
+int
+dispatch_send_msg(dispatch_t *d, msg_t *m)
+{
+  if (BUG(!d))
+    goto err;
+  if (BUG(!m))
+    goto err;
+  if (BUG(m->channel >= d->n_queues))
+    goto err;
+  if (BUG(m->msg >= d->n_msgs))
+    goto err;
+
+  dtbl_entry_t *ent = d->table[m->msg];
+  if (ent) {
+    if (BUG(m->type != ent->type))
+      goto err;
+    if (BUG(m->channel != ent->channel))
+      goto err;
+  }
+
+  return dispatch_send_msg_unchecked(d, m);
+ err:
+  /* Probably it isn't safe to free m, since type could be wrong. */
+  return -1;
+}
+
+/**
+ * Send a message on the appropriate queue, notifying that queue if necessary.
+ *
+ * This function takes ownership of the message object and its auxiliary data;
+ * it can't be static or stack-allocated, and the caller isn't allowed to use
+ * it afterwards.
+ *
+ * This function does not check the various fields of the message object for
+ * consistency, and can crash if they are out of range.  Only functions that
+ * have already constructed the message in a safe way, or checked it for
+ * correctness themselves, should call this function.
+ **/
+int
+dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m)
+{
+  /* Find the right queue. */
+  dqueue_t *q = &d->queues[m->channel];
+  bool was_empty = TOR_SIMPLEQ_EMPTY(&q->queue);
+
+  /* Append the message. */
+  TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next);
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Queued: %s (%s) from %s, on %s.",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  /* If we just made the queue nonempty for the first time, call the alert
+   * function. */
+  if (was_empty) {
+    q->alert_fn(d, m->channel, q->alert_fn_arg);
+  }
+
+  return 0;
+}
+
+/**
+ * Run all of the callbacks on <b>d</b> associated with <b>m</b>.
+ **/
+static void
+dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m)
+{
+  tor_assert(m->msg <= d->n_msgs);
+  dtbl_entry_t *ent = d->table[m->msg];
+  int n_fns = ent->n_fns;
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Delivering: %s (%s) from %s, on %s:",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  int i;
+  for (i=0; i < n_fns; ++i) {
+    if (ent->rcv[i].enabled) {
+      log_debug(LD_MESG, "  Delivering to %s.",
+                get_subsys_id_name(ent->rcv[i].sys));
+      ent->rcv[i].fn(m);
+    }
+  }
+}
+
+/**
+ * Run up to <b>max_msgs</b> callbacks for messages on the channel <b>ch</b>
+ * on the given dispatcher.  Return 0 on success or recoverable failure,
+ * -1 on unrecoverable error.
+ **/
+int
+dispatch_flush(dispatch_t *d, channel_id_t ch, int max_msgs)
+{
+  if (BUG(ch >= d->n_queues))
+    return 0;
+
+  int n_flushed = 0;
+  dqueue_t *q = &d->queues[ch];
+
+  while (n_flushed < max_msgs) {
+    msg_t *m = TOR_SIMPLEQ_FIRST(&q->queue);
+    if (!m)
+      break;
+    TOR_SIMPLEQ_REMOVE_HEAD(&q->queue, next);
+    dispatcher_run_msg_cbs(d, m);
+    dispatch_free_msg(d, m);
+    ++n_flushed;
+  }
+
+  return 0;
+}
diff --git a/src/lib/dispatch/dispatch_naming.c b/src/lib/dispatch/dispatch_naming.c
new file mode 100644
index 0000000000..83d9a2d604
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.c
@@ -0,0 +1,63 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "lib/cc/compat_compiler.h"
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+
+#include "lib/log/util_bug.h"
+#include "lib/log/log.h"
+
+#include <stdlib.h>
+
+/** Global namemap for message IDs. */
+static namemap_t message_id_map = NAMEMAP_INIT();
+/** Global namemap for subsystem IDs. */
+static namemap_t subsys_id_map = NAMEMAP_INIT();
+/** Global namemap for channel IDs. */
+static namemap_t channel_id_map = NAMEMAP_INIT();
+/** Global namemap for message type IDs. */
+static namemap_t msg_type_id_map = NAMEMAP_INIT();
+
+void
+dispatch_naming_init(void)
+{
+}
+
+/* Helper macro: declare functions to map IDs to and from names for a given
+ * type in a namemap_t.
+ */
+#define DECLARE_ID_MAP_FNS(type)                                        \
+  type##_id_t                                                           \
+  get_##type##_id(const char *name)                                     \
+  {                                                                     \
+    unsigned u = namemap_get_or_create_id(&type##_id_map, name);        \
+    tor_assert(u != NAMEMAP_ERR);                                       \
+    tor_assert(u != ERROR_ID);                                          \
+    return (type##_id_t) u;                                             \
+  }                                                                     \
+  const char *                                                          \
+  get_##type##_id_name(type##_id_t id)                                  \
+  {                                                                     \
+    return namemap_fmt_name(&type##_id_map, id);                        \
+  }                                                                     \
+  size_t                                                                \
+  get_num_##type##_ids(void)                                            \
+  {                                                                     \
+    return namemap_get_size(&type##_id_map);                            \
+  }                                                                     \
+  EAT_SEMICOLON
+
+DECLARE_ID_MAP_FNS(message);
+DECLARE_ID_MAP_FNS(channel);
+DECLARE_ID_MAP_FNS(subsys);
+DECLARE_ID_MAP_FNS(msg_type);
diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h
new file mode 100644
index 0000000000..fd6c83cc12
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.h
@@ -0,0 +1,46 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_NAMING_H
+#define TOR_DISPATCH_NAMING_H
+
+#include "lib/dispatch/msgtypes.h"
+#include <stddef.h>
+
+/**
+ * Return an existing channel ID by name, allocating the channel ID if
+ * if necessary.  Returns ERROR_ID if we have run out of
+ * channels
+ */
+channel_id_t get_channel_id(const char *);
+/**
+ * Return the name corresponding to a given channel ID.
+ **/
+const char *get_channel_id_name(channel_id_t);
+/**
+ * Return the total number of _named_ channel IDs.
+ **/
+size_t get_num_channel_ids(void);
+
+/* As above, but for messages. */
+message_id_t get_message_id(const char *);
+const char *get_message_id_name(message_id_t);
+size_t get_num_message_ids(void);
+
+/* As above, but for subsystems */
+subsys_id_t get_subsys_id(const char *);
+const char *get_subsys_id_name(subsys_id_t);
+size_t get_num_subsys_ids(void);
+
+/* As above, but for types. Note that types additionally must be
+ * "defined", if any message is to use them. */
+msg_type_id_t get_msg_type_id(const char *);
+const char *get_msg_type_id_name(msg_type_id_t);
+size_t get_num_msg_type_ids(void);
+
+void dispatch_naming_init(void);
+
+#endif /* !defined(TOR_DISPATCH_NAMING_H) */
diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c
new file mode 100644
index 0000000000..d8e59d610a
--- /dev/null
+++ b/src/lib/dispatch/dispatch_new.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_new.c
+ * \brief Code to construct a dispatch_t from a dispatch_cfg_t.
+ **/
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+
+#include "lib/cc/ctassert.h"
+#include "lib/intmath/cmp.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/** Given a smartlist full of (possibly NULL) pointers to uint16_t values,
+ * return the largest value, or dflt if the list is empty. */
+STATIC int
+max_in_u16_sl(const smartlist_t *sl, int dflt)
+{
+  uint16_t *maxptr = NULL;
+  SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) {
+    if (!maxptr)
+      maxptr = u;
+    else if (u && *u > *maxptr)
+      maxptr = u;
+  } SMARTLIST_FOREACH_END(u);
+
+  return maxptr ? *maxptr : dflt;
+}
+
+/* The above function is only safe to call if we are sure that channel_id_t
+ * and msg_type_id_t are really uint16_t.  They should be so defined in
+ * msgtypes.h, but let's be extra cautious.
+ */
+CTASSERT(sizeof(uint16_t) == sizeof(msg_type_id_t));
+CTASSERT(sizeof(uint16_t) == sizeof(channel_id_t));
+
+/** Helper: Format an unformattable message auxiliary data item: just return a
+* copy of the string <>. */
+static char *
+type_fmt_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+  return tor_strdup("<>");
+}
+
+/** Helper: Free an unfreeable message auxiliary data item: do nothing. */
+static void
+type_free_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+}
+
+/** Type functions to use when no type functions are provided. */
+static dispatch_typefns_t nop_typefns = {
+  .free_fn = type_free_nop,
+  .fmt_fn = type_fmt_nop
+};
+
+/**
+ * Alert function to use when none is configured: do nothing.
+ **/
+static void
+alert_fn_nop(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)d;
+  (void)ch;
+  (void)arg;
+}
+
+/**
+ * Given a list of recvfn_t, create and return a new dtbl_entry_t mapping
+ * to each of those functions.
+ **/
+static dtbl_entry_t *
+dtbl_entry_from_lst(smartlist_t *receivers)
+{
+  if (!receivers)
+    return NULL;
+
+  size_t n_recv = smartlist_len(receivers);
+  dtbl_entry_t *ent;
+  ent = tor_malloc_zero(offsetof(dtbl_entry_t, rcv) +
+                        sizeof(dispatch_rcv_t) * n_recv);
+
+  ent->n_fns = n_recv;
+
+  SMARTLIST_FOREACH_BEGIN(receivers, const dispatch_rcv_t *, rcv) {
+    memcpy(&ent->rcv[rcv_sl_idx], rcv, sizeof(*rcv));
+    if (rcv->enabled) {
+      ++ent->n_enabled;
+    }
+  } SMARTLIST_FOREACH_END(rcv);
+
+  return ent;
+}
+
+/** Create and return a new dispatcher from a given dispatch_cfg_t. */
+dispatch_t *
+dispatch_new(const dispatch_cfg_t *cfg)
+{
+  dispatch_t *d = tor_malloc_zero(sizeof(dispatch_t));
+
+  /* Any message that has a type or a receiver counts towards our messages */
+  const size_t n_msgs = MAX(smartlist_len(cfg->type_by_msg),
+                            smartlist_len(cfg->recv_by_msg)) + 1;
+
+  /* Any channel that any message has counts towards the number of channels. */
+  const size_t n_chans = (size_t)
+    MAX(1, max_in_u16_sl(cfg->chan_by_msg,0)) + 1;
+
+  /* Any type that a message has, or that has functions, counts towards
+   * the number of types. */
+  const size_t n_types = (size_t) MAX(max_in_u16_sl(cfg->type_by_msg,0),
+                                      smartlist_len(cfg->fns_by_type)) + 1;
+
+  d->n_msgs = n_msgs;
+  d->n_queues = n_chans;
+  d->n_types = n_types;
+
+  /* Initialize the array of type-functions. */
+  d->typefns = tor_calloc(n_types, sizeof(dispatch_typefns_t));
+  for (size_t i = 0; i < n_types; ++i) {
+    /* Default to no-op for everything... */
+    memcpy(&d->typefns[i], &nop_typefns, sizeof(dispatch_typefns_t));
+  }
+  SMARTLIST_FOREACH_BEGIN(cfg->fns_by_type, dispatch_typefns_t *, fns) {
+    /* Set the functions if they are provided. */
+    if (fns) {
+      if (fns->free_fn)
+        d->typefns[fns_sl_idx].free_fn = fns->free_fn;
+      if (fns->fmt_fn)
+        d->typefns[fns_sl_idx].fmt_fn = fns->fmt_fn;
+    }
+  } SMARTLIST_FOREACH_END(fns);
+
+  /* Initialize the message queues: one for each channel. */
+  d->queues = tor_calloc(d->n_queues, sizeof(dqueue_t));
+  for (size_t i = 0; i < d->n_queues; ++i) {
+    TOR_SIMPLEQ_INIT(&d->queues[i].queue);
+    d->queues[i].alert_fn = alert_fn_nop;
+  }
+
+  /* Build the dispatch tables mapping message IDs to receivers. */
+  d->table = tor_calloc(d->n_msgs, sizeof(dtbl_entry_t *));
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, rcv) {
+    d->table[rcv_sl_idx] = dtbl_entry_from_lst(rcv);
+  } SMARTLIST_FOREACH_END(rcv);
+
+  /* Fill in the empty entries in the dispatch tables:
+   * types and channels for each message. */
+  SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, msg_type_id_t *, type) {
+    if (d->table[type_sl_idx])
+      d->table[type_sl_idx]->type = *type;
+  } SMARTLIST_FOREACH_END(type);
+
+  SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, channel_id_t *, chan) {
+    if (d->table[chan_sl_idx])
+      d->table[chan_sl_idx]->channel = *chan;
+  } SMARTLIST_FOREACH_END(chan);
+
+  return d;
+}
diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h
new file mode 100644
index 0000000000..ee42518b5a
--- /dev/null
+++ b/src/lib/dispatch/dispatch_st.h
@@ -0,0 +1,108 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_st.h
+ *
+ * \brief private structures used for the dispatcher module
+ */
+
+#ifndef TOR_DISPATCH_ST_H
+#define TOR_DISPATCH_ST_H
+
+#ifdef DISPATCH_PRIVATE
+
+#include "lib/container/smartlist.h"
+
+/**
+ * Information about the recipient of a message.
+ **/
+typedef struct dispatch_rcv_t {
+  /** The subsystem receiving a message. */
+  subsys_id_t sys;
+  /** True iff this recipient is enabled. */
+  bool enabled;
+  /** The function that will handle the message. */
+  recv_fn_t fn;
+} dispatch_rcv_t;
+
+/**
+ * Information used by a dispatcher to handle and dispatch a single message
+ * ID.  It maps that message ID to its type, channel, and list of receiver
+ * functions.
+ *
+ * This structure is used when the dispatcher is running.
+ **/
+typedef struct dtbl_entry_t {
+  /** The number of enabled non-stub subscribers for this message.
+   *
+   * Note that for now, this will be the same as <b>n_fns</b>, since there is
+   * no way to turn these subscribers on an off yet. */
+  uint16_t n_enabled;
+  /** The channel that handles this message. */
+  channel_id_t channel;
+  /** The associated C type for this message. */
+  msg_type_id_t type;
+  /**
+   * The number of functions pointers for subscribers that receive this
+   * message, in rcv. */
+  uint16_t n_fns;
+  /**
+   * The recipients for this message.
+   */
+  dispatch_rcv_t rcv[FLEXIBLE_ARRAY_MEMBER];
+} dtbl_entry_t;
+
+/**
+ * A queue of messages for a given channel, used by a live dispatcher.
+ */
+typedef struct dqueue_t {
+  /** The queue of messages itself. */
+  TOR_SIMPLEQ_HEAD( , msg_t) queue;
+  /** A function to be called when the queue becomes nonempty. */
+  dispatch_alertfn_t alert_fn;
+  /** An argument for the alert_fn. */
+  void *alert_fn_arg;
+} dqueue_t ;
+
+/**
+ * A single dispatcher for cross-module messages.
+ */
+struct dispatch_t {
+  /**
+   * The length of <b>table</b>: the number of message IDs that this
+   * dispatcher can handle.
+   */
+  size_t n_msgs;
+  /**
+   * The length of <b>queues</b>: the number of channels that this dispatcher
+   * has configured.
+   */
+  size_t n_queues;
+  /**
+   * The length of <b>typefns</b>: the number of C type IDs that this
+   * dispatcher has configured.
+   */
+  size_t n_types;
+  /**
+   * An array of message queues, indexed by channel ID.
+   */
+  dqueue_t *queues;
+  /**
+   * An array of entries about how to handle particular message types, indexed
+   * by message ID.
+   */
+  dtbl_entry_t **table;
+  /**
+   * An array of function tables for manipulating types, index by message
+   * type ID.
+   **/
+  dispatch_typefns_t *typefns;
+};
+
+#endif /* defined(DISPATCH_PRIVATE) */
+
+#endif /* !defined(TOR_DISPATCH_ST_H) */
diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am
new file mode 100644
index 0000000000..4a0e0dfd90
--- /dev/null
+++ b/src/lib/dispatch/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-dispatch.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_dispatch_a_SOURCES =			\
+	src/lib/dispatch/dispatch_cfg.c			\
+	src/lib/dispatch/dispatch_core.c		\
+	src/lib/dispatch/dispatch_naming.c		\
+	src/lib/dispatch/dispatch_new.c
+
+src_lib_libtor_dispatch_testing_a_SOURCES = \
+	$(src_lib_libtor_dispatch_a_SOURCES)
+src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/dispatch/dispatch.h			\
+	src/lib/dispatch/dispatch_cfg.h 		\
+	src/lib/dispatch/dispatch_cfg_st.h 		\
+	src/lib/dispatch/dispatch_naming.h		\
+	src/lib/dispatch/dispatch_st.h			\
+	src/lib/dispatch/msgtypes.h
diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h
new file mode 100644
index 0000000000..b4c4a10248
--- /dev/null
+++ b/src/lib/dispatch/msgtypes.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file msgtypes.h
+ * \brief Types used for messages in the dispatcher code.
+ **/
+
+#ifndef TOR_DISPATCH_MSGTYPES_H
+#define TOR_DISPATCH_MSGTYPES_H
+
+#include <stdint.h>
+
+#include "ext/tor_queue.h"
+
+/**
+ * These types are aliases for subsystems, channels, and message IDs.
+ **/
+typedef uint16_t subsys_id_t;
+typedef uint16_t channel_id_t;
+typedef uint16_t message_id_t;
+
+/**
+ * This identifies a C type that can be sent along with a message.
+ **/
+typedef uint16_t msg_type_id_t;
+
+/**
+ * An ID value returned for *_type_t when none exists.
+ */
+#define ERROR_ID 65535
+
+/**
+ * Auxiliary (untyped) data sent along with a message.
+ *
+ * We define this as a union of a pointer and a u64, so that the integer
+ * types will have the same range across platforms.
+ **/
+typedef union {
+  void *ptr;
+  uint64_t u64;
+} msg_aux_data_t;
+
+/**
+ * Structure of a received message.
+ **/
+typedef struct msg_t {
+  TOR_SIMPLEQ_ENTRY(msg_t) next;
+  subsys_id_t sender;
+  channel_id_t channel;
+  message_id_t msg;
+  /** We could omit this field, since it is implicit in the message type, but
+   * IMO let's leave it in for safety. */
+  msg_type_id_t type;
+  /** Untyped auxiliary data. You shouldn't have to mess with this
+   * directly. */
+  msg_aux_data_t aux_data__;
+} msg_t;
+
+/**
+ * A function that a subscriber uses to receive a message.
+ **/
+typedef void (*recv_fn_t)(const msg_t *m);
+
+/**
+ * Table of functions to use for a given C type.  Any omitted (NULL) functions
+ * will be treated as no-ops.
+ **/
+typedef struct dispatch_typefns_t {
+  /** Release storage held for the auxiliary data of this type. */
+  void (*free_fn)(msg_aux_data_t);
+  /** Format and return a newly allocated string describing the contents
+   * of this data element. */
+  char *(*fmt_fn)(msg_aux_data_t);
+} dispatch_typefns_t;
+
+#endif /* !defined(TOR_DISPATCH_MSGTYPES_H) */
diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include
index 7c2ef36929..c9bf4b1786 100644
--- a/src/lib/encoding/.may_include
+++ b/src/lib/encoding/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/container/*.h
 lib/ctime/*.h
 lib/encoding/*.h
 lib/intmath/*.h
diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c
index bd063440d6..fc64e014e7 100644
--- a/src/lib/encoding/binascii.c
+++ b/src/lib/encoding/binascii.c
@@ -84,7 +84,7 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
 }
 
 /** Implements base32 decoding as in RFC 4648.
- * Returns 0 if successful, -1 otherwise.
+ * Return the number of bytes decoded if successful; -1 otherwise.
  */
 int
 base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
@@ -147,7 +147,7 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
   memset(tmp, 0, srclen); /* on the heap, this should be safe */
   tor_free(tmp);
   tmp = NULL;
-  return 0;
+  return i;
 }
 
 #define BASE64_OPENSSL_LINELEN 64
@@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags)
   return enclen;
 }
 
+/** Return an upper bound on the number of bytes that might be needed to hold
+ * the data from decoding the base64 string <b>srclen</b>.  This is only an
+ * upper bound, since some part of the base64 string might be padding or
+ * space. */
+size_t
+base64_decode_maxsize(size_t srclen)
+{
+  tor_assert(srclen < INT_MAX / 3);
+
+  return CEIL_DIV(srclen * 3, 4);
+}
+
 /** Internal table mapping 6 bit values to the Base64 alphabet. */
 static const char base64_encode_table[64] = {
   'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
@@ -309,8 +321,10 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
   return (int) enclen;
 }
 
-/** As base64_encode, but do not add any internal spaces or external padding
- * to the output stream. */
+/** As base64_encode, but do not add any internal spaces, and remove external
+ * padding from the output stream.
+ * dest must be at least base64_encode_size(srclen, 0), including space for
+ * the removed external padding. */
 int
 base64_encode_nopad(char *dest, size_t destlen,
                     const uint8_t *src, size_t srclen)
diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h
index 7e3cc04f09..40c5593b11 100644
--- a/src/lib/encoding/binascii.h
+++ b/src/lib/encoding/binascii.h
@@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen);
 
 #define BASE64_ENCODE_MULTILINE 1
 size_t base64_encode_size(size_t srclen, int flags);
+size_t base64_decode_maxsize(size_t srclen);
 int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
                   int flags);
 int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
@@ -57,4 +58,4 @@ size_t base32_encoded_size(size_t srclen);
 void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 
-#endif /* !defined(TOR_UTIL_FORMAT_H) */
+#endif /* !defined(TOR_BINASCII_H) */
diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c
index 8110f3dd9c..0d8384db13 100644
--- a/src/lib/encoding/confline.c
+++ b/src/lib/encoding/confline.c
@@ -82,6 +82,19 @@ config_line_find(const config_line_t *lines,
   return NULL;
 }
 
+/** As config_line_find(), but perform a case-insensitive comparison. */
+const config_line_t *
+config_line_find_case(const config_line_t *lines,
+                      const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcasecmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
 /** Auxiliary function that does all the work of config_get_lines.
  * <b>recursion_level</b> is the count of how many nested %includes we have.
  * <b>opened_lst</b> will have a list of opened files if provided.
@@ -243,7 +256,7 @@ config_lines_dup_and_filter(const config_line_t *inp,
 /** Return true iff a and b contain identical keys and values in identical
  * order. */
 int
-config_lines_eq(config_line_t *a, config_line_t *b)
+config_lines_eq(const config_line_t *a, const config_line_t *b)
 {
   while (a && b) {
     if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h
index 3d9ae8a662..12c554c6e7 100644
--- a/src/lib/encoding/confline.h
+++ b/src/lib/encoding/confline.h
@@ -48,7 +48,9 @@ config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
                                            const char *key);
 const config_line_t *config_line_find(const config_line_t *lines,
                                       const char *key);
-int config_lines_eq(config_line_t *a, config_line_t *b);
+const config_line_t *config_line_find_case(const config_line_t *lines,
+                                           const char *key);
+int config_lines_eq(const config_line_t *a, const config_line_t *b);
 int config_count_key(const config_line_t *a, const char *key);
 void config_free_lines_(config_line_t *front);
 #define config_free_lines(front) \
diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am
index 2d2aa3988a..48d0120bfc 100644
--- a/src/lib/encoding/include.am
+++ b/src/lib/encoding/include.am
@@ -4,12 +4,15 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-encoding-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_encoding_a_SOURCES =			\
 	src/lib/encoding/binascii.c			\
 	src/lib/encoding/confline.c			\
 	src/lib/encoding/cstring.c			\
 	src/lib/encoding/keyval.c			\
+	src/lib/encoding/kvline.c			\
 	src/lib/encoding/pem.c				\
+	src/lib/encoding/qstring.c			\
 	src/lib/encoding/time_fmt.c
 
 src_lib_libtor_encoding_testing_a_SOURCES = \
@@ -17,10 +20,13 @@ src_lib_libtor_encoding_testing_a_SOURCES = \
 src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/encoding/binascii.h			\
 	src/lib/encoding/confline.h			\
 	src/lib/encoding/cstring.h			\
 	src/lib/encoding/keyval.h			\
+	src/lib/encoding/kvline.h			\
 	src/lib/encoding/pem.h				\
+	src/lib/encoding/qstring.h			\
 	src/lib/encoding/time_fmt.h
diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h
index cd327b7a82..dcddfa3396 100644
--- a/src/lib/encoding/keyval.h
+++ b/src/lib/encoding/keyval.h
@@ -14,4 +14,4 @@
 
 int string_is_key_value(int severity, const char *string);
 
-#endif
+#endif /* !defined(TOR_KEYVAL_H) */
diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c
new file mode 100644
index 0000000000..d4a8f510ba
--- /dev/null
+++ b/src/lib/encoding/kvline.c
@@ -0,0 +1,289 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.c
+ *
+ * \brief Manipulating lines of key-value pairs.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
+
+/** Return true iff we need to quote and escape the string <b>s</b> to encode
+ * it. */
+static bool
+needs_escape(const char *s, bool as_keyless_val)
+{
+  if (as_keyless_val && *s == 0)
+    return true;
+
+  for (; *s; ++s) {
+    if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) ||
+        *s == '\'' || *s == '\"') {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff the key in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_key(const config_line_t *line)
+{
+  return line->key == NULL || strlen(line->key) == 0;
+}
+
+/**
+ * Return true iff the value in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_val(const config_line_t *line)
+{
+  return line->value == NULL || strlen(line->value) == 0;
+}
+
+/**
+ * Return true iff the all the lines in <b>line</b> can be encoded
+ * using <b>flags</b>.
+ **/
+static bool
+kvline_can_encode_lines(const config_line_t *line, unsigned flags)
+{
+  for ( ; line; line = line->next) {
+    const bool keyless = line_has_no_key(line);
+    if (keyless) {
+      if (! (flags & KV_OMIT_KEYS)) {
+        /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */
+        return false;
+      }
+      if (strchr(line->value, '=') && !( flags & KV_QUOTED)) {
+        /* We can't have a keyless value with = without quoting it. */
+        return false;
+      }
+    }
+
+    if (needs_escape(line->value, keyless) && ! (flags & KV_QUOTED)) {
+      /* If KV_QUOTED is false, we can't encode a value that needs quotes. */
+      return false;
+    }
+    if (line->key && strlen(line->key) &&
+        (needs_escape(line->key, false) || strchr(line->key, '='))) {
+      /* We can't handle keys that need quoting. */
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value'
+ * pairs, using the provided <b>flags</b> to encode it.  Return a newly
+ * allocated string on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then all values that contain
+ * spaces or unusual characters are escaped and quoted.  Otherwise, such
+ * values are not allowed.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are
+ * allowed, and are encoded as 'Value'.  Otherwise, such pairs are not
+ * allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then an empty value is
+ * encoded as 'Key', not as 'Key=' or 'Key=""'.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * KV_QUOTED_QSTRING is not supported.
+ */
+char *
+kvline_encode(const config_line_t *line,
+              unsigned flags)
+{
+  tor_assert(! (flags & KV_QUOTED_QSTRING));
+
+  if (!kvline_can_encode_lines(line, flags))
+    return NULL;
+
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+
+  smartlist_t *elements = smartlist_new();
+
+  for (; line; line = line->next) {
+
+    const char *k = "";
+    const char *eq = "=";
+    const char *v = "";
+    const bool keyless = line_has_no_key(line);
+    bool esc = needs_escape(line->value, keyless);
+    char *tmp = NULL;
+
+    if (! keyless) {
+      k = line->key;
+    } else {
+      eq = "";
+      if (strchr(line->value, '=')) {
+        esc = true;
+      }
+    }
+
+    if ((flags & KV_OMIT_VALS) && line_has_no_val(line)) {
+      eq = "";
+      v = "";
+    } else if (esc) {
+      tmp = esc_for_log(line->value);
+      v = tmp;
+    } else {
+      v = line->value;
+    }
+
+    smartlist_add_asprintf(elements, "%s%s%s", k, eq, v);
+    tor_free(tmp);
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/**
+ * Decode a <b>line</b> containing a series of space-separated 'Key=Value'
+ * pairs, using the provided <b>flags</b> to decode it.  Return a newly
+ * allocated list of pairs on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are
+ * allowed and handled as C strings. Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are
+ * allowed.  Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then keys without values are
+ * allowed.  Otherwise, such keys are not allowed.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * If KV_QUOTED_QSTRING is set in <b>flags</b>, then double-quoted values
+ * are allowed and handled as QuotedStrings per qstring.c.  Do not add
+ * new users of this flag.
+ */
+config_line_t *
+kvline_parse(const char *line, unsigned flags)
+{
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+
+  const char *cp = line, *cplast = NULL;
+  const bool omit_keys = (flags & KV_OMIT_KEYS) != 0;
+  const bool omit_vals = (flags & KV_OMIT_VALS) != 0;
+  const bool quoted = (flags & (KV_QUOTED|KV_QUOTED_QSTRING)) != 0;
+  const bool c_quoted = (flags & (KV_QUOTED)) != 0;
+
+  config_line_t *result = NULL;
+  config_line_t **next_line = &result;
+
+  char *key = NULL;
+  char *val = NULL;
+
+  while (*cp) {
+    key = val = NULL;
+    /* skip all spaces */
+    {
+      size_t idx = strspn(cp, " \t\r\v\n");
+      cp += idx;
+    }
+    if (BUG(cp == cplast)) {
+      /* If we didn't parse anything since the last loop, this code is
+       * broken. */
+      goto err; // LCOV_EXCL_LINE
+    }
+    cplast = cp;
+    if (! *cp)
+      break; /* End of string; we're done. */
+
+    /* Possible formats are K=V, K="V", K, V, and "V", depending on flags. */
+
+    /* Find where the key ends */
+    if (*cp != '\"') {
+      size_t idx = strcspn(cp, " \t\r\v\n=");
+
+      if (cp[idx] == '=') {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx + 1;
+      } else if (omit_vals) {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx;
+        goto commit;
+      } else {
+        if (!omit_keys)
+          goto err;
+      }
+    }
+
+    if (*cp == '\"') {
+      /* The type is "V". */
+      if (!quoted)
+        goto err;
+      size_t len=0;
+      if (c_quoted) {
+        cp = unescape_string(cp, &val, &len);
+      } else {
+        cp = decode_qstring(cp, strlen(cp), &val, &len);
+      }
+      if (cp == NULL || len != strlen(val)) {
+        // The string contains a NUL or is badly coded.
+        goto err;
+      }
+    } else {
+      size_t idx = strcspn(cp, " \t\r\v\n");
+      val = tor_memdup_nulterm(cp, idx);
+      cp += idx;
+    }
+
+  commit:
+    if (key && strlen(key) == 0) {
+      /* We don't allow empty keys. */
+      goto err;
+    }
+
+    *next_line = tor_malloc_zero(sizeof(config_line_t));
+    (*next_line)->key = key ? key : tor_strdup("");
+    (*next_line)->value = val ? val : tor_strdup("");
+    next_line = &(*next_line)->next;
+    key = val = NULL;
+  }
+
+  if (! (flags & KV_QUOTED_QSTRING)) {
+    if (!kvline_can_encode_lines(result, flags)) {
+      goto err;
+    }
+  }
+  return result;
+
+ err:
+  tor_free(key);
+  tor_free(val);
+  config_free_lines(result);
+  return NULL;
+}
diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h
new file mode 100644
index 0000000000..dea2ce1809
--- /dev/null
+++ b/src/lib/encoding/kvline.h
@@ -0,0 +1,26 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.h
+ *
+ * \brief Header for kvline.c
+ **/
+
+#ifndef TOR_KVLINE_H
+#define TOR_KVLINE_H
+
+struct config_line_t;
+
+#define KV_QUOTED    (1u<<0)
+#define KV_OMIT_KEYS (1u<<1)
+#define KV_OMIT_VALS (1u<<2)
+#define KV_QUOTED_QSTRING (1u<<3)
+
+struct config_line_t *kvline_parse(const char *line, unsigned flags);
+char *kvline_encode(const struct config_line_t *line, unsigned flags);
+
+#endif /* !defined(TOR_KVLINE_H) */
diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h
index 0bbb06a794..6b20350aa8 100644
--- a/src/lib/encoding/pem.h
+++ b/src/lib/encoding/pem.h
@@ -23,4 +23,4 @@ int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen,
 int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen,
                const char *objtype);
 
-#endif
+#endif /* !defined(TOR_PEM_H) */
diff --git a/src/lib/encoding/qstring.c b/src/lib/encoding/qstring.c
new file mode 100644
index 0000000000..a92d28c706
--- /dev/null
+++ b/src/lib/encoding/qstring.c
@@ -0,0 +1,90 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.c
+ * \brief Implement QuotedString parsing.
+ *
+ * Note that this is only used for controller authentication; do not
+ * create new users for this.  Instead, prefer the cstring.c functions.
+ **/
+
+#include "orconfig.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
+ * QuotedString, return the length of that
+ * string (as encoded, including quotes).  Otherwise return -1. */
+static inline int
+get_qstring_length(const char *start, size_t in_len_max,
+                          int *chars_out)
+{
+  const char *cp, *end;
+  int chars = 0;
+
+  if (*start != '\"')
+    return -1;
+
+  cp = start+1;
+  end = start+in_len_max;
+
+  /* Calculate length. */
+  while (1) {
+    if (cp >= end) {
+      return -1; /* Too long. */
+    } else if (*cp == '\\') {
+      if (++cp == end)
+        return -1; /* Can't escape EOS. */
+      ++cp;
+      ++chars;
+    } else if (*cp == '\"') {
+      break;
+    } else {
+      ++cp;
+      ++chars;
+    }
+  }
+  if (chars_out)
+    *chars_out = chars;
+  return (int)(cp - start+1);
+}
+
+/** Given a pointer to a string starting at <b>start</b> containing
+ * <b>in_len_max</b> characters, decode a string beginning with one double
+ * quote, containing any number of non-quote characters or characters escaped
+ * with a backslash, and ending with a final double quote.  Place the resulting
+ * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
+ * store its length in <b>out_len</b>.  On success, return a pointer to the
+ * character immediately following the escaped string.  On failure, return
+ * NULL. */
+const char *
+decode_qstring(const char *start, size_t in_len_max,
+               char **out, size_t *out_len)
+{
+  const char *cp, *end;
+  char *outp;
+  int len, n_chars = 0;
+
+  len = get_qstring_length(start, in_len_max, &n_chars);
+  if (len<0)
+    return NULL;
+
+  end = start+len-1; /* Index of last quote. */
+  tor_assert(*end == '\"');
+  outp = *out = tor_malloc(len+1);
+  *out_len = n_chars;
+
+  cp = start+1;
+  while (cp < end) {
+    if (*cp == '\\')
+      ++cp;
+    *outp++ = *cp++;
+  }
+  *outp = '\0';
+  tor_assert((outp - *out) == (int)*out_len);
+
+  return end+1;
+}
diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h
new file mode 100644
index 0000000000..840e1044ce
--- /dev/null
+++ b/src/lib/encoding/qstring.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.h
+ * \brief Header for qstring.c
+ */
+
+#ifndef TOR_ENCODING_QSTRING_H
+#define TOR_ENCODING_QSTRING_H
+
+#include <stddef.h>
+
+const char *decode_qstring(const char *start, size_t in_len_max,
+                           char **out, size_t *out_len);
+
+#endif /* !defined(TOR_ENCODING_QSTRING_H) */
diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h
index 0ddeca57fc..d14bc1f902 100644
--- a/src/lib/encoding/time_fmt.h
+++ b/src/lib/encoding/time_fmt.h
@@ -41,4 +41,4 @@ int parse_iso_time_nospace(const char *cp, time_t *t);
 int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
-#endif
+#endif /* !defined(TOR_TIME_FMT_H) */
diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include
index 48cc0ef088..314424545e 100644
--- a/src/lib/err/.may_include
+++ b/src/lib/err/.may_include
@@ -1,3 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/defs/*.h
 lib/err/*.h
+lib/subsys/*.h
+lib/version/*.h
diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c
index 8606f42177..ce8ddcd7c0 100644
--- a/src/lib/err/backtrace.c
+++ b/src/lib/err/backtrace.c
@@ -52,12 +52,14 @@
 #include <pthread.h>
 #endif
 
+#include "lib/cc/ctassert.h"
+
 #define EXPOSE_CLEAN_BACKTRACE
 #include "lib/err/backtrace.h"
-#include "lib/err/torerr.h"
 
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
-  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
+  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \
+  defined(HAVE_PTHREAD_H)
 #define USE_BACKTRACE
 #endif
 
@@ -72,15 +74,40 @@
 static char bt_version[128] = "";
 
 #ifdef USE_BACKTRACE
+
 /** Largest stack depth to try to dump. */
 #define MAX_DEPTH 256
-/** Static allocation of stack to dump. This is static so we avoid stack
- * pressure. */
-static void *cb_buf[MAX_DEPTH];
+/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */
+#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *))
 /** Protects cb_buf from concurrent access. Pthreads, since this code
  * is Unix-only, and since this code needs to be lowest-level. */
 static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+/** Lock and return a static stack pointer buffer that can hold up to
+ *  MAX_DEPTH function pointers. */
+static void **
+lock_cb_buf(void)
+{
+  /* Lock the mutex first, before even declaring the buffer. */
+  pthread_mutex_lock(&cb_buf_mutex);
+
+  /** Static allocation of stack to dump. This is static so we avoid stack
+   * pressure. */
+  static void *cb_buf[MAX_DEPTH];
+  CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf));
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+
+  return cb_buf;
+}
+
+/** Unlock the static stack pointer buffer. */
+static void
+unlock_cb_buf(void **cb_buf)
+{
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+  pthread_mutex_unlock(&cb_buf_mutex);
+}
+
 /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will
  * log the correct function from which a signal was received with context
  * <b>ctx</b>.  (When we get a signal, the current function will not have
@@ -104,7 +131,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
     return;
 
   stack[n] = (void*) ctx->PC_FROM_UCONTEXT;
-#else /* !(defined(PC_FROM_UCONTEXT)) */
+#else /* !defined(PC_FROM_UCONTEXT) */
   (void) depth;
   (void) ctx;
   (void) stack;
@@ -115,14 +142,14 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
  * that with a backtrace log.  Send messages via the tor_log function at
  * logger". */
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   size_t depth;
   char **symbols;
   size_t i;
 
-  pthread_mutex_lock(&cb_buf_mutex);
+  void **cb_buf = lock_cb_buf();
 
   depth = backtrace(cb_buf, MAX_DEPTH);
   symbols = backtrace_symbols(cb_buf, (int)depth);
@@ -140,7 +167,7 @@ log_backtrace_impl(int severity, int domain, const char *msg,
   raw_free(symbols);
 
  done:
-  pthread_mutex_unlock(&cb_buf_mutex);
+  unlock_cb_buf(cb_buf);
 }
 
 static void crash_handler(int sig, siginfo_t *si, void *ctx_)
@@ -156,6 +183,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   int n_fds, i;
   const int *fds = NULL;
 
+  void **cb_buf = lock_cb_buf();
+
   (void) si;
 
   depth = backtrace(cb_buf, MAX_DEPTH);
@@ -172,7 +201,9 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
 
-  abort();
+  unlock_cb_buf(cb_buf);
+
+  tor_raw_abort_();
 }
 
 /** Write a backtrace to all of the emergency-error fds. */
@@ -183,20 +214,26 @@ dump_stack_symbols_to_error_fds(void)
   const int *fds = NULL;
   size_t depth;
 
+  void **cb_buf = lock_cb_buf();
+
   depth = backtrace(cb_buf, MAX_DEPTH);
 
   n_fds = tor_log_get_sigsafe_err_fds(&fds);
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
+
+  unlock_cb_buf(cb_buf);
 }
 
+/* The signals that we want our backtrace handler to trap */
+static int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
+  SIGIO, -1 };
+
 /** Install signal handlers as needed so that when we crash, we produce a
  * useful stack trace. Return 0 on success, -errno on failure. */
 static int
 install_bt_handler(void)
 {
-  int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
-                         SIGIO, -1 };
   int i, rv=0;
 
   struct sigaction sa;
@@ -219,10 +256,12 @@ install_bt_handler(void)
      * libc has pre-loaded the symbols we need to dump things, so that later
      * reads won't be denied by the sandbox code */
     char **symbols;
+    void **cb_buf = lock_cb_buf();
     size_t depth = backtrace(cb_buf, MAX_DEPTH);
     symbols = backtrace_symbols(cb_buf, (int) depth);
     if (symbols)
       raw_free(symbols);
+    unlock_cb_buf(cb_buf);
   }
 
   return rv;
@@ -232,12 +271,29 @@ install_bt_handler(void)
 static void
 remove_bt_handler(void)
 {
+  int i;
+
+  struct sigaction sa;
+
+  memset(&sa, 0, sizeof(sa));
+  sa.sa_handler = SIG_DFL;
+  sigfillset(&sa.sa_mask);
+
+  for (i = 0; trap_signals[i] >= 0; ++i) {
+    /* remove_bt_handler() is called on shutdown, from low-level code.
+     * It's not a fatal error, so we just ignore it. */
+    (void)sigaction(trap_signals[i], &sa, NULL);
+  }
+
+  /* cb_buf_mutex is statically initialised, so we can not destroy it.
+   * If we destroy it, and then re-initialise tor, all our backtraces will
+   * fail. */
 }
 #endif /* defined(USE_BACKTRACE) */
 
 #ifdef NO_BACKTRACE_IMPL
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   logger(severity, domain, "%s: %s. (Stack trace not available)",
diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h
index 48b41fca02..7e09a0a5a7 100644
--- a/src/lib/err/backtrace.h
+++ b/src/lib/err/backtrace.h
@@ -12,11 +12,14 @@
 
 #include "orconfig.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/defs/logging_types.h"
 
-typedef void (*tor_log_fn)(int, unsigned, const char *fmt, ...)
+typedef void (*tor_log_fn)(int, log_domain_mask_t, const char *fmt, ...)
   CHECK_PRINTF(3,4);
 
-void log_backtrace_impl(int severity, int domain, const char *msg,
+void log_backtrace_impl(int severity, log_domain_mask_t domain,
+                        const char *msg,
                         tor_log_fn logger);
 int configure_backtrace_handler(const char *tor_version);
 void clean_up_backtrace_handler(void);
diff --git a/src/lib/err/include.am b/src/lib/err/include.am
index f2a409c51e..883ac91511 100644
--- a/src/lib/err/include.am
+++ b/src/lib/err/include.am
@@ -5,15 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-err-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_err_a_SOURCES =			\
-	src/lib/err/backtrace.c				\
-	src/lib/err/torerr.c
+	src/lib/err/backtrace.c			\
+	src/lib/err/torerr.c			\
+	src/lib/err/torerr_sys.c
 
 src_lib_libtor_err_testing_a_SOURCES = \
 	$(src_lib_libtor_err_a_SOURCES)
 src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/err/backtrace.h				\
-	src/lib/err/torerr.h
+	src/lib/err/torerr.h				\
+	src/lib/err/torerr_sys.h
diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c
index 6b5224273a..a5c00ca389 100644
--- a/src/lib/err/torerr.c
+++ b/src/lib/err/torerr.c
@@ -110,6 +110,14 @@ tor_log_get_sigsafe_err_fds(const int **out)
  * Update the list of fds that get errors from inside a signal handler or
  * other emergency condition. Ignore any beyond the first
  * TOR_SIGSAFE_LOG_MAX_FDS.
+ *
+ * These fds must remain open even after the log module has shut down. (And
+ * they should remain open even while logs are being reconfigured.) Therefore,
+ * any fds closed by the log module should be dup()ed, and the duplicate fd
+ * should be given to the err module in fds. In particular, the log module
+ * closes the file log fds, but does not close the stdio log fds.
+ *
+ * If fds is NULL or n is 0, clears the list of error fds.
  */
 void
 tor_log_set_sigsafe_err_fds(const int *fds, int n)
@@ -118,8 +126,28 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n)
     n = TOR_SIGSAFE_LOG_MAX_FDS;
   }
 
-  memcpy(sigsafe_log_fds, fds, n * sizeof(int));
-  n_sigsafe_log_fds = n;
+  /* Clear the entire array. This code mitigates against some race conditions,
+   * but there are still some races here:
+   * - err logs are disabled while the array is cleared, and
+   * - a thread can read the old value of n_sigsafe_log_fds, then read a
+   *   partially written array.
+   * We could fix these races using atomics, but atomics use the err module. */
+  n_sigsafe_log_fds = 0;
+  memset(sigsafe_log_fds, 0, sizeof(sigsafe_log_fds));
+  if (fds && n > 0) {
+    memcpy(sigsafe_log_fds, fds, n * sizeof(int));
+    n_sigsafe_log_fds = n;
+  }
+}
+
+/**
+ * Reset the list of emergency error fds to its default.
+ */
+void
+tor_log_reset_sigsafe_err_fds(void)
+{
+  int fds[] = { STDERR_FILENO };
+  tor_log_set_sigsafe_err_fds(fds, 1);
 }
 
 /**
@@ -161,6 +189,17 @@ tor_raw_assertion_failed_msg_(const char *file, int line, const char *expr,
   tor_log_err_sigsafe_write("\n");
 }
 
+/**
+ * Call the abort() function to kill the current process with a fatal
+ * error. This is a separate function, so that log users don't have to include
+ * the header for abort().
+ **/
+void
+tor_raw_abort_(void)
+{
+  abort();
+}
+
 /* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument
  * in range 2..16 inclusive. */
 static int
@@ -195,7 +234,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
     unsigned digit = (unsigned) (x % radix);
     if (cp <= buf) {
       /* Not tor_assert(); see above. */
-      abort();
+      tor_raw_abort_();
     }
     --cp;
     *cp = "0123456789ABCDEF"[digit];
@@ -204,7 +243,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
 
   /* NOT tor_assert; see above. */
   if (cp != buf) {
-    abort(); // LCOV_EXCL_LINE
+    tor_raw_abort_(); // LCOV_EXCL_LINE
   }
 
   return len;
@@ -224,8 +263,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
  * does not guarantee that an int is wider than a char (an int must be at
  * least 16 bits but it is permitted for a char to be that wide as well), we
  * can't assume a signed int is sufficient to accommodate an unsigned char.
- * Thus, format_helper_exit_status() will still need to emit any require '-'
- * on its own.
+ * Thus, callers will still need to add any required '-' to the final string.
  *
  * For most purposes, you'd want to use tor_snprintf("%x") instead of this
  * function; it's designed to be used in code paths where you can't call
diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h
index 6ae91fbe85..a1822d9c9a 100644
--- a/src/lib/err/torerr.h
+++ b/src/lib/err/torerr.h
@@ -20,13 +20,13 @@
 #define raw_assert(expr) STMT_BEGIN                                     \
     if (!(expr)) {                                                      \
       tor_raw_assertion_failed_msg_(__FILE__, __LINE__, #expr, NULL);   \
-      abort();                                                          \
+      tor_raw_abort_();                                                 \
     }                                                                   \
   STMT_END
 #define raw_assert_unreached(expr) raw_assert(0)
 #define raw_assert_unreached_msg(msg) STMT_BEGIN                    \
     tor_raw_assertion_failed_msg_(__FILE__, __LINE__, "0", (msg));  \
-    abort();                                                        \
+    tor_raw_abort_();                                               \
   STMT_END
 
 void tor_raw_assertion_failed_msg_(const char *file, int line,
@@ -39,9 +39,12 @@ void tor_raw_assertion_failed_msg_(const char *file, int line,
 void tor_log_err_sigsafe(const char *m, ...);
 int tor_log_get_sigsafe_err_fds(const int **out);
 void tor_log_set_sigsafe_err_fds(const int *fds, int n);
+void tor_log_reset_sigsafe_err_fds(void);
 void tor_log_sigsafe_err_set_granularity(int ms);
 
+void tor_raw_abort_(void) ATTR_NORETURN;
+
 int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len);
 int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_TORERR_H) */
diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c
new file mode 100644
index 0000000000..c42e958093
--- /dev/null
+++ b/src/lib/err/torerr_sys.c
@@ -0,0 +1,43 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.c
+ * \brief Subsystem object for the error handling subsystem.
+ **/
+
+#include "orconfig.h"
+#include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/subsys/subsys.h"
+#include "lib/version/torversion.h"
+
+#include <stddef.h>
+
+static int
+subsys_torerr_initialize(void)
+{
+  if (configure_backtrace_handler(get_version()) < 0)
+    return -1;
+  tor_log_reset_sigsafe_err_fds();
+
+  return 0;
+}
+static void
+subsys_torerr_shutdown(void)
+{
+  /* Stop handling signals with backtraces. */
+  clean_up_backtrace_handler();
+}
+
+const subsys_fns_t sys_torerr = {
+  .name = "err",
+  /* Low-level error handling is a diagnostic feature, we want it to init
+   * right after windows process security, and shutdown last.
+   * (Security never shuts down.) */
+  .level = -99,
+  .supported = true,
+  .initialize = subsys_torerr_initialize,
+  .shutdown = subsys_torerr_shutdown
+};
diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h
new file mode 100644
index 0000000000..c947695689
--- /dev/null
+++ b/src/lib/err/torerr_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.h
+ * \brief Declare subsystem object for torerr.c
+ **/
+
+#ifndef TOR_TORERR_SYS_H
+#define TOR_TORERR_SYS_H
+
+extern const struct subsys_fns_t sys_torerr;
+
+#endif /* !defined(TOR_TORERR_SYS_H) */
diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include
index 30af508914..54aa75fbff 100644
--- a/src/lib/evloop/.may_include
+++ b/src/lib/evloop/.may_include
@@ -8,9 +8,10 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/time/*.h
 
-src/ext/timeouts/timeout.c
-tor_queue.h
\ No newline at end of file
+ext/timeouts/timeout.c
+ext/tor_queue.h
\ No newline at end of file
diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c
new file mode 100644
index 0000000000..56641a3175
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.c
@@ -0,0 +1,49 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.c
+ * @brief Subsystem definition for the event loop module
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/evloop_sys.h"
+#include "lib/log/log.h"
+
+static int
+subsys_evloop_initialize(void)
+{
+  if (tor_init_libevent_rng() < 0) {
+    log_warn(LD_NET, "Problem initializing libevent RNG.");
+    return -1;
+  }
+  return 0;
+}
+
+static void
+subsys_evloop_postfork(void)
+{
+#ifdef TOR_UNIT_TESTS
+  tor_libevent_postfork();
+#endif
+}
+
+static void
+subsys_evloop_shutdown(void)
+{
+  tor_libevent_free_all();
+}
+
+const struct subsys_fns_t sys_evloop = {
+  .name = "evloop",
+  .supported = true,
+  .level = -20,
+  .initialize = subsys_evloop_initialize,
+  .shutdown = subsys_evloop_shutdown,
+  .postfork = subsys_evloop_postfork,
+};
diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h
new file mode 100644
index 0000000000..e6155c25b0
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.h
+ * @brief Declare subsystem object for the event loop module.
+ **/
+
+#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H
+#define TOR_LIB_EVLOOP_EVLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_evloop;
+
+#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */
diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am
index 6b0076272a..41cd2f45c5 100644
--- a/src/lib/evloop/include.am
+++ b/src/lib/evloop/include.am
@@ -5,21 +5,24 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-evloop-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_evloop_a_SOURCES =			\
 	src/lib/evloop/compat_libevent.c		\
+	src/lib/evloop/evloop_sys.c			\
 	src/lib/evloop/procmon.c			\
 	src/lib/evloop/timers.c				\
 	src/lib/evloop/token_bucket.c			\
 	src/lib/evloop/workqueue.c
 
-
 src_lib_libtor_evloop_testing_a_SOURCES = \
 	$(src_lib_libtor_evloop_a_SOURCES)
 src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/evloop/compat_libevent.h		\
+	src/lib/evloop/evloop_sys.h			\
 	src/lib/evloop/procmon.h			\
 	src/lib/evloop/timers.h				\
 	src/lib/evloop/token_bucket.h			\
diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c
index 52469fa5fc..b2d81fc14b 100644
--- a/src/lib/evloop/procmon.c
+++ b/src/lib/evloop/procmon.c
@@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_)
       tor_free(errmsg);
     }
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   /* Unix makes this part easy, if a bit racy. */
   its_dead_jim = kill(procmon->pid, 0);
   its_dead_jim = its_dead_jim && (errno == ESRCH);
diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c
index e46d2635a8..4b2a96ef7d 100644
--- a/src/lib/evloop/timers.c
+++ b/src/lib/evloop/timers.c
@@ -80,7 +80,8 @@ struct timeout_cb {
  * use 32-bit math. */
 #define WHEEL_BIT 5
 #endif
-#include "src/ext/timeouts/timeout.c"
+
+#include "ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
 static struct mainloop_event_t *global_timer_event = NULL;
diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c
index ee6d631e3b..ec62d1b018 100644
--- a/src/lib/evloop/token_bucket.c
+++ b/src/lib/evloop/token_bucket.c
@@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket,
     flags |= TB_WRITE;
   return flags;
 }
+
+/** Initialize a token bucket in <b>bucket</b>, set up to allow <b>rate</b>
+ * per second, with a maximum burst of <b>burst</b>. The bucket is created
+ * such that <b>now_ts</b> is the current timestamp. The bucket starts out
+ * full. */
+void
+token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                      uint32_t burst, uint32_t now_ts)
+{
+  memset(bucket, 0, sizeof(token_bucket_ctr_t));
+  token_bucket_ctr_adjust(bucket, rate, burst);
+  token_bucket_ctr_reset(bucket, now_ts);
+}
+
+/** Change the configured rate and burst of the given token bucket object in
+ * <b>bucket</b>. */
+void
+token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                        uint32_t burst)
+{
+  token_bucket_cfg_init(&bucket->cfg, rate, burst);
+  token_bucket_raw_adjust(&bucket->counter, &bucket->cfg);
+}
+
+/** Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. */
+void
+token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  token_bucket_raw_reset(&bucket->counter, &bucket->cfg);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
+
+/** Refill <b>bucket</b> as appropriate, given that the current timestamp is
+ * <b>now_ts</b>. */
+void
+token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  const uint32_t elapsed_ticks =
+    (now_ts - bucket->last_refilled_at_timestamp);
+  if (elapsed_ticks > UINT32_MAX-(300*1000)) {
+    /* Either about 48 days have passed since the last refill, or the
+     * monotonic clock has somehow moved backwards. (We're looking at you,
+     * Windows.).  We accept up to a 5 minute jump backwards as
+     * "unremarkable".
+     */
+    return;
+  }
+
+  token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg,
+                                elapsed_ticks);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h
index 9398d2baa3..dde9bd65a4 100644
--- a/src/lib/evloop/token_bucket.h
+++ b/src/lib/evloop/token_bucket.h
@@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
   return token_bucket_raw_get(&bucket->write_bucket);
 }
 
+/**
+ * A specialized bucket containing a single counter.
+ */
+
+typedef struct token_bucket_ctr_t {
+  token_bucket_cfg_t cfg;
+  token_bucket_raw_t counter;
+  uint32_t last_refilled_at_timestamp;
+} token_bucket_ctr_t;
+
+void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                           uint32_t burst, uint32_t now_ts);
+void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                             uint32_t burst);
+void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts);
+void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts);
+
+static inline bool
+token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->counter, n);
+}
+
+static inline size_t
+token_bucket_ctr_get(const token_bucket_ctr_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->counter);
+}
+
 #ifdef TOKEN_BUCKET_PRIVATE
 
 /* To avoid making the rates too small, we consider units of "steps",
@@ -112,6 +141,6 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
 
 STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate);
 
-#endif
+#endif /* defined(TOKEN_BUCKET_PRIVATE) */
 
-#endif /* TOR_TOKEN_BUCKET_H */
+#endif /* !defined(TOR_TOKEN_BUCKET_H) */
diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c
index 931f65e710..015b694290 100644
--- a/src/lib/evloop/workqueue.c
+++ b/src/lib/evloop/workqueue.c
@@ -15,7 +15,7 @@
  *
  * The main thread informs the worker threads of pending work by using a
  * condition variable.  The workers inform the main process of completed work
- * by using an alert_sockets_t object, as implemented in compat_threads.c.
+ * by using an alert_sockets_t object, as implemented in net/alertsock.c.
  *
  * The main thread can also queue an "update" that will be handled by all the
  * workers.  This is useful for updating state that all the workers share.
@@ -36,7 +36,7 @@
 #include "lib/net/socket.h"
 #include "lib/thread/threads.h"
 
-#include "tor_queue.h"
+#include "ext/tor_queue.h"
 #include <event2/event.h>
 #include <string.h>
 
@@ -59,9 +59,6 @@ struct threadpool_s {
    * <b>p</b> is work[p]. */
   work_tailq_t work[WORKQUEUE_N_PRIORITIES];
 
-  /** Weak RNG, used to decide when to ignore priority. */
-  tor_weak_rng_t weak_rng;
-
   /** The current 'update generation' of the threadpool.  Any thread that is
    * at an earlier generation needs to run the update function. */
   unsigned generation;
@@ -238,7 +235,7 @@ worker_thread_extract_next_work(workerthread_t *thread)
     this_queue = &pool->work[i];
     if (!TOR_TAILQ_EMPTY(this_queue)) {
       queue = this_queue;
-      if (! tor_weak_random_one_in_n(&pool->weak_rng,
+      if (! crypto_fast_rng_one_in_n(get_thread_fast_rng(),
                                      thread->lower_priority_chance)) {
         /* Usually we'll just break now, so that we can get out of the loop
          * and use the queue where we found work. But with a small
@@ -555,11 +552,6 @@ threadpool_new(int n_threads,
   for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
     TOR_TAILQ_INIT(&pool->work[i]);
   }
-  {
-    unsigned seed;
-    crypto_rand((void*)&seed, sizeof(seed));
-    tor_init_weak_random(&pool->weak_rng, seed);
-  }
 
   pool->new_thread_state_fn = new_thread_state_fn;
   pool->new_thread_state_arg = arg;
@@ -622,8 +614,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg)
     tp->reply_cb(tp);
 }
 
-/** Register the threadpool <b>tp</b>'s reply queue with the libevent
- * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after
+/** Register the threadpool <b>tp</b>'s reply queue with Tor's global
+ * libevent mainloop. If <b>cb</b> is provided, it is run after
  * each time there is work to process from the reply queue. Return 0 on
  * success, -1 on failure.
  */
diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h
index 333a3f6dde..d0ee8f2be2 100644
--- a/src/lib/evloop/workqueue.h
+++ b/src/lib/evloop/workqueue.h
@@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp);
 replyqueue_t *replyqueue_new(uint32_t alertsocks_flags);
 void replyqueue_process(replyqueue_t *queue);
 
-struct event_base;
 int threadpool_register_reply_event(threadpool_t *tp,
                                     void (*cb)(threadpool_t *tp));
 
diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c
index d723d04d2a..2d0864cc19 100644
--- a/src/lib/fdio/fdio.c
+++ b/src/lib/fdio/fdio.c
@@ -17,12 +17,16 @@
 #ifdef _WIN32
 #include <windows.h>
 #endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 
 #include "lib/fdio/fdio.h"
 #include "lib/cc/torint.h"
 #include "lib/err/torerr.h"
 
 #include <stdlib.h>
+#include <stdio.h>
 
 /** @{ */
 /** Some old versions of Unix didn't define constants for these values,
diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am
index 6c18f00a0d..545bbc929e 100644
--- a/src/lib/fdio/include.am
+++ b/src/lib/fdio/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fdio-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fdio_a_SOURCES =			\
 	src/lib/fdio/fdio.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_fdio_testing_a_SOURCES = \
 src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fdio/fdio.h
diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include
index b1e49fc891..c192e6181c 100644
--- a/src/lib/fs/.may_include
+++ b/src/lib/fs/.may_include
@@ -13,4 +13,4 @@ lib/malloc/*.h
 lib/memarea/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h
index 7af9119dbb..29115e1085 100644
--- a/src/lib/fs/conffile.h
+++ b/src/lib/fs/conffile.h
@@ -20,4 +20,4 @@ int config_get_lines_include(const char *string, struct config_line_t **result,
                              int extended, int *has_include,
                              struct smartlist_t *opened_lst);
 
-#endif /* !defined(TOR_CONFLINE_H) */
+#endif /* !defined(TOR_CONFFILE_H) */
diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c
index 3c31e00d99..390836b048 100644
--- a/src/lib/fs/dir.c
+++ b/src/lib/fs/dir.c
@@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check,
     }
   }
   close(fd);
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   /* Win32 case: we can't open() a directory. */
   (void)effective_user;
 
@@ -347,7 +347,7 @@ tor_listdir, (const char *dirname))
   }
   FindClose(handle);
   tor_free(pattern);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *prot_dname = sandbox_intern_string(dirname);
   DIR *d;
   struct dirent *de;
diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h
index 826bc2dfc5..9ff81faa42 100644
--- a/src/lib/fs/dir.h
+++ b/src/lib/fs/dir.h
@@ -30,4 +30,4 @@ MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check,
 
 MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
 
-#endif
+#endif /* !defined(TOR_DIR_H) */
diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h
index 52c94c914f..ed983f3b3c 100644
--- a/src/lib/fs/files.h
+++ b/src/lib/fs/files.h
@@ -27,7 +27,7 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -108,7 +108,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
  * Tor is built for unit tests, or when Tor is built on an operating system
  * without its own getdelim(). */
 ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
-#endif
+#endif /* !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS) */
 
 #ifdef HAVE_GETDELIM
 /**
@@ -123,10 +123,10 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getdelim(lineptr, n, delim, stream) \
   getdelim((lineptr), (n), (delim), (stream))
-#else
+#else /* !defined(HAVE_GETDELIM) */
 #define tor_getdelim(lineptr, n, delim, stream) \
   compat_getdelim_((lineptr), (n), (delim), (stream))
-#endif
+#endif /* defined(HAVE_GETDELIM) */
 
 #ifdef HAVE_GETLINE
 /**
@@ -137,9 +137,9 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getline(lineptr, n, stream) \
   getline((lineptr), (n), (stream))
-#else
+#else /* !defined(HAVE_GETLINE) */
 #define tor_getline(lineptr, n, stream) \
   tor_getdelim((lineptr), (n), '\n', (stream))
-#endif
+#endif /* defined(HAVE_GETLINE) */
 
-#endif
+#endif /* !defined(TOR_FS_H) */
diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am
index f33e4d6430..493db8f044 100644
--- a/src/lib/fs/include.am
+++ b/src/lib/fs/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fs-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fs_a_SOURCES =			\
 	src/lib/fs/conffile.c			\
 	src/lib/fs/dir.c			\
@@ -25,6 +26,7 @@ src_lib_libtor_fs_testing_a_SOURCES = \
 src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fs/conffile.h				\
 	src/lib/fs/dir.h				\
diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h
index 8aeee4cc7f..fc0281e253 100644
--- a/src/lib/fs/lockfile.h
+++ b/src/lib/fs/lockfile.h
@@ -17,4 +17,4 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
                                   int *locked_out);
 void tor_lockfile_unlock(tor_lockfile_t *lockfile);
 
-#endif
+#endif /* !defined(TOR_LOCKFILE_H) */
diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c
index daaee1f9b1..f71c0cff7a 100644
--- a/src/lib/fs/mmap.c
+++ b/src/lib/fs/mmap.c
@@ -237,4 +237,4 @@ tor_munmap_file(tor_mmap_t *handle)
 }
 #else
 #error "cannot implement tor_mmap_file"
-#endif /* defined(HAVE_MMAP) || ... || ... */
+#endif /* defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) || ... */
diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h
index 18fb18a13c..61aad544b2 100644
--- a/src/lib/fs/mmap.h
+++ b/src/lib/fs/mmap.h
@@ -38,4 +38,4 @@ typedef struct tor_mmap_t {
 tor_mmap_t *tor_mmap_file(const char *filename);
 int tor_munmap_file(tor_mmap_t *handle);
 
-#endif
+#endif /* !defined(TOR_MMAP_H) */
diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c
index b3ef61979d..28dde62aea 100644
--- a/src/lib/fs/path.c
+++ b/src/lib/fs/path.c
@@ -72,7 +72,7 @@ expand_filename(const char *filename)
    *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
    */
   return tor_strdup(filename);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   if (*filename == '~') {
     char *home, *result=NULL;
     const char *rest;
@@ -102,7 +102,7 @@ expand_filename(const char *filename)
       }
       tor_free(username);
       rest = slash ? (slash+1) : "";
-#else /* !(defined(HAVE_PWD_H)) */
+#else /* !defined(HAVE_PWD_H) */
       log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
       return tor_strdup(filename);
 #endif /* defined(HAVE_PWD_H) */
@@ -153,7 +153,7 @@ clean_fname_for_stat(char *name)
       return;
     name[len-1]='\0';
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)name;
 #endif /* defined(_WIN32) */
 }
@@ -233,7 +233,7 @@ alloc_getcwd(void)
     raw_free(cwd); // alias for free to avoid tripping check-spaces.
   }
   return result;
-#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */
   size_t size = 1024;
   char *buf = NULL;
   char *ptr = NULL;
@@ -268,7 +268,7 @@ make_path_absolute(char *fname)
   if (absfname_malloced) raw_free(absfname_malloced);
 
   return absfname;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   char *absfname = NULL, *path = NULL;
 
   tor_assert(fname);
diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h
index 4675ac84e8..28a1838b88 100644
--- a/src/lib/fs/path.h
+++ b/src/lib/fs/path.h
@@ -27,4 +27,4 @@ void clean_fname_for_stat(char *name);
 int get_parent_directory(char *fname);
 char *make_path_absolute(char *fname);
 
-#endif
+#endif /* !defined(TOR_PATH_H) */
diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h
index 5c39794873..5e5ddb89a3 100644
--- a/src/lib/fs/userdb.h
+++ b/src/lib/fs/userdb.h
@@ -21,6 +21,6 @@ struct passwd;
 const struct passwd *tor_getpwnam(const char *username);
 const struct passwd *tor_getpwuid(uid_t uid);
 char *get_user_homedir(const char *username);
-#endif
+#endif /* !defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_USERDB_H) */
diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h
index 64a22439e5..7237226c76 100644
--- a/src/lib/fs/winlib.h
+++ b/src/lib/fs/winlib.h
@@ -17,6 +17,6 @@
 #include <tchar.h>
 
 HANDLE load_windows_system_library(const TCHAR *library_name);
-#endif
+#endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_WINLIB_H) */
diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h
index 9a8911d494..a24a1c4c0d 100644
--- a/src/lib/geoip/country.h
+++ b/src/lib/geoip/country.h
@@ -13,4 +13,4 @@ typedef int16_t country_t;
 
 #define COUNTRY_MAX INT16_MAX
 
-#endif
+#endif /* !defined(TOR_COUNTRY_H) */
diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am
index 9710d75ac7..ea426d14bc 100644
--- a/src/lib/geoip/include.am
+++ b/src/lib/geoip/include.am
@@ -4,6 +4,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-geoip-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_geoip_a_SOURCES =			\
 	src/lib/geoip/geoip.c
 
@@ -12,6 +13,7 @@ src_lib_libtor_geoip_testing_a_SOURCES = \
 src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/geoip/geoip.h   			\
 	src/lib/geoip/country.h
diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h
index 83efa82919..3f745d457d 100644
--- a/src/lib/intmath/addsub.h
+++ b/src/lib/intmath/addsub.h
@@ -16,4 +16,4 @@
 
 uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
 
-#endif /* !defined(TOR_INTMATH_MULDIV_H) */
+#endif /* !defined(TOR_INTMATH_ADDSUB_H) */
diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h
index d0b0e8b954..67a738861b 100644
--- a/src/lib/intmath/cmp.h
+++ b/src/lib/intmath/cmp.h
@@ -36,4 +36,7 @@
     ((v) > (max)) ? (max) :                     \
     (v) )
 
+/** Give the absolute value of <b>x</b>, independent of its type. */
+#define ABS(x) ( ((x)<0) ? -(x) : (x) )
+
 #endif /* !defined(TOR_INTMATH_CMP_H) */
diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am
index 45ee3bd53b..155ffa145a 100644
--- a/src/lib/intmath/include.am
+++ b/src/lib/intmath/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-intmath-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_intmath_a_SOURCES =			\
 	src/lib/intmath/addsub.c			\
 	src/lib/intmath/bits.c				\
@@ -16,6 +17,7 @@ src_lib_libtor_intmath_testing_a_SOURCES = \
 src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/intmath/addsub.h			\
 	src/lib/intmath/cmp.h				\
diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h
index a4cecd69cc..b2f77462e1 100644
--- a/src/lib/intmath/logic.h
+++ b/src/lib/intmath/logic.h
@@ -17,4 +17,4 @@
 /** Macro: true if two values have different boolean values. */
 #define bool_neq(a,b) (!(a)!=!(b))
 
-#endif
+#endif /* !defined(HAVE_TOR_LOGIC_H) */
diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h
index e26bf58cbb..40941e59b2 100644
--- a/src/lib/intmath/weakrng.h
+++ b/src/lib/intmath/weakrng.h
@@ -28,4 +28,4 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
  * <b>n</b> */
 #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n)))
 
-#endif
+#endif /* !defined(TOR_WEAKRNG_H) */
diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c
index 4ad5929715..670bd0174c 100644
--- a/src/lib/lock/compat_mutex.c
+++ b/src/lib/lock/compat_mutex.c
@@ -29,7 +29,15 @@ tor_mutex_new_nonrecursive(void)
   tor_mutex_init_nonrecursive(m);
   return m;
 }
-/** Release all storage and system resources held by <b>m</b>. */
+/** Release all storage and system resources held by <b>m</b>.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_free_(tor_mutex_t *m)
 {
diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h
index b63ce24024..e0c3d7cb78 100644
--- a/src/lib/lock/compat_mutex.h
+++ b/src/lib/lock/compat_mutex.h
@@ -48,7 +48,7 @@ typedef struct tor_mutex_t {
 #else
   /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
   int _unused;
-#endif /* defined(USE_WIN32_MUTEX) || ... */
+#endif /* defined(USE_WIN32_THREADS) || ... */
 } tor_mutex_t;
 
 tor_mutex_t *tor_mutex_new(void);
diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c
index ee5f520cd0..f82ad9f0e8 100644
--- a/src/lib/lock/compat_mutex_pthreads.c
+++ b/src/lib/lock/compat_mutex_pthreads.c
@@ -88,12 +88,26 @@ tor_mutex_release(tor_mutex_t *m)
 }
 /** Clean up the mutex <b>m</b> so that it no longer uses any system
  * resources.  Does not free <b>m</b>.  This function must only be called on
- * mutexes from tor_mutex_init(). */
+ * mutexes from tor_mutex_init().
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_uninit(tor_mutex_t *m)
 {
   int err;
   raw_assert(m);
+  /* If the mutex is already locked, wait until after it is unlocked to destroy
+   * it. Locking and releasing the mutex makes undefined behaviour less likely,
+   * but does not prevent it. Another thread can lock the mutex between release
+   * and destroy. */
+  tor_mutex_acquire(m);
+  tor_mutex_release(m);
   err = pthread_mutex_destroy(&m->mutex);
   if (PREDICT_UNLIKELY(err)) {
     // LCOV_EXCL_START
diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am
index 4e6f444347..1475b9911b 100644
--- a/src/lib/lock/include.am
+++ b/src/lib/lock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-lock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_lock_a_SOURCES =			\
 	src/lib/lock/compat_mutex.c
 
@@ -20,5 +21,6 @@ src_lib_libtor_lock_testing_a_SOURCES = \
 src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/lock/compat_mutex.h
diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include
index 852173aab3..54d96324db 100644
--- a/src/lib/log/.may_include
+++ b/src/lib/log/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/cc/*.h
+lib/defs/*.h
 lib/smartlist_core/*.h
 lib/err/*.h
 lib/fdio/*.h
@@ -9,7 +10,7 @@ lib/lock/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
+lib/version/*.h
 lib/wallclock/*.h
-
-micro-revision.i
\ No newline at end of file
diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h
index 2f726186c5..0b9fc3406b 100644
--- a/src/lib/log/escape.h
+++ b/src/lib/log/escape.h
@@ -20,4 +20,4 @@ char *esc_for_log(const char *string) ATTR_MALLOC;
 char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC;
 const char *escaped(const char *string);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_ESCAPE_H) */
diff --git a/src/lib/log/include.am b/src/lib/log/include.am
index 4a6c9b3686..5b9f7113ba 100644
--- a/src/lib/log/include.am
+++ b/src/lib/log/include.am
@@ -5,11 +5,12 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-log-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_log_a_SOURCES =			\
 	src/lib/log/escape.c			\
-        src/lib/log/git_revision.c              \
 	src/lib/log/ratelim.c			\
 	src/lib/log/log.c			\
+	src/lib/log/log_sys.c			\
 	src/lib/log/util_bug.c
 
 if WIN32
@@ -21,16 +22,11 @@ src_lib_libtor_log_testing_a_SOURCES = \
 src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
-# Declare that these object files depend on micro-revision.i.  Without this
-# rule, we could try to build them before micro-revision.i was created.
-src/lib/log/git_revision.$(OBJEXT) \
-  src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \
-	micro-revision.i
-
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/log/escape.h				\
-        src/lib/log/git_revision.h                      \
 	src/lib/log/ratelim.h				\
 	src/lib/log/log.h				\
+	src/lib/log/log_sys.h				\
 	src/lib/log/util_bug.h				\
 	src/lib/log/win32err.h
diff --git a/src/lib/log/log.c b/src/lib/log/log.c
index a9ad38fb25..c53db46a8c 100644
--- a/src/lib/log/log.c
+++ b/src/lib/log/log.c
@@ -32,7 +32,8 @@
 
 #define LOG_PRIVATE
 #include "lib/log/log.h"
-#include "lib/log/git_revision.h"
+#include "lib/log/log_sys.h"
+#include "lib/version/git_revision.h"
 #include "lib/log/ratelim.h"
 #include "lib/lock/compat_mutex.h"
 #include "lib/smartlist_core/smartlist_core.h"
@@ -48,15 +49,12 @@
 #include "lib/wallclock/approx_time.h"
 #include "lib/wallclock/time_to_tm.h"
 #include "lib/fdio/fdio.h"
+#include "lib/cc/ctassert.h"
 
 #ifdef HAVE_ANDROID_LOG_H
 #include <android/log.h>
 #endif // HAVE_ANDROID_LOG_H.
 
-/** Given a severity, yields an index into log_severity_list_t.masks to use
- * for that severity. */
-#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
-
 /** @{ */
 /** The string we stick at the end of a log message when it is too long,
  * and its length. */
@@ -153,7 +151,7 @@ severity_to_android_log_priority(int severity)
       // LCOV_EXCL_STOP
   }
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** A mutex to guard changes to logfiles and logging. */
 static tor_mutex_t log_mutex;
@@ -223,6 +221,7 @@ int log_global_min_severity_ = LOG_NOTICE;
 
 static void delete_log(logfile_t *victim);
 static void close_log(logfile_t *victim);
+static void close_log_sigsafe(logfile_t *victim);
 
 static char *domain_to_string(log_domain_mask_t domain,
                              char *buf, size_t buflen);
@@ -533,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
     if (m != msg_after_prefix) {
       tor_free(m);
     }
-#else /* !(defined(MAXLINE)) */
+#else /* !defined(MAXLINE) */
     /* We have syslog but not MAXLINE.  That's promising! */
     syslog(severity, "%s", msg_after_prefix);
 #endif /* defined(MAXLINE) */
@@ -585,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname,
   /* check that severity is sane.  Overrunning the masks array leads to
    * interesting and hard to diagnose effects */
   raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG);
-  /* check that we've initialised the log mutex before we try to lock it */
-  raw_assert(log_mutex_initialized);
+
   LOCK_LOGS();
 
   if ((! (domain & LD_NOCB)) && pending_cb_messages
@@ -630,6 +628,10 @@ void
 tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
 {
   va_list ap;
+
+  /* check that domain is composed of known domains and flags */
+  raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain);
+
   if (severity > log_global_min_severity_)
     return;
   va_start(ap,format);
@@ -663,18 +665,26 @@ tor_log_update_sigsafe_err_fds(void)
   const logfile_t *lf;
   int found_real_stderr = 0;
 
-  int fds[TOR_SIGSAFE_LOG_MAX_FDS];
+  /* log_fds and err_fds contain matching entries: log_fds are the fds used by
+   * the log module, and err_fds are the fds used by the err module.
+   * For stdio logs, the log_fd and err_fd values are identical.
+   * For file logs, the err_fd is a dup() of the log_fd.
+   */
+  int log_fds[TOR_SIGSAFE_LOG_MAX_FDS];
+  int err_fds[TOR_SIGSAFE_LOG_MAX_FDS];
   int n_fds;
 
   LOCK_LOGS();
   /* Reserve the first one for stderr. This is safe because when we daemonize,
-   * we dup2 /dev/null to stderr, */
-  fds[0] = STDERR_FILENO;
+   * we dup2 /dev/null to stderr.
+   * For stderr, log_fds and err_fds are the same. */
+  log_fds[0] = err_fds[0] = STDERR_FILENO;
   n_fds = 1;
 
   for (lf = logfiles; lf; lf = lf->next) {
-     /* Don't try callback to the control port, or syslogs: We can't
-      * do them from a signal handler. Don't try stdout: we always do stderr.
+     /* Don't try callback to the control port, syslogs, android logs, or any
+      * other non-file descriptor log: We can't call arbitrary functions from a
+      * signal handler.
       */
     if (lf->is_temporary || logfile_is_external(lf)
         || lf->seems_dead || lf->fd < 0)
@@ -683,25 +693,40 @@ tor_log_update_sigsafe_err_fds(void)
         (LD_BUG|LD_GENERAL)) {
       if (lf->fd == STDERR_FILENO)
         found_real_stderr = 1;
-      /* Avoid duplicates */
-      if (int_array_contains(fds, n_fds, lf->fd))
+      /* Avoid duplicates by checking the log module fd against log_fds */
+      if (int_array_contains(log_fds, n_fds, lf->fd))
         continue;
-      fds[n_fds++] = lf->fd;
+      /* Update log_fds using the log module's fd */
+      log_fds[n_fds] = lf->fd;
+      if (lf->needs_close) {
+        /* File log fds are duplicated, because close_log() closes the log
+         * module's fd. Both refer to the same file. */
+        err_fds[n_fds] = dup(lf->fd);
+      } else {
+        /* stdio log fds are not closed by the log module. */
+        err_fds[n_fds] = lf->fd;
+      }
+      n_fds++;
       if (n_fds == TOR_SIGSAFE_LOG_MAX_FDS)
         break;
     }
   }
 
   if (!found_real_stderr &&
-      int_array_contains(fds, n_fds, STDOUT_FILENO)) {
-    /* Don't use a virtual stderr when we're also logging to stdout. */
+      int_array_contains(log_fds, n_fds, STDOUT_FILENO)) {
+    /* Don't use a virtual stderr when we're also logging to stdout.
+     * If we reached max_fds logs, we'll now have (max_fds - 1) logs.
+     * That's ok, max_fds is large enough that most tor instances don't exceed
+     * it. */
     raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */
-    fds[0] = fds[--n_fds];
+    --n_fds;
+    log_fds[0] = log_fds[n_fds];
+    err_fds[0] = err_fds[n_fds];
   }
 
   UNLOCK_LOGS();
 
-  tor_log_set_sigsafe_err_fds(fds, n_fds);
+  tor_log_set_sigsafe_err_fds(err_fds, n_fds);
 }
 
 /** Add to <b>out</b> a copy of every currently configured log file name. Used
@@ -727,7 +752,7 @@ tor_log_get_logfile_names(smartlist_t *out)
 
 /** Implementation of the log_fn backend, used when we have
  * variadic macros. All arguments are as for log_fn, except for
- * <b>fn</b>, which is the name of the calling functions. */
+ * <b>fn</b>, which is the name of the calling function. */
 void
 log_fn_(int severity, log_domain_mask_t domain, const char *fn,
         const char *format, ...)
@@ -804,16 +829,16 @@ logs_free_all(void)
   }
 
   /* We _could_ destroy the log mutex here, but that would screw up any logs
-   * that happened between here and the end of execution. */
+   * that happened between here and the end of execution.
+   * If tor is re-initialized, log_mutex_initialized will still be 1. So we
+   * won't trigger any undefined behaviour by trying to re-initialize the
+   * log mutex. */
 }
 
 /** Remove and free the log entry <b>victim</b> from the linked-list
  * logfiles (it is probably present, but it might not be due to thread
  * racing issues). After this function is called, the caller shouldn't
  * refer to <b>victim</b> anymore.
- *
- * Long-term, we need to do something about races in the log subsystem
- * in general. See bug 222 for more details.
  */
 static void
 delete_log(logfile_t *victim)
@@ -833,13 +858,26 @@ delete_log(logfile_t *victim)
 }
 
 /** Helper: release system resources (but not memory) held by a single
- * logfile_t. */
+ * signal-safe logfile_t. If the log's resources can not be released in
+ * a signal handler, does nothing. */
 static void
-close_log(logfile_t *victim)
+close_log_sigsafe(logfile_t *victim)
 {
   if (victim->needs_close && victim->fd >= 0) {
+    /* We can't do anything useful here if close() fails: we're shutting
+     * down logging, and the err module only does fatal errors. */
     close(victim->fd);
     victim->fd = -1;
+  }
+}
+
+/** Helper: release system resources (but not memory) held by a single
+ * logfile_t. */
+static void
+close_log(logfile_t *victim)
+{
+  if (victim->needs_close) {
+    close_log_sigsafe(victim);
   } else if (victim->is_syslog) {
 #ifdef HAVE_SYSLOG_H
     if (--syslog_count == 0) {
@@ -863,7 +901,7 @@ set_log_severity_config(int loglevelMin, int loglevelMax,
   raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
   memset(severity_out, 0, sizeof(log_severity_list_t));
   for (i = loglevelMin; i >= loglevelMax; --i) {
-    severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+    severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
 }
 
@@ -1020,7 +1058,7 @@ flush_pending_log_callbacks(void)
   do {
     SMARTLIST_FOREACH_BEGIN(messages, pending_log_message_t *, msg) {
       const int severity = msg->severity;
-      const int domain = msg->domain;
+      const log_domain_mask_t domain = msg->domain;
       for (lf = logfiles; lf; lf = lf->next) {
         if (! lf->callback || lf->seems_dead ||
             ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) {
@@ -1231,7 +1269,7 @@ add_android_log(const log_severity_list_t *severity,
   UNLOCK_LOGS();
   return 0;
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** If <b>level</b> is a valid log severity, return the corresponding
  * numeric value.  Otherwise, return -1. */
@@ -1267,9 +1305,16 @@ static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
   "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL",
-  "SCHED", "GUARD", "CONSDIFF", "DOS", NULL
+  "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", "MESG",
+  NULL
 };
 
+CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1);
+
+CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS);
+CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_);
+CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS);
+
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,
  * or 0 if there is no such name. */
 static log_domain_mask_t
@@ -1278,7 +1323,7 @@ parse_log_domain(const char *domain)
   int i;
   for (i=0; domain_list[i]; ++i) {
     if (!strcasecmp(domain, domain_list[i]))
-      return (1u<<i);
+      return (UINT64_C(1)<<i);
   }
   return 0;
 }
@@ -1350,7 +1395,7 @@ parse_log_severity_config(const char **cfg_ptr,
     const char *dash, *space;
     char *sev_lo, *sev_hi;
     int low, high, i;
-    log_domain_mask_t domains = ~0u;
+    log_domain_mask_t domains = LD_ALL_DOMAINS;
 
     if (*cfg == '[') {
       int err = 0;
@@ -1368,9 +1413,9 @@ parse_log_severity_config(const char **cfg_ptr,
       tor_free(domains_str);
       SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) {
             if (!strcmp(domain, "*")) {
-              domains = ~0u;
+              domains = LD_ALL_DOMAINS;
             } else {
-              int d;
+              log_domain_mask_t d;
               int negate=0;
               if (*domain == '~') {
                 negate = 1;
@@ -1464,7 +1509,7 @@ switch_logs_debug(void)
   LOCK_LOGS();
   for (lf = logfiles; lf; lf=lf->next) {
     for (i = LOG_DEBUG; i >= LOG_ERR; --i)
-      lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+      lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
   log_global_min_severity_ = get_min_log_level();
   UNLOCK_LOGS();
diff --git a/src/lib/log/log.h b/src/lib/log/log.h
index d7a5070610..f09c63cdd9 100644
--- a/src/lib/log/log.h
+++ b/src/lib/log/log.h
@@ -11,10 +11,12 @@
  **/
 
 #ifndef TOR_TORLOG_H
+#define TOR_TORLOG_H
 
 #include <stdarg.h>
 #include "lib/cc/torint.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/logging_types.h"
 #include "lib/testsupport/testsupport.h"
 
 #ifdef HAVE_SYSLOG_H
@@ -24,7 +26,7 @@
 #error "Your syslog.h thinks high numbers are more important.  " \
        "We aren't prepared to deal with that."
 #endif
-#else /* !(defined(HAVE_SYSLOG_H)) */
+#else /* !defined(HAVE_SYSLOG_H) */
 /* Note: Syslog's logging code refers to priorities, with 0 being the most
  * important.  Thus, all our comparisons needed to be reversed when we added
  * syslog support.
@@ -55,75 +57,92 @@
 /* Logging domains */
 
 /** Catch-all for miscellaneous events and fatal errors. */
-#define LD_GENERAL  (1u<<0)
+#define LD_GENERAL  (UINT64_C(1)<<0)
 /** The cryptography subsystem. */
-#define LD_CRYPTO   (1u<<1)
+#define LD_CRYPTO   (UINT64_C(1)<<1)
 /** Networking. */
-#define LD_NET      (1u<<2)
+#define LD_NET      (UINT64_C(1)<<2)
 /** Parsing and acting on our configuration. */
-#define LD_CONFIG   (1u<<3)
+#define LD_CONFIG   (UINT64_C(1)<<3)
 /** Reading and writing from the filesystem. */
-#define LD_FS       (1u<<4)
+#define LD_FS       (UINT64_C(1)<<4)
 /** Other servers' (non)compliance with the Tor protocol. */
-#define LD_PROTOCOL (1u<<5)
+#define LD_PROTOCOL (UINT64_C(1)<<5)
 /** Memory management. */
-#define LD_MM       (1u<<6)
+#define LD_MM       (UINT64_C(1)<<6)
 /** HTTP implementation. */
-#define LD_HTTP     (1u<<7)
+#define LD_HTTP     (UINT64_C(1)<<7)
 /** Application (socks) requests. */
-#define LD_APP      (1u<<8)
+#define LD_APP      (UINT64_C(1)<<8)
 /** Communication via the controller protocol. */
-#define LD_CONTROL  (1u<<9)
+#define LD_CONTROL  (UINT64_C(1)<<9)
 /** Building, using, and managing circuits. */
-#define LD_CIRC     (1u<<10)
+#define LD_CIRC     (UINT64_C(1)<<10)
 /** Hidden services. */
-#define LD_REND     (1u<<11)
+#define LD_REND     (UINT64_C(1)<<11)
 /** Internal errors in this Tor process. */
-#define LD_BUG      (1u<<12)
+#define LD_BUG      (UINT64_C(1)<<12)
 /** Learning and using information about Tor servers. */
-#define LD_DIR      (1u<<13)
+#define LD_DIR      (UINT64_C(1)<<13)
 /** Learning and using information about Tor servers. */
-#define LD_DIRSERV  (1u<<14)
+#define LD_DIRSERV  (UINT64_C(1)<<14)
 /** Onion routing protocol. */
-#define LD_OR       (1u<<15)
+#define LD_OR       (UINT64_C(1)<<15)
 /** Generic edge-connection functionality. */
-#define LD_EDGE     (1u<<16)
+#define LD_EDGE     (UINT64_C(1)<<16)
 #define LD_EXIT     LD_EDGE
 /** Bandwidth accounting. */
-#define LD_ACCT     (1u<<17)
+#define LD_ACCT     (UINT64_C(1)<<17)
 /** Router history */
-#define LD_HIST     (1u<<18)
+#define LD_HIST     (UINT64_C(1)<<18)
 /** OR handshaking */
-#define LD_HANDSHAKE (1u<<19)
+#define LD_HANDSHAKE (UINT64_C(1)<<19)
 /** Heartbeat messages */
-#define LD_HEARTBEAT (1u<<20)
+#define LD_HEARTBEAT (UINT64_C(1)<<20)
 /** Abstract channel_t code */
-#define LD_CHANNEL   (1u<<21)
+#define LD_CHANNEL   (UINT64_C(1)<<21)
 /** Scheduler */
-#define LD_SCHED     (1u<<22)
+#define LD_SCHED     (UINT64_C(1)<<22)
 /** Guard nodes */
-#define LD_GUARD     (1u<<23)
+#define LD_GUARD     (UINT64_C(1)<<23)
 /** Generation and application of consensus diffs. */
-#define LD_CONSDIFF  (1u<<24)
+#define LD_CONSDIFF  (UINT64_C(1)<<24)
 /** Denial of Service mitigation. */
-#define LD_DOS       (1u<<25)
-/** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 26
+#define LD_DOS       (UINT64_C(1)<<25)
+/** Processes */
+#define LD_PROCESS   (UINT64_C(1)<<26)
+/** Pluggable Transports. */
+#define LD_PT        (UINT64_C(1)<<27)
+/** Bootstrap tracker. */
+#define LD_BTRACK    (UINT64_C(1)<<28)
+/** Message-passing backend. */
+#define LD_MESG      (UINT64_C(1)<<29)
 
-/** This log message is not safe to send to a callback-based logger
- * immediately.  Used as a flag, not a log domain. */
-#define LD_NOCB (1u<<31)
-/** This log message should not include a function name, even if it otherwise
- * would. Used as a flag, not a log domain. */
-#define LD_NOFUNCNAME (1u<<30)
+/** The number of log domains. */
+#define N_LOGGING_DOMAINS 30
+/** The highest log domain */
+#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1))
+/** All log domains. */
+#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS))
+
+/** The number of log flags. */
+#define N_LOGGING_FLAGS 3
+/** First bit that is reserved in log_domain_mask_t for non-domain flags. */
+#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS))
+/** All log flags. */
+#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS))
 
 #ifdef TOR_UNIT_TESTS
 /** This log message should not be intercepted by mock_saving_logv */
-#define LD_NO_MOCK (1u<<29)
+#define LD_NO_MOCK (UINT64_C(1)<<61)
 #endif
 
-/** Mask of zero or more log domains, OR'd together. */
-typedef uint32_t log_domain_mask_t;
+/** This log message is not safe to send to a callback-based logger
+ * immediately.  Used as a flag, not a log domain. */
+#define LD_NOCB (UINT64_C(1)<<62)
+/** This log message should not include a function name, even if it otherwise
+ * would. Used as a flag, not a log domain. */
+#define LD_NOFUNCNAME (UINT64_C(1)<<63)
 
 /** Configures which severities are logged for each logging domain for a given
  * log target. */
@@ -134,7 +153,8 @@ typedef struct log_severity_list_t {
 } log_severity_list_t;
 
 /** Callback type used for add_callback_log. */
-typedef void (*log_callback)(int severity, uint32_t domain, const char *msg);
+typedef void (*log_callback)(int severity, log_domain_mask_t domain,
+                             const char *msg);
 
 void init_logging(int disable_startup_queue);
 int parse_log_level(const char *level);
@@ -186,6 +206,21 @@ void tor_log_get_logfile_names(struct smartlist_t *out);
 
 extern int log_global_min_severity_;
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we try to avoid our log_debug optimization, since it
+ * can have weird effects on internal macro coverage. */
+#define debug_logging_enabled() (1)
+#else
+static inline bool debug_logging_enabled(void);
+/**
+ * Return true iff debug logging is enabled for at least one domain.
+ */
+static inline bool debug_logging_enabled(void)
+{
+  return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG);
+}
+#endif /* defined(TOR_COVERAGE) */
+
 void log_fn_(int severity, log_domain_mask_t domain,
              const char *funcname, const char *format, ...)
   CHECK_PRINTF(4,5);
@@ -215,8 +250,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
   log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, args)
 #define log_debug(domain, args...)                                      \
   STMT_BEGIN                                                            \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))        \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);            \
+    if (debug_logging_enabled())                                        \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);                   \
   STMT_END
 #define log_info(domain, args...)                           \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args)
@@ -233,8 +268,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
 
 #define log_debug(domain, args, ...)                                        \
   STMT_BEGIN                                                                \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))            \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \
+    if (debug_logging_enabled())                                            \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__);        \
   STMT_END
 #define log_info(domain, args,...)                                      \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args, ##__VA_ARGS__)
@@ -272,5 +307,10 @@ MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
     va_list ap) CHECK_PRINTF(5,0));
 #endif
 
-# define TOR_TORLOG_H
+#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS)
+/** Given a severity, yields an index into log_severity_list_t.masks to use
+ * for that severity. */
+#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
+#endif
+
 #endif /* !defined(TOR_TORLOG_H) */
diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c
new file mode 100644
index 0000000000..826358546a
--- /dev/null
+++ b/src/lib/log/log_sys.c
@@ -0,0 +1,37 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.c
+ * \brief Setup and tear down the logging module.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/log_sys.h"
+
+static int
+subsys_logging_initialize(void)
+{
+  init_logging(0);
+  return 0;
+}
+
+static void
+subsys_logging_shutdown(void)
+{
+  logs_free_all();
+  escaped(NULL);
+}
+
+const subsys_fns_t sys_logging = {
+  .name = "log",
+  .supported = true,
+  /* Logging depends on threads, approx time, raw logging, and security.
+   * Most other lib modules depend on logging. */
+  .level = -90,
+  .initialize = subsys_logging_initialize,
+  .shutdown = subsys_logging_shutdown,
+};
diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h
new file mode 100644
index 0000000000..7043253066
--- /dev/null
+++ b/src/lib/log/log_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.h
+ * \brief Declare subsystem object for the logging module.
+ **/
+
+#ifndef TOR_LOG_SYS_H
+#define TOR_LOG_SYS_H
+
+extern const struct subsys_fns_t sys_logging;
+
+#endif /* !defined(TOR_LOG_SYS_H) */
diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h
index 48edd7c849..1db54ba726 100644
--- a/src/lib/log/ratelim.h
+++ b/src/lib/log/ratelim.h
@@ -50,4 +50,4 @@ typedef struct ratelim_t {
 
 char *rate_limit_log(ratelim_t *lim, time_t now);
 
-#endif
+#endif /* !defined(TOR_RATELIM_H) */
diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c
index c65a91ae9e..af1ac40732 100644
--- a/src/lib/log/util_bug.c
+++ b/src/lib/log/util_bug.c
@@ -11,6 +11,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/log/log.h"
 #include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
 #ifdef TOR_UNIT_TESTS
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -63,32 +64,52 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 {
   failed_assertion_cb = fn;
 }
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
 #define capturing_bugs() (0)
 #define add_captured_bug(s) do { } while (0)
 #endif /* defined(TOR_UNIT_TESTS) */
 
 /** Helper for tor_assert: report the assertion failure. */
 void
+CHECK_PRINTF(5, 6)
 tor_assertion_failed_(const char *fname, unsigned int line,
-                      const char *func, const char *expr)
+                      const char *func, const char *expr,
+                      const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
+  char *extra = NULL;
+  va_list ap;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+  if (fmt) {
+    va_start(ap,fmt);
+    tor_vasprintf(&extra, fmt, ap);
+    va_end(ap);
+  }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
   log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.",
           fname, line, func, expr);
-  tor_snprintf(buf, sizeof(buf),
-               "Assertion %s failed in %s at %s:%u",
-               expr, func, fname, line);
+  tor_asprintf(&buf, "Assertion %s failed in %s at %s:%u: %s",
+               expr, func, fname, line, extra ? extra : "");
+  tor_free(extra);
   log_backtrace(LOG_ERR, LD_BUG, buf);
+  tor_free(buf);
 }
 
 /** Helper for tor_assert_nonfatal: report the assertion failure. */
 void
+CHECK_PRINTF(6, 7)
 tor_bug_occurred_(const char *fname, unsigned int line,
                   const char *func, const char *expr,
-                  int once)
+                  int once, const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
   const char *once_str = once ?
     " (Future instances of this warning will be silenced.)": "";
   if (! expr) {
@@ -98,7 +119,7 @@ tor_bug_occurred_(const char *fname, unsigned int line,
     }
     log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s",
              fname, line, func, once_str);
-    tor_snprintf(buf, sizeof(buf),
+    tor_asprintf(&buf,
                  "Line unexpectedly reached at %s at %s:%u",
                  func, fname, line);
   } else {
@@ -106,13 +127,32 @@ tor_bug_occurred_(const char *fname, unsigned int line,
       add_captured_bug(expr);
       return;
     }
+
+    va_list ap;
+    char *extra = NULL;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+    if (fmt) {
+      va_start(ap,fmt);
+      tor_vasprintf(&extra, fmt, ap);
+      va_end(ap);
+    }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
     log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s",
              fname, line, func, expr, once_str);
-    tor_snprintf(buf, sizeof(buf),
-                 "Non-fatal assertion %s failed in %s at %s:%u",
-                 expr, func, fname, line);
+    tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s",
+                 expr, func, fname, line, fmt ? " : " : "",
+                 extra ? extra : "");
+    tor_free(extra);
   }
   log_backtrace(LOG_WARN, LD_BUG, buf);
+  tor_free(buf);
 
 #ifdef TOR_UNIT_TESTS
   if (failed_assertion_cb) {
@@ -122,16 +162,16 @@ tor_bug_occurred_(const char *fname, unsigned int line,
 }
 
 /**
- * Call the abort() function to kill the current process with a fatal
- * error.
+ * Call the tor_raw_abort_() function to close raw logs, then kill the current
+ * process with a fatal error.
  *
  * (This is a separate function so that we declare it in util_bug.h without
- * including stdlib in all the users of util_bug.h)
+ * including torerr.h in all the users of util_bug.h)
  **/
 void
 tor_abort_(void)
 {
-  abort();
+  tor_raw_abort_();
 }
 
 #ifdef _WIN32
diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h
index 2a4d68127e..7eaa6173c8 100644
--- a/src/lib/log/util_bug.h
+++ b/src/lib/log/util_bug.h
@@ -80,10 +80,10 @@
     tor__assert_tmp_value__;                    \
   } )
 #define ASSERT_PREDICT_LIKELY_(e) ASSERT_PREDICT_UNLIKELY_(e)
-#else
+#else /* !(defined(TOR_UNIT_TESTS) && defined(__GNUC__)) */
 #define ASSERT_PREDICT_UNLIKELY_(e) PREDICT_UNLIKELY(e)
 #define ASSERT_PREDICT_LIKELY_(e) PREDICT_LIKELY(e)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) && defined(__GNUC__) */
 
 /* Sometimes we don't want to use assertions during branch coverage tests; it
  * leads to tons of unreached branches which in reality are only assertions we
@@ -92,21 +92,28 @@
 #define tor_assert(a) STMT_BEGIN                                        \
   (void)(a);                                                            \
   STMT_END
-#else
+#define tor_assertf(a, fmt, ...) STMT_BEGIN                             \
+  (void)(a);                                                            \
+  (void)(fmt);                                                          \
+  STMT_END
+#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */
 /** Like assert(3), but send assertion failures to the log as well as to
  * stderr. */
-#define tor_assert(expr) STMT_BEGIN                                     \
+#define tor_assert(expr) tor_assertf(expr, NULL)
+
+#define tor_assertf(expr, fmt, ...) STMT_BEGIN                          \
   if (ASSERT_PREDICT_LIKELY_(expr)) {                                   \
   } else {                                                              \
-    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr);     \
-    tor_abort_();                                                       \
+    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr,      \
+                          fmt, ##__VA_ARGS__);                          \
+    tor_abort_();                                                        \
   } STMT_END
 #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */
 
 #define tor_assert_unreached()                                  \
   STMT_BEGIN {                                                  \
     tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__,     \
-                          "line should be unreached");          \
+                          "line should be unreached", NULL);    \
     tor_abort_();                                               \
   } STMT_END
 
@@ -136,34 +143,47 @@
 #ifdef ALL_BUGS_ARE_FATAL
 #define tor_assert_nonfatal_unreached() tor_assert(0)
 #define tor_assert_nonfatal(cond) tor_assert((cond))
+#define tor_assertf_nonfatal(cond, fmt, ...)    \
+  tor_assertf(cond, fmt, ##__VA_ARGS__)
 #define tor_assert_nonfatal_unreached_once() tor_assert(0)
 #define tor_assert_nonfatal_once(cond) tor_assert((cond))
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \
+   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",NULL), \
     tor_abort_(), 1)                                                    \
    : 0)
 #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS)
 #define tor_assert_nonfatal_unreached() STMT_NIL
 #define tor_assert_nonfatal(cond) ((void)(cond))
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  (void)cond;                                                           \
+  (void)fmt;                                                            \
+  STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_NIL
 #define tor_assert_nonfatal_once(cond) ((void)(cond))
 #define BUG(cond) (ASSERT_PREDICT_UNLIKELY_(cond) ? 1 : 0)
 #else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */
 #define tor_assert_nonfatal_unreached() STMT_BEGIN                      \
-  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0);         \
+  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0, NULL);   \
   STMT_END
 #define tor_assert_nonfatal(cond) STMT_BEGIN                            \
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else {                                                              \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, NULL);\
+  }                                                                     \
+  STMT_END
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  if (ASSERT_PREDICT_UNLIKELY_(cond)) {                                 \
+  } else {                                                              \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0,        \
+                      fmt, ##__VA_ARGS__);                               \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_BEGIN                 \
   static int warning_logged__ = 0;                                      \
   if (!warning_logged__) {                                              \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1);       \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1, NULL); \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_once(cond) STMT_BEGIN                       \
@@ -171,12 +191,12 @@
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else if (!warning_logged__) {                                       \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1, NULL);\
   }                                                                     \
   STMT_END
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \
+  (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0,NULL),1) \
    : 0)
 #endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
 
@@ -188,17 +208,17 @@
       if (bool_result && !var) {                                        \
         var = 1;                                                        \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                          "!("#cond")", 1);                             \
+                          "!("#cond")", 1, NULL);                       \
       }                                                                 \
       bool_result; } ))
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 #define IF_BUG_ONCE__(cond,var)                                         \
   static int var = 0;                                                   \
   if ((cond) ?                                                          \
       (var ? 1 :                                                        \
        (var=1,                                                          \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                           "!("#cond")", 1),                            \
+                           "!("#cond")", 1, NULL),                      \
         1))                                                             \
       : 0)
 #endif /* defined(__GNUC__) */
@@ -221,10 +241,13 @@
 #define tor_fragile_assert() tor_assert_nonfatal_unreached_once()
 
 void tor_assertion_failed_(const char *fname, unsigned int line,
-                           const char *func, const char *expr);
+                           const char *func, const char *expr,
+                           const char *fmt, ...)
+    CHECK_PRINTF(5,6);
 void tor_bug_occurred_(const char *fname, unsigned int line,
                        const char *func, const char *expr,
-                       int once);
+                       int once, const char *fmt, ...)
+  CHECK_PRINTF(6,7);
 
 void tor_abort_(void) ATTR_NORETURN;
 
diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c
index dc45cb4c3d..03d5c9fad2 100644
--- a/src/lib/log/win32err.c
+++ b/src/lib/log/win32err.c
@@ -47,7 +47,7 @@ format_win32_error(DWORD err)
     result = tor_malloc(len);
     wcstombs(result,str,len);
     result[len-1] = '\0';
-#else /* !(defined(UNICODE)) */
+#else /* !defined(UNICODE) */
     result = tor_strdup(str);
 #endif /* defined(UNICODE) */
   } else {
diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h
index 33413dfd15..ecfa88792d 100644
--- a/src/lib/log/win32err.h
+++ b/src/lib/log/win32err.h
@@ -19,4 +19,4 @@
 char *format_win32_error(DWORD err);
 #endif
 
-#endif
+#endif /* !defined(TOR_WIN32ERR_H) */
diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include
index cc62bb1013..7686bf862a 100644
--- a/src/lib/malloc/.may_include
+++ b/src/lib/malloc/.may_include
@@ -3,4 +3,4 @@ orconfig.h
 lib/cc/*.h
 lib/err/*.h
 lib/malloc/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am
index 502cc1c6b7..b74292bc6e 100644
--- a/src/lib/malloc/include.am
+++ b/src/lib/malloc/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-malloc-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_malloc_a_SOURCES =			\
-	src/lib/malloc/malloc.c
+	src/lib/malloc/malloc.c				\
+	src/lib/malloc/map_anon.c
 
 if USE_OPENBSD_MALLOC
 src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c
@@ -17,5 +19,7 @@ src_lib_libtor_malloc_testing_a_SOURCES = \
 src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/malloc/malloc.h
+	src/lib/malloc/malloc.h				\
+	src/lib/malloc/map_anon.h
diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h
index ef6b509ca4..39a45901a1 100644
--- a/src/lib/malloc/malloc.h
+++ b/src/lib/malloc/malloc.h
@@ -48,12 +48,12 @@ void tor_free_(void *mem);
     raw_free(*tor_free__tmpvar);                               \
     *tor_free__tmpvar=NULL;                                    \
   STMT_END
-#else
+#else /* !defined(__GNUC__) */
 #define tor_free(p) STMT_BEGIN                                 \
   raw_free(p);                                                 \
   (p)=NULL;                                                    \
   STMT_END
-#endif
+#endif /* defined(__GNUC__) */
 
 #define tor_malloc(size)       tor_malloc_(size)
 #define tor_malloc_zero(size)  tor_malloc_zero_(size)
diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c
new file mode 100644
index 0000000000..9559cbe2d4
--- /dev/null
+++ b/src/lib/malloc/map_anon.c
@@ -0,0 +1,271 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.c
+ * \brief Manage anonymous mappings.
+ **/
+
+#include "orconfig.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/malloc/malloc.h"
+#include "lib/err/torerr.h"
+
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_MACH_VM_INHERIT_H
+#include <mach/vm_inherit.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <string.h>
+#include <errno.h>
+
+/**
+ * Macro to get the high bytes of a size_t, if there are high bytes.
+ * Windows needs this; other operating systems define a size_t that does
+ * what it should.
+ */
+#if SIZEOF_SIZE_T > 4
+#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32)
+#else
+#define HIGH_SIZE_T_BYTES(sz) (0)
+#endif
+
+/* Here we define a MINHERIT macro that is minherit() or madvise(), depending
+ * on what we actually want.
+ *
+ * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO
+ * to be that flag.  If there's a flag unmaps pages after fork, we define
+ * FLAG_NOINHERIT to be that flag.
+ */
+#if defined(HAVE_MINHERIT)
+#define MINHERIT minherit
+
+#ifdef INHERIT_ZERO
+#define FLAG_ZERO INHERIT_ZERO
+#elif defined(MAP_INHERIT_ZERO)
+#define FLAG_ZERO MAP_INHERIT_ZERO
+#endif
+#ifdef INHERIT_NONE
+#define FLAG_NOINHERIT INHERIT_NONE
+#elif defined(VM_INHERIT_NONE)
+#define FLAG_NOINHERIT VM_INHERIT_NONE
+#elif defined(MAP_INHERIT_NONE)
+#define FLAG_NOINHERIT MAP_INHERIT_NONE
+#endif /* defined(INHERIT_NONE) || ... */
+
+#elif defined(HAVE_MADVISE)
+
+#define MINHERIT madvise
+
+#ifdef MADV_WIPEONFORK
+#define FLAG_ZERO MADV_WIPEONFORK
+#endif
+#ifdef MADV_DONTFORK
+#define FLAG_NOINHERIT MADV_DONTFORK
+#endif
+
+#endif /* defined(HAVE_MINHERIT) || ... */
+
+#if defined(HAVE_MINHERIT) && !defined(FLAG_ZERO) && !defined(FLAG_NOINHERIT)
+#warn "minherit() is defined, but we couldn't find the right flag for it."
+#warn "This is probably a bug in Tor's support for this platform."
+#endif
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped
+ * to disk.  Return 0 on success or if the facility is not available on this
+ * OS; return -1 on failure.
+ */
+static int
+lock_mem(void *mem, size_t sz)
+{
+#ifdef _WIN32
+  return VirtualLock(mem, sz) ? 0 : -1;
+#elif defined(HAVE_MLOCK)
+  return mlock(mem, sz);
+#else
+  (void) mem;
+  (void) sz;
+
+  return 0;
+#endif /* defined(_WIN32) || ... */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in
+ * a core dump.  Return 0 on success or if the facility is not available on
+ * this OS; return -1 on failure.
+ */
+static int
+nodump_mem(void *mem, size_t sz)
+{
+#if defined(MADV_DONTDUMP)
+  int rv = madvise(mem, sz, MADV_DONTDUMP);
+  if (rv == 0) {
+    return 0;
+  } else if (errno == ENOSYS || errno == EINVAL) {
+    return 0; // syscall not supported, or flag not supported.
+  } else {
+    tor_log_err_sigsafe("Unexpected error from madvise: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !defined(MADV_DONTDUMP) */
+  (void) mem;
+  (void) sz;
+  return 0;
+#endif /* defined(MADV_DONTDUMP) */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being
+ * accessible in child processes -- ideally by having them set to 0 after a
+ * fork, and if that doesn't work, by having them unmapped after a fork.
+ * Return 0 on success or if the facility is not available on this OS; return
+ * -1 on failure.
+ *
+ * If we successfully make the memory uninheritable, adjust the value of
+ * *<b>inherit_result_out</b>.
+ */
+static int
+noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out)
+{
+#ifdef FLAG_ZERO
+  int r = MINHERIT(mem, sz, FLAG_ZERO);
+  if (r == 0) {
+    *inherit_result_out = INHERIT_RES_ZERO;
+    return 0;
+  }
+#endif /* defined(FLAG_ZERO) */
+
+#ifdef FLAG_NOINHERIT
+  int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT);
+  if (r2 == 0) {
+    *inherit_result_out = INHERIT_RES_DROP;
+    return 0;
+  }
+#endif /* defined(FLAG_NOINHERIT) */
+
+#if defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)
+  /* At least one operation was tried, and neither succeeded. */
+
+  if (errno == ENOSYS || errno == EINVAL) {
+    /* Syscall not supported, or flag not supported. */
+    return 0;
+  } else {
+    tor_log_err_sigsafe("Unexpected error from minherit: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !(defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)) */
+  (void)inherit_result_out;
+  (void)mem;
+  (void)sz;
+  return 0;
+#endif /* defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) */
+}
+
+/**
+ * Return a new anonymous memory mapping that holds <b>sz</b> bytes.
+ *
+ * Memory mappings are unlike the results from malloc() in that they are
+ * handled separately by the operating system, and as such can have different
+ * kernel-level flags set on them.
+ *
+ * The "flags" argument may be zero or more of ANONMAP_PRIVATE and
+ * ANONMAP_NOINHERIT.
+ *
+ * Memory returned from this function must be released with
+ * tor_munmap_anonymous().
+ *
+ * If <b>inherit_result_out</b> is non-NULL, set it to one of
+ * INHERIT_RES_KEEP, INHERIT_RES_DROP, or INHERIT_RES_ZERO, depending on the
+ * properties of the returned memory.
+ *
+ * [Note: OS people use the word "anonymous" here to mean that the memory
+ * isn't associated with any file. This has *nothing* to do with the kind of
+ * anonymity that Tor is trying to provide.]
+ */
+void *
+tor_mmap_anonymous(size_t sz, unsigned flags,
+                   inherit_res_t *inherit_result_out)
+{
+  void *ptr;
+  inherit_res_t itmp=0;
+  if (inherit_result_out == NULL) {
+    inherit_result_out = &itmp;
+  }
+  *inherit_result_out = INHERIT_RES_KEEP;
+
+#if defined(_WIN32)
+  HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE,
+                                     NULL, /*attributes*/
+                                     PAGE_READWRITE,
+                                     HIGH_SIZE_T_BYTES(sz),
+                                     sz & 0xffffffff,
+                                     NULL /* name */);
+  raw_assert(mapping != NULL);
+  ptr = MapViewOfFile(mapping, FILE_MAP_WRITE,
+                      0, 0, /* Offset */
+                      0 /* Extend to end of mapping */);
+  raw_assert(ptr);
+  CloseHandle(mapping); /* mapped view holds a reference */
+#elif defined(HAVE_SYS_MMAN_H)
+  ptr = mmap(NULL, sz,
+             PROT_READ|PROT_WRITE,
+             MAP_ANON|MAP_PRIVATE,
+             -1, 0);
+  raw_assert(ptr != MAP_FAILED);
+  raw_assert(ptr != NULL);
+#else
+  ptr = tor_malloc_zero(sz);
+#endif /* defined(_WIN32) || ... */
+
+  if (flags & ANONMAP_PRIVATE) {
+    int lock_result = lock_mem(ptr, sz);
+    raw_assert(lock_result == 0);
+    int nodump_result = nodump_mem(ptr, sz);
+    raw_assert(nodump_result == 0);
+  }
+
+  if (flags & ANONMAP_NOINHERIT) {
+    int noinherit_result = noinherit_mem(ptr, sz, inherit_result_out);
+    raw_assert(noinherit_result == 0);
+  }
+
+  return ptr;
+}
+
+/**
+ * Release <b>sz</b> bytes of memory that were previously mapped at
+ * <b>mapping</b> by tor_mmap_anonymous().
+ **/
+void
+tor_munmap_anonymous(void *mapping, size_t sz)
+{
+  if (!mapping)
+    return;
+
+#if defined(_WIN32)
+  (void)sz;
+  UnmapViewOfFile(mapping);
+#elif defined(HAVE_SYS_MMAN_H)
+  munmap(mapping, sz);
+#else
+  (void)sz;
+  tor_free(mapping);
+#endif /* defined(_WIN32) || ... */
+}
diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h
new file mode 100644
index 0000000000..4c4690e12f
--- /dev/null
+++ b/src/lib/malloc/map_anon.h
@@ -0,0 +1,71 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.h
+ * \brief Headers for map_anon.c
+ **/
+
+#ifndef TOR_MAP_ANON_H
+#define TOR_MAP_ANON_H
+
+#include "lib/malloc/malloc.h"
+#include <stddef.h>
+
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * swapped or dumped.
+ *
+ * In some operating systems, this flag is not implemented.
+ */
+#define ANONMAP_PRIVATE   (1u<<0)
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * inherited after a fork().  In some operating systems, trying to access it
+ * afterwards will cause its contents to be zero.  In others, trying to access
+ * it afterwards will cause a crash.
+ *
+ * In some operating systems, this flag is not implemented at all.
+ */
+#define ANONMAP_NOINHERIT (1u<<1)
+
+typedef enum {
+  /** Possible value for inherit_result_out: the memory will be kept
+   * by any child process. */
+  INHERIT_RES_KEEP=0,
+  /** Possible value for inherit_result_out: the memory will be dropped in the
+   * child process. Attempting to access it will likely cause a segfault. */
+  INHERIT_RES_DROP,
+  /** Possible value for inherit_result_out: the memory will be cleared in
+   * the child process. */
+  INHERIT_RES_ZERO
+} inherit_res_t;
+
+/* Here we define the NOINHERIT_CAN_FAIL macro if and only if
+ * it's possible that ANONMAP_NOINHERIT might yield inheritable memory.
+ */
+#ifdef _WIN32
+/* Windows can't fork, so NOINHERIT is never needed. */
+#elif defined(HAVE_MINHERIT)
+/* minherit() will always have a working MAP_INHERIT_NONE or MAP_INHERIT_ZERO.
+ * NOINHERIT should always work.
+ */
+#elif defined(HAVE_MADVISE)
+/* madvise() sometimes has neither MADV_DONTFORK and MADV_WIPEONFORK.
+ * We need to be ready for the possibility it failed.
+ *
+ * (Linux added DONTFORK in 2.6.16 and WIPEONFORK in 4.14. If we someday
+ * require 2.6.16 or later, we can assume that DONTFORK will work.)
+ */
+#define NOINHERIT_CAN_FAIL
+#else
+#define NOINHERIT_CAN_FAIL
+#endif /* defined(_WIN32) || ... */
+
+void *tor_mmap_anonymous(size_t sz, unsigned flags,
+                         inherit_res_t *inherit_result_out);
+void tor_munmap_anonymous(void *mapping, size_t sz);
+
+#endif /* !defined(TOR_MAP_ANON_H) */
diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include
index 1fd26864dc..f8bc264a5f 100644
--- a/src/lib/math/.may_include
+++ b/src/lib/math/.may_include
@@ -3,3 +3,5 @@ orconfig.h
 lib/cc/*.h
 lib/log/*.h
 lib/math/*.h
+lib/testsupport/*.h
+lib/crypt_ops/*.h
diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c
index eafad358c3..49a2a6a2ca 100644
--- a/src/lib/math/fp.c
+++ b/src/lib/math/fp.c
@@ -75,7 +75,7 @@ clamp_double_to_int64(double number)
 */
 #define PROBLEMATIC_FLOAT_CONVERSION_WARNING
 DISABLE_GCC_WARNING(float-conversion)
-#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */
+#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */
 
 /*
   With clang 4.0 we apparently run into "double promotion" warnings here,
@@ -121,3 +121,23 @@ ENABLE_GCC_WARNING(double-promotion)
 ENABLE_GCC_WARNING(float-conversion)
 #endif
 }
+
+/* isinf() wrapper for tor */
+int
+tor_isinf(double x)
+{
+  /* Same as above, work around the "double promotion" warnings */
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+DISABLE_GCC_WARNING(float-conversion)
+#endif
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+DISABLE_GCC_WARNING(double-promotion)
+#endif
+  return isinf(x);
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+ENABLE_GCC_WARNING(double-promotion)
+#endif
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+ENABLE_GCC_WARNING(float-conversion)
+#endif
+}
diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h
index 6f07152e92..a73789c945 100644
--- a/src/lib/math/fp.h
+++ b/src/lib/math/fp.h
@@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST;
 long tor_lround(double d) ATTR_CONST;
 int64_t tor_llround(double d) ATTR_CONST;
 int64_t clamp_double_to_int64(double number);
+int tor_isinf(double x);
 
-#endif
+#endif /* !defined(TOR_FP_H) */
diff --git a/src/lib/math/include.am b/src/lib/math/include.am
index b088b3f3cc..b2ca280f47 100644
--- a/src/lib/math/include.am
+++ b/src/lib/math/include.am
@@ -5,16 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-math-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_math_a_SOURCES =	\
 		src/lib/math/fp.c		\
-		src/lib/math/laplace.c
-
+		src/lib/math/laplace.c 	\
+		src/lib/math/prob_distr.c
 
 src_lib_libtor_math_testing_a_SOURCES = \
 	$(src_lib_libtor_math_a_SOURCES)
 src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/math/fp.h		\
-		src/lib/math/laplace.h
+		src/lib/math/laplace.h  \
+		src/lib/math/prob_distr.h
diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h
index e8651e5197..02b0e890f0 100644
--- a/src/lib/math/laplace.h
+++ b/src/lib/math/laplace.h
@@ -19,4 +19,4 @@ int64_t sample_laplace_distribution(double mu, double b, double p);
 int64_t add_laplace_noise(int64_t signal, double random, double delta_f,
                           double epsilon);
 
-#endif
+#endif /* !defined(TOR_LAPLACE_H) */
diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c
new file mode 100644
index 0000000000..d44dc28265
--- /dev/null
+++ b/src/lib/math/prob_distr.c
@@ -0,0 +1,1688 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file prob_distr.c
+ *
+ * \brief
+ *  Implements various probability distributions.
+ *  Almost all code is courtesy of Riastradh.
+ *
+ * \details
+ * Here are some details that might help you understand this file:
+ *
+ * - Throughout this file, `eps' means the largest relative error of a
+ *   correctly rounded floating-point operation, which in binary64
+ *   floating-point arithmetic is 2^-53.  Here the relative error of a
+ *   true value x from a computed value y is |x - y|/|x|.  This
+ *   definition of epsilon is conventional for numerical analysts when
+ *   writing error analyses.  (If your libm doesn't provide correctly
+ *   rounded exp and log, their relative error is usually below 2*2^-53
+ *   and probably closer to 1.1*2^-53 instead.)
+ *
+ *   The C constant DBL_EPSILON is actually twice this, and should
+ *   perhaps rather be named ulp(1) -- that is, it is the distance from
+ *   1 to the next greater floating-point number, which is usually of
+ *   more interest to programmers and hardware engineers.
+ *
+ *   Since this file is concerned mainly with error bounds rather than
+ *   with low-level bit-hacking of floating-point numbers, we adopt the
+ *   numerical analysts' definition in the comments, though we do use
+ *   DBL_EPSILON in a handful of places where it is convenient to use
+ *   some function of eps = DBL_EPSILON/2 in a case analysis.
+ *
+ * - In various functions (e.g. sample_log_logistic()) we jump through hoops so
+ *   that we can use reals closer to 0 than closer to 1, since we achieve much
+ *   greater accuracy for floating point numbers near 0. In particular, we can
+ *   represent differences as small as 10^-300 for numbers near 0, but of no
+ *   less than 10^-16 for numbers near 1.
+ **/
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "lib/math/prob_distr.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/cc/ctassert.h"
+#include "lib/log/util_bug.h"
+
+#include <float.h>
+#include <math.h>
+#include <stddef.h>
+
+/** Declare a function that downcasts from a generic dist struct to the actual
+ *  subtype probablity distribution it represents. */
+#define DECLARE_PROB_DISTR_DOWNCAST_FN(name) \
+  static inline                           \
+  const struct name *                             \
+  dist_to_const_##name(const struct dist *obj) {    \
+    tor_assert(obj->ops == &name##_ops);            \
+    return SUBTYPE_P(obj, struct name, base);   \
+  }
+DECLARE_PROB_DISTR_DOWNCAST_FN(uniform)
+DECLARE_PROB_DISTR_DOWNCAST_FN(geometric)
+DECLARE_PROB_DISTR_DOWNCAST_FN(logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(log_logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(genpareto)
+DECLARE_PROB_DISTR_DOWNCAST_FN(weibull)
+
+/**
+ * Count number of one bits in 32-bit word.
+ */
+static unsigned
+bitcount32(uint32_t x)
+{
+
+  /* Count two-bit groups.  */
+  x -= (x >> 1) & UINT32_C(0x55555555);
+
+  /* Count four-bit groups.  */
+  x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333));
+
+  /* Count eight-bit groups.  */
+  x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f);
+
+  /* Sum all eight-bit groups, and extract the sum.  */
+  return (x * UINT32_C(0x01010101)) >> 24;
+}
+
+/**
+ * Count leading zeros in 32-bit word.
+ */
+static unsigned
+clz32(uint32_t x)
+{
+
+  /* Round up to a power of two.  */
+  x |= x >> 1;
+  x |= x >> 2;
+  x |= x >> 4;
+  x |= x >> 8;
+  x |= x >> 16;
+
+  /* Subtract count of one bits from 32.  */
+  return (32 - bitcount32(x));
+}
+
+/*
+ * Some lemmas that will be used throughout this file to prove various error
+ * bounds:
+ *
+ * Lemma 1.  If |d| <= 1/2, then 1/(1 + d) <= 2.
+ *
+ * Proof.  If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1.
+ * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2.  QED.
+ *
+ * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b,
+ * then b = a*(1 + e) for |e| <= 2|d' - d|.
+ *
+ * Proof.  |a - b|/|a|
+ *             = |a - a*(1 + d)/(1 + d')|/|a|
+ *             = |1 - (1 + d)/(1 + d')|
+ *             = |(1 + d' - 1 - d)/(1 + d')|
+ *             = |(d' - d)/(1 + d')|
+ *            <= 2|d' - d|, by Lemma 1,
+ *
+ * QED.
+ *
+ * Lemma 3.  For |d|, |d'| < 1/4,
+ *
+ *     |log((1 + d)/(1 + d'))| <= 4|d - d'|.
+ *
+ * Proof.  Write
+ *
+ *     log((1 + d)/(1 + d'))
+ *      = log(1 + (1 + d)/(1 + d') - 1)
+ *      = log(1 + (1 + d - 1 - d')/(1 + d')
+ *      = log(1 + (d - d')/(1 + d')).
+ *
+ * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor
+ * series of log(1 + x) converges absolutely for (d - d')/(1 + d'),
+ * and thus we have
+ *
+ *     |log(1 + (d - d')/(1 + d'))|
+ *      = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n|
+ *     <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n
+ *      = 1/(1 - |2(d' - d)|)
+ *     <= 4|d' - d|,
+ *
+ * QED.
+ *
+ * Lemma 4.  If 1/e <= 1 + x <= e, then
+ *
+ *     log(1 + (1 + d) x) = (1 + d') log(1 + x)
+ *
+ * for |d'| < 8|d|.
+ *
+ * Proof.  Write
+ *
+ *     log(1 + (1 + d) x)
+ *     = log(1 + x + x*d)
+ *     = log((1 + x) (1 + x + x*d)/(1 + x))
+ *     = log(1 + x) + log((1 + x + x*d)/(1 + x))
+ *     = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)).
+ *
+ * The relative error is bounded by
+ *
+ *     |log((1 + x + x*d)/(1 + x))/log(1 + x)|
+ *     <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3,
+ *      = 4|x*d|/|log(1 + x)|
+ *      < 8|d|,
+ *
+ * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2.  QED.
+ */
+
+/**
+ * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x).
+ * Maps a log-odds-space probability in [-\infty, +\infty] into a direct-space
+ * probability in [0,1].  Inverse of logit.
+ *
+ * Ill-conditioned for large x; the identity logistic(-x) = 1 -
+ * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may
+ * help to rearrange a computation.
+ *
+ * This implementation gives relative error bounded by 7 eps.
+ */
+STATIC double
+logistic(double x)
+{
+  if (x <= log(DBL_EPSILON/2)) {
+    /*
+     * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case
+     * we will approximate the logistic() function with e^x because the
+     * relative error is less than eps. Here is a calculation of the
+     * relative error between the logistic() function and e^x and a proof
+     * that it's less than eps:
+     *
+     *     |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)|
+     *     <= |1 - 1/(1 + e^x)|*|1 + e^x|
+     *      = |e^x/(1 + e^x)|*|1 + e^x|
+     *      = |e^x|
+     *     <= eps.
+     */
+    return exp(x); /* return e^x */
+  } else if (x <= -log(DBL_EPSILON/2)) {
+    /*
+     * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 +
+     * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we
+     * also have 0 < 1/(1 + e^{-x}) < 1.  Thus, if exp has
+     * relative error d0, + has relative error d1, and /
+     * has relative error d2, then we get
+     *
+     *     (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)]
+     *     = (1 + d0)/[1 + e^{-x} + d0 e^{-x}
+     *                     + d1 + d1 e^{-x} + d0 d1 e^{-x}]
+     *     = (1 + d0)/[(1 + e^{-x})
+     *                 * (1 + d0 e^{-x}/(1 + e^{-x})
+     *                      + d1/(1 + e^{-x})
+     *                      + d0 d1 e^{-x}/(1 + e^{-x}))].
+     *     = (1 + d0)/[(1 + e^{-x})(1 + d')]
+     *     = [1/(1 + e^{-x})] (1 + d0)/(1 + d')
+     *
+     * where
+     *
+     *     d' = d0 e^{-x}/(1 + e^{-x})
+     *          + d1/(1 + e^{-x})
+     *          + d0 d1 e^{-x}/(1 + e^{-x}).
+     *
+     * By Lemma 2 this relative error is bounded by
+     *
+     *     2|d0 - d'|
+     *      = 2|d0 - d0 e^{-x}/(1 + e^{-x})
+     *             - d1/(1 + e^{-x})
+     *             - d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})|
+     *             + 2|d1/(1 + e^{-x})|
+     *             + 2|d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1|
+     *     <= 4|d0| + 2|d1| + 2|d0 d1|
+     *     <= 6 eps + 2 eps^2.
+     */
+    return 1/(1 + exp(-x));
+  } else {
+    /*
+     * e^{-x} <= eps, so the relative error of 1 from 1/(1
+     * + e^{-x}) is
+     *
+     *     |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})|
+     *      = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})|
+     *      = |e^{-x}|
+     *     <= eps.
+     *
+     * This computation avoids an intermediate overflow
+     * exception, although the effect on the result is
+     * harmless.
+     *
+     * XXX Should maybe raise inexact here.
+     */
+    return 1;
+  }
+}
+
+/**
+ * Compute the logit function: log p/(1 - p).  Defined on [0,1].  Maps
+ * a direct-space probability in [0,1] to a log-odds-space probability
+ * in [-\infty, +\infty].  Inverse of logistic.
+ *
+ * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) =
+ * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help
+ * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)].
+ *
+ * This implementation gives relative error bounded by 10 eps.
+ */
+STATIC double
+logit(double p)
+{
+
+  /* logistic(-1) <= p <= logistic(+1) */
+  if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) {
+    /*
+     * For inputs near 1/2, we want to compute log1p(near
+     * 0) rather than log(near 1), so write this as:
+     *
+     * log(p/(1 - p)) = -log((1 - p)/p)
+     * = -log(1 + (1 - p)/p - 1)
+     * = -log(1 + (1 - p - p)/p)
+     * = -log(1 + (1 - 2p)/p).
+     *
+     * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point
+     * evaluation of 1 - 2p is exact; the only error arises
+     * from division and log1p.  First, note that if
+     * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies
+     * in the bounds of Lemma 4.
+     *
+     * If division has relative error d0 and log1p has
+     * relative error d1, the outcome is
+     *
+     *     -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p)
+     *     = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p)
+     *     = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p).
+     *
+     * where |d'| < 8|d0| by Lemma 4.  The relative error
+     * is then bounded by
+     *
+     *     |d1 + d' + d1 d'|
+     *     <= |d1| + 8|d0| + 8|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return -log1p((1 - 2*p)/p);
+  } else {
+    /*
+     * For inputs near 0, although 1 - p may be rounded to
+     * 1, it doesn't matter much because the magnitude of
+     * the result is so much larger.  For inputs near 1, we
+     * can compute 1 - p exactly, although the precision on
+     * the input is limited so we won't ever get more than
+     * about 700 for the output.
+     *
+     * If - has relative error d0, / has relative error d1,
+     * and log has relative error d2, then
+     *
+     *     (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)])
+     *     = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))]
+     *     = log(p/(1 - p)) + d2 log(p/(1 - p))
+     *       + (1 + d2) log((1 + d0)/(1 + d1))
+     *     = log(p/(1 - p))*[1 + d2 +
+     *         + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))]
+     *
+     * Since 0 <= p < logistic(-1) or logistic(+1) < p <=
+     * 1, we have |log(p/(1 - p))| > 1.  Hence this error
+     * is bounded by
+     *
+     *     |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))
+     *                      / log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))|
+     *     <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3,
+     *     <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0|
+     *     <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return log(p/(1 - p));
+  }
+}
+
+/**
+ * Compute the logit function, translated in input by 1/2: logithalf(p)
+ * = logit(1/2 + p).  Defined on [-1/2, 1/2].  Inverse of logistichalf.
+ *
+ * Ill-conditioned near +/-1/2.  If |p0| > 1/2 - 1/(1 + e), it may be
+ * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead.
+ * This implementation gives relative error bounded by 34 eps.
+ */
+STATIC double
+logithalf(double p0)
+{
+
+  if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) {
+    /*
+     * logit(1/2 + p0)
+     * = log((1/2 + p0)/(1 - (1/2 + p0)))
+     * = log((1/2 + p0)/(1/2 - p0))
+     * = log(1 + (1/2 + p0)/(1/2 - p0) - 1)
+     * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0))
+     * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0))
+     * = log(1 + 2 p0/(1/2 - p0))
+     *
+     * If the error of subtraction is d0, the error of
+     * division is d1, and the error of log1p is d2, then
+     * what we compute is
+     *
+     *  (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)])
+     *  = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0))
+     *  = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0))
+     *  = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)),
+     *
+     * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and
+     * |d''| < 8|d'| < 32 eps by Lemma 4 since
+     *
+     *  1/e <= 1 + 2*p0/(1/2 - p0) <= e
+     *
+     * when |p0| <= 1/2 - 1/(1 + e).  Hence the relative
+     * error is bounded by
+     *
+     *  |d2 + d'' + d2 d''|
+     *  <= |d2| + |d''| + |d2 d''|
+     *  <= |d1| + 32 |d0| + 32 |d1 d0|
+     *  <= 33 eps + 32 eps^2.
+     */
+    return log1p(2*p0/(0.5 - p0));
+  } else {
+    /*
+     * We have a choice of computing logit(1/2 + p0) or
+     * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0).  It
+     * doesn't matter which way we do this: either way,
+     * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference
+     * are computed exactly.  So let's do the one that
+     * skips the final negation.
+     *
+     * The result is
+     *
+     *  (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)])
+     *  = (1 + d1) (1 + log((1 + d0)/(1 + d2))
+     *                  / log((1/2 + p0)/(1/2 - p0)))
+     *    * log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') logit(1/2 + p0)
+     *
+     * where
+     *
+     *  d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0)
+     *       + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0).
+     *
+     * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1.
+     * Provided |d0|, |d2| < 1/4, by Lemma 3 we have
+     *
+     *  |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|.
+     *
+     * Hence the relative error is bounded by
+     *
+     *  |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2|
+     *       <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2|
+     *       <= 9 eps + 8 eps^2.
+     */
+    return log((0.5 + p0)/(0.5 - p0));
+  }
+}
+
+/*
+ * The following random_uniform_01 is tailored for IEEE 754 binary64
+ * floating-point or smaller.  It can be adapted to larger
+ * floating-point formats like i387 80-bit or IEEE 754 binary128, but
+ * it may require sampling more bits.
+ */
+CTASSERT(FLT_RADIX == 2);
+CTASSERT(-DBL_MIN_EXP <= 1021);
+CTASSERT(DBL_MANT_DIG <= 53);
+
+/**
+ * Draw a floating-point number in [0, 1] with uniform distribution.
+ *
+ * Note that the probability of returning 0 is less than 2^-1074, so
+ * callers need not check for it.  However, callers that cannot handle
+ * rounding to 1 must deal with that, because it occurs with
+ * probability 2^-54, which is small but nonnegligible.
+ */
+STATIC double
+random_uniform_01(void)
+{
+  uint32_t z, x, hi, lo;
+  double s;
+
+  /*
+   * Draw an exponent, geometrically distributed, but give up if
+   * we get a run of more than 1088 zeros, which really means the
+   * system is broken.
+   */
+  z = 0;
+  while ((x = crypto_fast_rng_get_u32(get_thread_fast_rng())) == 0) {
+    if (z >= 1088)
+      /* Your bit sampler is broken.  Go home.  */
+      return 0;
+    z += 32;
+  }
+  z += clz32(x);
+
+  /*
+   * Pick 32-bit halves of an odd normalized significand.
+   * Picking it odd breaks ties in the subsequent rounding, which
+   * occur only with measure zero in the uniform distribution on
+   * [0, 1].
+   */
+  hi = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x80000000);
+  lo = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x00000001);
+
+  /* Round to nearest scaled significand in [2^63, 2^64].  */
+  s = hi*(double)4294967296 + lo;
+
+  /* Rescale into [1/2, 1] and apply exponent in one swell foop.  */
+  return s * ldexp(1, -(64 + z));
+}
+
+/*******************************************************************/
+
+/* Functions for specific probability distributions start here: */
+
+/*
+ * Logistic(mu, sigma) distribution, supported on (-\infty,+\infty)
+ *
+ * This is the uniform distribution on [0,1] mapped into log-odds
+ * space, scaled by sigma and translated by mu.
+ *
+ * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2
+ * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma)
+ * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma)
+ * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p)
+ * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p)
+ */
+
+/**
+ * Compute the CDF of the Logistic(mu, sigma) distribution: the
+ * logistic function.  Well-conditioned for negative inputs and small
+ * positive inputs; ill-conditioned for large positive inputs.
+ */
+STATIC double
+cdf_logistic(double x, double mu, double sigma)
+{
+  return logistic((x - mu)/sigma);
+}
+
+/**
+ * Compute the SF of the Logistic(mu, sigma) distribution: the logistic
+ * function reflected over the y axis.  Well-conditioned for positive
+ * inputs and small negative inputs; ill-conditioned for large negative
+ * inputs.
+ */
+STATIC double
+sf_logistic(double x, double mu, double sigma)
+{
+  return logistic(-(x - mu)/sigma);
+}
+
+/**
+ * Compute the inverse of the CDF of the Logistic(mu, sigma)
+ * distribution: the logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+icdf_logistic(double p, double mu, double sigma)
+{
+  return mu + sigma*logit(p);
+}
+
+/**
+ * Compute the inverse of the SF of the Logistic(mu, sigma)
+ * distribution: the -logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+isf_logistic(double p, double mu, double sigma)
+{
+  return mu - sigma*logit(p);
+}
+
+/*
+ * LogLogistic(alpha, beta) distribution, supported on (0, +\infty).
+ *
+ * This is the uniform distribution on [0,1] mapped into odds space,
+ * scaled by positive alpha and shaped by positive beta.
+ *
+ * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample.
+ * (Name arises because the pdf has LogLogistic(x; alpha, beta) =
+ * Logistic(log x; log alpha, 1/beta) and mathematicians got their
+ * covariance contravariant.)
+ *
+ * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2
+ *        = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} /
+ *              (1 + (x/e^mu)^{1/sigma})^2
+ * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x}/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x - mu})^{-1/sigma})
+ *        = 1/(1 + e^{-(log x - mu)/sigma})
+ *        = logistic((log x - mu)/sigma)
+ *        = logistic((log x - log alpha)/(1/beta))
+ * sf(x) = 1 - 1/(1 + (x/alpha)^-beta)
+ *       = (x/alpha)^-beta/(1 + (x/alpha)^-beta)
+ *       = 1/((x/alpha)^beta + 1)
+ *       = 1/(1 + (x/alpha)^beta)
+ * icdf(p) = alpha (p/(1 - p))^{1/beta}
+ *         = alpha e^{logit(p)/beta}
+ *         = e^{mu + sigma logit(p)}
+ * isf(p) = alpha ((1 - p)/p)^{1/beta}
+ *        = alpha e^{-logit(p)/beta}
+ *        = e^{mu - sigma logit(p)}
+ */
+
+/**
+ * Compute the CDF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      -beta/[1 + (x/alpha)^{-beta}]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps),
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, around which point it levels off at 1.
+ */
+STATIC double
+cdf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and
+   * d3, of the final quotient.  The exponentiation gives
+   *
+   *    ((1 + d0) x/alpha)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + d0)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1)
+   *    = (x/alpha)^{-beta} (1 + d')
+   *
+   * where d' = (1 + d0)^{-beta} - 1.  If y = (x/alpha)^{-beta},
+   * the denominator is
+   *
+   *    (1 + d2) (1 + (1 + d1) (1 + d') y)
+   *    = (1 + d2) (1 + y + (d1 + d' + d1 d') y)
+   *    = 1 + y + (1 + d2) (d1 + d' + d1 d') y
+   *    = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y))
+   *    = (1 + y) (1 + d''),
+   *
+   * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y).  The
+   * final result is
+   *
+   *    (1 + d3) / [(1 + d2) (1 + d'') (1 + y)]
+   *    = (1 + d''') / (1 + y)
+   *
+   * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2
+   * (which may not be the case for very large beta).  This
+   * relative error is therefore bounded by
+   *
+   *    |d'''|
+   *    <= 2|d3 - d''|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')|
+   *     = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'|
+   *      <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'|
+   *         + 2|d2 d1 d'|
+   *      <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|.
+   *
+   * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps,
+   * until it levels off at 1.
+   */
+  return 1/(1 + pow(x/alpha, -beta));
+}
+
+/**
+ * Compute the SF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      beta/[1 + (x/alpha)^beta]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps)
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, beyond which point it grows unbounded.
+ */
+STATIC double
+sf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * The error analysis here is essentially the same as in
+   * cdf_log_logistic, except that rather than levelling off at
+   * 1, |(1 + d0)^beta - 1| grows unbounded.
+   */
+  return 1/(1 + pow(x/alpha, beta));
+}
+
+/**
+ * Compute the inverse of the CDF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and beta near 0 with
+ * condition number 1/[beta (1 - p)].
+ */
+STATIC double
+icdf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow(p/(1 - p), 1/beta);
+}
+
+/**
+ * Compute the inverse of the SF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and for large beta, with
+ * condition number -1/[beta (1 - p)].
+ */
+STATIC double
+isf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow((1 - p)/p, 1/beta);
+}
+
+/*
+ * Weibull(lambda, k) distribution, supported on (0, +\infty).
+ *
+ * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k}
+ * cdf(x) = 1 - e^{-(x/lambda)^k}
+ * icdf(p) = lambda * (-log (1 - p))^{1/k}
+ * sf(x) = e^{-(x/lambda)^k}
+ * isf(p) = lambda * (-log p)^{1/k}
+ */
+
+/**
+ * Compute the CDF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1]
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+cdf_weibull(double x, double lambda, double k)
+{
+  return -expm1(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the SF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+sf_weibull(double x, double lambda, double k)
+{
+  return exp(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the inverse of the CDF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 1, and for k near 0;
+ * condition number is
+ *
+ *      (p/(1 - p))/(k log(1 - p)).
+ */
+STATIC double
+icdf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log1p(-p), 1/k);
+}
+
+/**
+ * Compute the inverse of the SF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 0, and for k near 0;
+ * condition number is
+ *
+ *      1/(k log(p)).
+ */
+STATIC double
+isf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log(p), 1/k);
+}
+
+/*
+ * GeneralizedPareto(mu, sigma, xi), supported on (mu, +\infty) for
+ * nonnegative xi, or (mu, mu - sigma/xi) for negative xi.
+ *
+ * Samples:
+ * = mu - sigma log U, if xi = 0;
+ * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0,
+ * where U is uniform on (0,1].
+ * = mu + sigma (e^{xi X} - 1)/xi,
+ * where X has standard exponential distribution.
+ *
+ * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)}
+ * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi}
+ *        = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi}
+ *        --> 1 - e^{-(x - mu)/sigma}  as  xi --> 0
+ * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi}
+ *       --> e^{-(x - mu)/sigma}  as  xi --> 0
+ * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi
+ *         = mu + sigma*expm1(-xi log p)/xi
+ *         --> mu + sigma*log p  as  xi --> 0
+ * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi
+ *        = mu + sigma*expm1(-xi log1p(-p))/xi
+ *        --> mu + sigma*log1p(-p)  as  xi --> 0
+ */
+
+/**
+ * Compute the CDF of the GeneralizedPareto(mu, sigma, xi)
+ * distribution.  Well-conditioned everywhere.  For standard
+ * distribution (mu=0, sigma=1), condition number
+ *
+ *      (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1)
+ *
+ * is bounded by 1, attained only at x = 0.
+ */
+STATIC double
+cdf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  /*
+   * log(1 + xi x_0)/xi
+   * = (-1/xi) \sum_{n=1}^\infty (-xi x_0)^n/n
+   * = (-1/xi) (-xi x_0 + \sum_{n=2}^\infty (-xi x_0)^n/n)
+   * = x_0 - (1/xi) \sum_{n=2}^\infty (-xi x_0)^n/n
+   * = x_0 - x_0 \sum_{n=2}^\infty (-xi x_0)^{n-1}/n
+   * = x_0 (1 - d),
+   *
+   * where d = \sum_{n=2}^\infty (-xi x_0)^{n-1}/n.  If |xi| <
+   * eps/4|x_0|, then
+   *
+   * |d| <= \sum_{n=2}^\infty (eps/4)^{n-1}/n
+   *     <= \sum_{n=2}^\infty (eps/4)^{n-1}
+   *      = \sum_{n=1}^\infty (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^\infty (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2
+   *
+   * for any 0 < eps < 2.  Thus, the relative error of x_0 from
+   * log(1 + xi x_0)/xi is bounded by eps.
+   */
+  if (fabs(xi) < 1e-17/x_0)
+    return -expm1(-x_0);
+  else
+    return -expm1(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution.
+ * For standard distribution (mu=0, sigma=1), ill-conditioned for xi
+ * near 0; condition number
+ *
+ *      -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi}
+ *      = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi}
+ *      = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi}
+ *      = -x/(1 + x xi)
+ *
+ * is bounded by 1/xi.
+ */
+STATIC double
+sf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  if (fabs(xi) < 1e-17/x_0)
+    return exp(-x_0);
+  else
+    return exp(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; condition number is
+ *
+ *      xi (p/(1 - p))/(1 - (1 - p)^xi)
+ */
+STATIC double
+icdf_genpareto(double p, double mu, double sigma, double xi)
+{
+  /*
+   * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi
+   * for xi near zero (note f(xi) --> -log U as xi --> 0), write
+   * the absolutely convergent Taylor expansion
+   *
+   * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^\infty (-xi log U)^n/n!
+   *       = -log U + (1/xi)*\sum_{n=2}^\infty (-xi log U)^n/n!
+   *       = -log U + \sum_{n=2}^\infty xi^{n-1} (-log U)^n/n!
+   *       = -log U - log U \sum_{n=2}^\infty (-xi log U)^{n-1}/n!
+   *       = -log U (1 + \sum_{n=2}^\infty (-xi log U)^{n-1}/n!).
+   *
+   * Let d = \sum_{n=2}^\infty (-xi log U)^{n-1}/n!.  What do we
+   * lose if we discard it and use -log U as an approximation to
+   * f(xi)?  If |xi| < eps/-4log U, then
+   *
+   * |d| <= \sum_{n=2}^\infty |xi log U|^{n-1}/n!
+   *     <= \sum_{n=2}^\infty (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^\infty (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^\infty (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/-2log U,
+   * f(xi) = -log U (1 + d) for |d| <= eps/2.  |d| is the
+   * relative error of f(xi) from -log U; from this bound, the
+   * relative error of -log U from f(xi) is at most (eps/2)/(1 -
+   * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 <
+   * eps < 1.  Since -log U < 1000 for all U in (0, 1] in
+   * binary64 floating-point, we can safely cut xi off at 1e-20 <
+   * eps/4000 and attain <1ulp error from series truncation.
+   */
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log1p(-p);
+  else
+    return mu + sigma*expm1(-xi*log1p(-p))/xi;
+}
+
+/**
+ * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; conditon number is
+ *
+ *      -xi/(1 - p^{-xi})
+ */
+STATIC double
+isf_genpareto(double p, double mu, double sigma, double xi)
+{
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log(p);
+  else
+    return mu + sigma*expm1(-xi*log(p))/xi;
+}
+
+/*******************************************************************/
+
+/**
+ * Deterministic samplers, parametrized by uniform integer and (0,1]
+ * samples.  No guarantees are made about _which_ mapping from the
+ * integer and (0,1] samples these use; all that is guaranteed is the
+ * distribution of the outputs conditioned on a uniform distribution on
+ * the inputs.  The automatic tests in test_prob_distr.c double-check
+ * the particular mappings we use.
+ *
+ * Beware: Unlike random_uniform_01(), these are not guaranteed to be
+ * supported on all possible outputs.  See Ilya Mironov, `On the
+ * Significance of the Least Significant Bits for Differential
+ * Privacy', for an example of what can go wrong if you try to use
+ * these to conceal information from an adversary but you expose the
+ * specific full-precision floating-point values.
+ *
+ * Note: None of these samplers use rejection sampling; they are all
+ * essentially inverse-CDF transforms with tweaks.  If you were to add,
+ * say, a Gamma sampler with the Marsaglia-Tsang method, you would have
+ * to parametrize it by a potentially infinite stream of uniform (and
+ * perhaps normal) samples rather than a fixed number, which doesn't
+ * make for quite as nice automatic testing as for these.
+ */
+
+/**
+ * Deterministically sample from the interval [a, b], indexed by a
+ * uniform random floating-point number p0 in (0, 1].
+ *
+ * Note that even if p0 is nonzero, the result may be equal to a, if
+ * ulp(a)/2 is nonnegligible, e.g. if a = 1.  For maximum resolution,
+ * arrange |a| <= |b|.
+ */
+STATIC double
+sample_uniform_interval(double p0, double a, double b)
+{
+  /*
+   * XXX Prove that the distribution is, in fact, uniform on
+   * [a,b], particularly around p0 = 1, or at least has very
+   * small deviation from uniform, quantified appropriately
+   * (e.g., like in Monahan 1984, or by KL divergence).  It
+   * almost certainly does but it would be nice to quantify the
+   * error.
+   */
+  if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) {
+    /*
+     * When ab < 0, (1 - t) a + t b is monotonic, since for
+     * a <= b it is a sum of nondecreasing functions of t,
+     * and for b <= a, of nonincreasing functions of t.
+     * Further, clearly at 0 and 1 it attains a and b,
+     * respectively.  Hence it is bounded within [a, b].
+     */
+    return (1 - p0)*a + p0*b;
+  } else {
+    /*
+     * a + (b - a) t is monotonic -- it is obviously a
+     * nondecreasing function of t for a <= b.  Further, it
+     * attains a at 0, and while it may overshoot b at 1,
+     * we have a
+     *
+     * Theorem.  If 0 <= t < 1, then the floating-point
+     * evaluation of a + (b - a) t is bounded in [a, b].
+     *
+     * Lemma 1.  If 0 <= t < 1 is a floating-point number,
+     * then for any normal floating-point number x except
+     * the smallest in magnitude, |round(x*t)| < |x|.
+     *
+     * Proof.  WLOG, assume x >= 0.  Since the rounding
+     * function and t |---> x*t are nondecreasing, their
+     * composition t |---> round(x*t) is also
+     * nondecreasing, so it suffices to consider the
+     * largest floating-point number below 1, in particular
+     * t = 1 - ulp(1)/2.
+     *
+     * Case I: If x is a power of two, then the next
+     * floating-point number below x is x - ulp(x)/2 = x -
+     * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t
+     * is a floating-point number, multiplication is exact,
+     * and thus round(x*t) = x*t < x.
+     *
+     * Case II: If x is not a power of two, then the
+     * greatest lower bound of real numbers rounded to x is
+     * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2,
+     * where T(X) is the largest power of two below x.
+     * Anything below this bound is rounded to a
+     * floating-point number smaller than x, and x*t = x*(1
+     * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2
+     * since T(x) < x, so round(x*t) < x*t < x.  QED.
+     *
+     * Lemma 2.  If x and y are subnormal, then round(x +
+     * y) = x + y.
+     *
+     * Proof.  It is a matter of adding the significands,
+     * since if we treat subnormals as having an implicit
+     * zero bit before the `binary' point, their exponents
+     * are all the same.  There is at most one carry/borrow
+     * bit, which can always be acommodated either in a
+     * subnormal, or, at largest, in the implicit one bit
+     * of a normal.
+     *
+     * Lemma 3.  Let x and y be floating-point numbers.  If
+     * round(x - y) is subnormal or zero, then it is equal
+     * to x - y.
+     *
+     * Proof.  Case I (equal): round(x - y) = 0 iff x = y;
+     * hence if round(x - y) = 0, then round(x - y) = 0 = x
+     * - y.
+     *
+     * Case II (subnormal/subnormal): If x and y are both
+     * subnormal, this follows directly from Lemma 2.
+     *
+     * Case IIIa (normal/subnormal): If x is normal and y
+     * is subnormal, then x and y must share sign, or else
+     * x - y would be larger than x and thus rounded to
+     * normal.  If s is the smallest normal positive
+     * floating-point number, |x| < 2s since by
+     * construction 2s - |y| is normal for all subnormal y.
+     * This means that x and y must have the same exponent,
+     * so the difference is the difference of significands,
+     * which is exact.
+     *
+     * Case IIIb (subnormal/normal): Same as case IIIa for
+     * -(y - x).
+     *
+     * Case IV (normal/normal): If x and y are both normal,
+     * then they must share sign, or else x - y would be
+     * larger than x and thus rounded to normal.  Note that
+     * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <=
+     * -|x| but -|x| is normal like x.  Also, |x|/2 < |y|:
+     * if |x|/2 is subnormal, it must hold because y is
+     * normal; if |x|/2 is normal, then |x|/2 >= s, so
+     * since |x| - |y| < s,
+     *
+     *  |x|/2 = |x| - |x|/2 <= |x| - s <= |y|;
+     *
+     * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz
+     * lemma, round(x - y) = x - y.  QED.
+     *
+     * Proof of theorem.  WLOG, assume 0 <= a <= b.  Since
+     * round(a + round(round(b - a)*t) is nondecreasing in
+     * t and attains a at 0, the lower end of the bound is
+     * trivial; we must show the upper end of the bound
+     * strictly.  It suffices to show this for the largest
+     * floating-point number below 1, namely 1 - ulp(1)/2.
+     *
+     * Case I: round(b - a) is normal.  Then it is at most
+     * the smallest floating-point number above b - a.  By
+     * Lemma 1, round(round(b - a)*t) < round(b - a).
+     * Since the inequality is strict, and since
+     * round(round(b - a)*t) is a floating-point number
+     * below round(b - a), and since there are no
+     * floating-point numbers between b - a and round(b -
+     * a), we must have round(round(b - a)*t) < b - a.
+     * Then since y |---> round(a + y) is nondecreasing, we
+     * must have
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b) = b.
+     *
+     * Case II: round(b - a) is subnormal.  In this case,
+     * Lemma 1 falls apart -- we are not guaranteed the
+     * strict inequality.  However, by Lemma 3, the
+     * difference is exact: round(b - a) = b - a.  Thus,
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + round((b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b)
+     *   = b,
+     *
+     * QED.
+     */
+
+    /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal.  */
+    if (p0 >= 1)
+      return b;
+    return a + (b - a)*p0;
+  }
+}
+
+/**
+ * Deterministically sample from the standard logistic distribution,
+ * indexed by a uniform random 32-bit integer s and uniform random
+ * floating-point numbers t and p0 in (0, 1].
+ */
+STATIC double
+sample_logistic(uint32_t s, double t, double p0)
+{
+  double sign = (s & 1) ? -1 : +1;
+  double r;
+
+  /*
+   * We carve up the interval (0, 1) into subregions to compute
+   * the inverse CDF precisely:
+   *
+   * A = (0, 1/(1 + e)] ---> (-\infty, -1]
+   * B = [1/(1 + e), 1/2] ---> [-1, 0]
+   * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1]
+   * D = [1 - 1/(1 + e), 1) ---> [1, +\infty)
+   *
+   * Cases D and C are mirror images of cases A and B,
+   * respectively, so we choose between them by the sign chosen
+   * by a fair coin toss.  We choose between cases A and B by a
+   * coin toss weighted by
+   *
+   *    2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2):
+   *
+   * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)]
+   * sample p; if it comes up tails, scale p0 into a uniform (0,
+   * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p =
+   * 1/2 - p0.
+   */
+  if (t <= 2/(1 + exp(1))) {
+    /* p uniform in (0, 1/(1 + e)], represented by p.  */
+    p0 /= 1 + exp(1);
+    r = logit(p0);
+  } else {
+    /*
+     * p uniform in [1/(1 + e), 1/2), actually represented
+     * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so
+     * that p = 1/2 - p.
+     */
+    p0 *= 0.5 - 1/(1 + exp(1));
+    r = logithalf(p0);
+  }
+
+  /*
+   * We have chosen from the negative half of the standard
+   * logistic distribution, which is symmetric with the positive
+   * half.  Now use the sign to choose uniformly between them.
+   */
+  return sign*r;
+}
+
+/**
+ * Deterministically sample from the logistic distribution scaled by
+ * sigma and translated by mu.
+ */
+static double
+sample_logistic_locscale(uint32_t s, double t, double p0, double mu,
+    double sigma)
+{
+
+  return mu + sigma*sample_logistic(s, t, p0);
+}
+
+/**
+ * Deterministically sample from the standard log-logistic
+ * distribution, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_log_logistic(uint32_t s, double p0)
+{
+
+  /*
+   * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the
+   * condition numbers of the icdf and the isf coincide at 1/2.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0) {
+    /* p = p0 in (0, 1/2] */
+    return p0/(1 - p0);
+  } else {
+    /* p = 1 - p0 in [1/2, 1) */
+    return (1 - p0)/p0;
+  }
+}
+
+/**
+ * Deterministically sample from the log-logistic distribution with
+ * scale alpha and shape beta.
+ */
+static double
+sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha,
+    double beta)
+{
+  double x = sample_log_logistic(s, p0);
+
+  return alpha*pow(x, 1/beta);
+}
+
+/**
+ * Deterministically sample from the standard exponential distribution,
+ * indexed by a uniform random 32-bit integer s and a uniform random
+ * floating-point number p0 in (0, 1].
+ */
+static double
+sample_exponential(uint32_t s, double p0)
+{
+  /*
+   * We would like to evaluate log(p) for p near 0, and log1p(-p)
+   * for p near 1.  Simply carve the interval into (0, 1/2] and
+   * [1/2, 1) by a fair coin toss.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0)
+    /* p = p0 in (0, 1/2] */
+    return -log(p0);
+  else
+    /* p = 1 - p0 in [1/2, 1) */
+    return -log1p(-p0);
+}
+
+/**
+ * Deterministically sample from a Weibull distribution with scale
+ * lambda and shape k -- just an exponential with a shape parameter in
+ * addition to a scale parameter.  (Yes, lambda really is the scale,
+ * _not_ the rate.)
+ */
+STATIC double
+sample_weibull(uint32_t s, double p0, double lambda, double k)
+{
+
+  return lambda*pow(sample_exponential(s, p0), 1/k);
+}
+
+/**
+ * Deterministically sample from the generalized Pareto distribution
+ * with shape xi, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_genpareto(uint32_t s, double p0, double xi)
+{
+  double x = sample_exponential(s, p0);
+
+  /*
+   * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the
+   * absolutely convergent Taylor series
+   *
+   * f(x) = (1/xi) (xi x + \sum_{n=2}^\infty (xi x)^n/n!)
+   *      = x + (1/xi) \sum_{n=2}^\inty (xi x)^n/n!
+   *      = x + \sum_{n=2}^\infty xi^{n-1} x^n/n!
+   *      = x + x \sum_{n=2}^\infty (xi x)^{n-1}/n!
+   *      = x (1 + \sum_{n=2}^\infty (xi x)^{n-1}/n!).
+   *
+   * d = \sum_{n=2}^\infty (xi x)^{n-1}/n! is the relative error
+   * of f(x) from x.  If |xi| < eps/4x, then
+   *
+   * |d| <= \sum_{n=2}^\infty |xi x|^{n-1}/n!
+   *     <= \sum_{n=2}^\infty (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^\infty (eps/4)
+   *      = (eps/4) \sum_{n=0}^\infty (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/2x, f(xi)
+   * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'|
+   * <= eps.  What bound should we use for x?
+   *
+   * - If x is exponentially distributed, x > 200 with
+   *   probability below e^{-200} << 2^{-256}, i.e. never.
+   *
+   * - If x is computed by -log(U) for U in (0, 1], x is
+   *   guaranteed to be below 1000 in IEEE 754 binary64
+   *   floating-point.
+   *
+   * We can safely cut xi off at 1e-20 < eps/4000 and attain an
+   * error bounded by 0.5 ulp for this expression.
+   */
+  return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi);
+}
+
+/**
+ * Deterministically sample from a generalized Pareto distribution with
+ * shape xi, scaled by sigma and translated by mu.
+ */
+static double
+sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma,
+    double xi)
+{
+
+  return mu + sigma*sample_genpareto(s, p0, xi);
+}
+
+/**
+ * Deterministically sample from the geometric distribution with
+ * per-trial success probability p.
+ *
+ * XXX Quantify the error (KL divergence?) of this
+ * ceiling-of-exponential sampler from a true geometric distribution,
+ * which we could get by rejection sampling.  Relevant papers:
+ *
+ *      John F. Monahan, `Accuracy in Random Number Generation',
+ *      Mathematics of Computation 45(172), October 1984, pp. 559--568.
+*https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf
+ *
+ *      Karl Bringmann and Tobias Friedrich, `Exact and Efficient
+ *      Generation of Geometric Random Variates and Random Graphs', in
+ *      Proceedings of the 40th International Colloaquium on Automata,
+ *      Languages, and Programming -- ICALP 2013, Springer LNCS 7965,
+ *      pp.267--278.
+ *      https://doi.org/10.1007/978-3-642-39206-1_23
+ *      https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf
+ */
+static double
+sample_geometric(uint32_t s, double p0, double p)
+{
+  double x = sample_exponential(s, p0);
+
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1)
+    return 1;
+
+  return ceil(-x/log1p(-p));
+}
+
+/*******************************************************************/
+
+/** Public API for probability distributions:
+ *
+ *  These are wrapper functions on top of the various probability distribution
+ *  operations using the generic <b>dist</b> structure.
+
+ *  These are the functions that should be used by consumers of this API.
+ */
+
+/** Returns the name of the distribution in <b>dist</b>. */
+const char *
+dist_name(const struct dist *dist)
+{
+  return dist->ops->name;
+}
+
+/* Sample a value from <b>dist</b> and return it. */
+double
+dist_sample(const struct dist *dist)
+{
+  return dist->ops->sample(dist);
+}
+
+/** Compute the CDF of <b>dist</b> at <b>x</b>. */
+double
+dist_cdf(const struct dist *dist, double x)
+{
+  return dist->ops->cdf(dist, x);
+}
+
+/** Compute the SF (Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_sf(const struct dist *dist, double x)
+{
+  return dist->ops->sf(dist, x);
+}
+
+/** Compute the iCDF (Inverse CDF) of <b>dist</b> at <b>x</b>. */
+double
+dist_icdf(const struct dist *dist, double p)
+{
+  return dist->ops->icdf(dist, p);
+}
+
+/** Compute the iSF (Inverse Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_isf(const struct dist *dist, double p)
+{
+  return dist->ops->isf(dist, p);
+}
+
+/** Functions for uniform distribution */
+
+static double
+uniform_sample(const struct dist *dist)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  double p0 = random_uniform_01();
+
+  return sample_uniform_interval(p0, U->a, U->b);
+}
+
+static double
+uniform_cdf(const struct dist *dist, double x)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  if (x < U->a)
+    return 0;
+  else if (x < U->b)
+    return (x - U->a)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_sf(const struct dist *dist, double x)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+
+  if (x > U->b)
+    return 0;
+  else if (x > U->a)
+    return (U->b - x)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_icdf(const struct dist *dist, double p)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p)));
+}
+
+static double
+uniform_isf(const struct dist *dist, double p)
+{
+  const struct uniform *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p)));
+}
+
+const struct dist_ops uniform_ops = {
+  .name = "uniform",
+  .sample = uniform_sample,
+  .cdf = uniform_cdf,
+  .sf = uniform_sf,
+  .icdf = uniform_icdf,
+  .isf = uniform_isf,
+};
+
+/*******************************************************************/
+
+/** Private functions for each probability distribution. */
+
+/** Functions for logistic distribution: */
+
+static double
+logistic_sample(const struct dist *dist)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double t = random_uniform_01();
+  double p0 = random_uniform_01();
+
+  return sample_logistic_locscale(s, t, p0, L->mu, L->sigma);
+}
+
+static double
+logistic_cdf(const struct dist *dist, double x)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return cdf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_sf(const struct dist *dist, double x)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return sf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_icdf(const struct dist *dist, double p)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return icdf_logistic(p, L->mu, L->sigma);
+}
+
+static double
+logistic_isf(const struct dist *dist, double p)
+{
+  const struct logistic *L = dist_to_const_logistic(dist);
+  return isf_logistic(p, L->mu, L->sigma);
+}
+
+const struct dist_ops logistic_ops = {
+  .name = "logistic",
+  .sample = logistic_sample,
+  .cdf = logistic_cdf,
+  .sf = logistic_sf,
+  .icdf = logistic_icdf,
+  .isf = logistic_isf,
+};
+
+/** Functions for log-logistic distribution: */
+
+static double
+log_logistic_sample(const struct dist *dist)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_cdf(const struct dist *dist, double x)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return cdf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_sf(const struct dist *dist, double x)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return sf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_icdf(const struct dist *dist, double p)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return icdf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_isf(const struct dist *dist, double p)
+{
+  const struct log_logistic *LL = dist_to_const_log_logistic(dist);
+  return isf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+const struct dist_ops log_logistic_ops = {
+  .name = "log logistic",
+  .sample = log_logistic_sample,
+  .cdf = log_logistic_cdf,
+  .sf = log_logistic_sf,
+  .icdf = log_logistic_icdf,
+  .isf = log_logistic_isf,
+};
+
+/** Functions for Weibull distribution */
+
+static double
+weibull_sample(const struct dist *dist)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_weibull(s, p0, W->lambda, W->k);
+}
+
+static double
+weibull_cdf(const struct dist *dist, double x)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return cdf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_sf(const struct dist *dist, double x)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return sf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_icdf(const struct dist *dist, double p)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return icdf_weibull(p, W->lambda, W->k);
+}
+
+static double
+weibull_isf(const struct dist *dist, double p)
+{
+  const struct weibull *W = dist_to_const_weibull(dist);
+  return isf_weibull(p, W->lambda, W->k);
+}
+
+const struct dist_ops weibull_ops = {
+  .name = "Weibull",
+  .sample = weibull_sample,
+  .cdf = weibull_cdf,
+  .sf = weibull_sf,
+  .icdf = weibull_icdf,
+  .isf = weibull_isf,
+};
+
+/** Functions for generalized Pareto distributions */
+
+static double
+genpareto_sample(const struct dist *dist)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_cdf(const struct dist *dist, double x)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_sf(const struct dist *dist, double x)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return sf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_icdf(const struct dist *dist, double p)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_isf(const struct dist *dist, double p)
+{
+  const struct genpareto *GP = dist_to_const_genpareto(dist);
+  return isf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+const struct dist_ops genpareto_ops = {
+  .name = "generalized Pareto",
+  .sample = genpareto_sample,
+  .cdf = genpareto_cdf,
+  .sf = genpareto_sf,
+  .icdf = genpareto_icdf,
+  .isf = genpareto_isf,
+};
+
+/** Functions for geometric distribution on number of trials before success */
+
+static double
+geometric_sample(const struct dist *dist)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_geometric(s, p0, G->p);
+}
+
+static double
+geometric_cdf(const struct dist *dist, double x)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */
+  return -expm1(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_sf(const struct dist *dist, double x)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */
+  return exp(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_icdf(const struct dist *dist, double p)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  return log1p(-p)/log1p(-G->p);
+}
+
+static double
+geometric_isf(const struct dist *dist, double p)
+{
+  const struct geometric *G = dist_to_const_geometric(dist);
+
+  return log(p)/log1p(-G->p);
+}
+
+const struct dist_ops geometric_ops = {
+  .name = "geometric (1-based)",
+  .sample = geometric_sample,
+  .cdf = geometric_cdf,
+  .sf = geometric_sf,
+  .icdf = geometric_icdf,
+  .isf = geometric_isf,
+};
diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h
new file mode 100644
index 0000000000..7254dc8623
--- /dev/null
+++ b/src/lib/math/prob_distr.h
@@ -0,0 +1,253 @@
+
+/**
+ * \file prob_distr.h
+ *
+ * \brief Header for prob_distr.c
+ **/
+
+#ifndef TOR_PROB_DISTR_H
+#define TOR_PROB_DISTR_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * Container for distribution parameters for sampling, CDF, &c.
+ */
+struct dist {
+  const struct dist_ops *ops;
+};
+
+/**
+ * Untyped initializer element for struct dist using the specified
+ * struct dist_ops pointer.  Don't actually use this directly -- use
+ * the type-specific macro built out of DIST_BASE_TYPED below -- but if
+ * you did use this directly, it would be something like:
+ *
+ *   struct weibull mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     .lambda = ...,
+ *     .k = ...,
+ *   };
+ *
+ * Note there is NO COMPILER FEEDBACK if you accidentally do something
+ * like
+ *
+ *   struct geometric mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     ...
+ *   };
+ */
+#define DIST_BASE(OPS)  { .ops = (OPS) }
+
+/** A compile-time type-checking macro for use with DIST_BASE_TYPED.
+ *
+ *  This macro works by checking that &OBJ is a pointer type that is the same
+ *  type (except for qualifiers) as (const TYPE *)&OBJ. It's a C constraint
+ *  violation (which requires a diagnostic) if two pointers are different types
+ *  and are subtracted. The sizeof() forces compile-time evaluation, and the
+ *  multiplication by zero is to discard the result of the sizeof() from the
+ *  expression.
+ *
+ *  We define this conditionally to suppress false positives from
+ *  Coverity, which gets confused by the sizeof business.
+ */
+#ifdef __COVERITY__
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0
+#else
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \
+  (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ)))
+#endif /* defined(__COVERITY__) */
+
+/**
+* Typed initializer element for struct dist using the specified struct
+* dist_ops pointer.  Don't actually use this directly -- use a
+* type-specific macro built out of it -- but if you did use this
+* directly, it would be something like:
+*
+*   struct weibull mydist = {
+*     DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull),
+*     .lambda = ...,
+*     .k = ...,
+*   };
+*
+* If you want to define a distribution type, define a canonical set of
+* operations and define a type-specific initializer element like so:
+*
+*   struct foo {
+*     struct dist base;
+*     int omega;
+*     double tau;
+*     double phi;
+*   };
+*
+*   struct dist_ops foo_ops = ...;
+*
+*   #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo)
+*
+* Then users can do:
+*
+*   struct foo mydist = {
+*     FOO(mydist),
+*     .omega = ...,
+*     .tau = ...,
+*     .phi = ...,
+*   };
+*
+* If you accidentally write
+*
+*   struct bar mydist = {
+*     FOO(mydist),
+*     ...
+*   };
+*
+* then the compiler will report a type mismatch in the sizeof
+* expression, which otherwise evaporates at runtime.
+*/
+#define DIST_BASE_TYPED(OPS, OBJ, TYPE)                         \
+  DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE))
+
+/**
+ * Generic operations on distributions.  These simply defer to the
+ * corresponding dist_ops function.  In the parlance of C++, these call
+ * virtual member functions.
+ */
+const char *dist_name(const struct dist *);
+double dist_sample(const struct dist *);
+double dist_cdf(const struct dist *, double x);
+double dist_sf(const struct dist *, double x);
+double dist_icdf(const struct dist *, double p);
+double dist_isf(const struct dist *, double p);
+
+/**
+ * Set of operations on a potentially parametric family of
+ * distributions.  In the parlance of C++, this would be called a
+ * `vtable' and the members are virtual member functions.
+ */
+struct dist_ops {
+  const char *name;
+  double (*sample)(const struct dist *);
+  double (*cdf)(const struct dist *, double x);
+  double (*sf)(const struct dist *, double x);
+  double (*icdf)(const struct dist *, double p);
+  double (*isf)(const struct dist *, double p);
+};
+
+/* Geometric distribution on positive number of trials before first success */
+
+struct geometric {
+  struct dist base;
+  double p; /* success probability */
+};
+
+extern const struct dist_ops geometric_ops;
+
+#define GEOMETRIC(OBJ)                                      \
+  DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric)
+
+/* Pareto distribution */
+
+struct genpareto {
+  struct dist base;
+  double mu;
+  double sigma;
+  double xi;
+};
+
+extern const struct dist_ops genpareto_ops;
+
+#define GENPARETO(OBJ)                                      \
+  DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto)
+
+/* Weibull distribution */
+
+struct weibull {
+  struct dist base;
+  double lambda;
+  double k;
+};
+
+extern const struct dist_ops weibull_ops;
+
+#define WEIBULL(OBJ)                                    \
+  DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull)
+
+/* Log-logistic distribution */
+
+struct log_logistic {
+  struct dist base;
+  double alpha;
+  double beta;
+};
+
+extern const struct dist_ops log_logistic_ops;
+
+#define LOG_LOGISTIC(OBJ)                                       \
+  DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic)
+
+/* Logistic distribution */
+
+struct logistic {
+  struct dist base;
+  double mu;
+  double sigma;
+};
+
+extern const struct dist_ops logistic_ops;
+
+#define LOGISTIC(OBJ)                                   \
+  DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic)
+
+/* Uniform distribution */
+
+struct uniform {
+  struct dist base;
+  double a;
+  double b;
+};
+
+extern const struct dist_ops uniform_ops;
+
+#define UNIFORM(OBJ)                                    \
+  DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform)
+
+/** Only by unittests */
+
+#ifdef PROB_DISTR_PRIVATE
+
+STATIC double logithalf(double p0);
+STATIC double logit(double p);
+
+STATIC double random_uniform_01(void);
+
+STATIC double logistic(double x);
+STATIC double cdf_logistic(double x, double mu, double sigma);
+STATIC double sf_logistic(double x, double mu, double sigma);
+STATIC double icdf_logistic(double p, double mu, double sigma);
+STATIC double isf_logistic(double p, double mu, double sigma);
+STATIC double sample_logistic(uint32_t s, double t, double p0);
+
+STATIC double cdf_log_logistic(double x, double alpha, double beta);
+STATIC double sf_log_logistic(double x, double alpha, double beta);
+STATIC double icdf_log_logistic(double p, double alpha, double beta);
+STATIC double isf_log_logistic(double p, double alpha, double beta);
+STATIC double sample_log_logistic(uint32_t s, double p0);
+
+STATIC double cdf_weibull(double x, double lambda, double k);
+STATIC double sf_weibull(double x, double lambda, double k);
+STATIC double icdf_weibull(double p, double lambda, double k);
+STATIC double isf_weibull(double p, double lambda, double k);
+STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k);
+
+STATIC double sample_uniform_interval(double p0, double a, double b);
+
+STATIC double cdf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double sf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double icdf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double isf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double sample_genpareto(uint32_t s, double p0, double xi);
+
+#endif /* defined(PROB_DISTR_PRIVATE) */
+
+#endif /* !defined(TOR_PROB_DISTR_H) */
diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include
index 814652a93c..a1edaf2231 100644
--- a/src/lib/memarea/.may_include
+++ b/src/lib/memarea/.may_include
@@ -1,7 +1,7 @@
 orconfig.h
 lib/arch/*.h
 lib/cc/*.h
-lib/container/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/memarea/*.h
+lib/smartlist_core/*.h
\ No newline at end of file
diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am
index 94343dcead..83fb99ec73 100644
--- a/src/lib/memarea/include.am
+++ b/src/lib/memarea/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-memarea-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_memarea_a_SOURCES =			\
 	src/lib/memarea/memarea.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_memarea_testing_a_SOURCES = \
 src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/memarea/memarea.h
diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c
index 486673116c..f3bb79a1e2 100644
--- a/src/lib/memarea/memarea.c
+++ b/src/lib/memarea/memarea.c
@@ -16,7 +16,8 @@
 
 #include "lib/arch/bytes.h"
 #include "lib/cc/torint.h"
-#include "lib/container/smartlist.h"
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
@@ -67,7 +68,7 @@
   uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]);     \
   tor_assert(sent_val == SENTINEL_VAL);                                 \
   STMT_END
-#else /* !(defined(USE_SENTINELS)) */
+#else /* !defined(USE_SENTINELS) */
 #define SENTINEL_LEN 0
 #define SET_SENTINEL(chunk) STMT_NIL
 #define CHECK_SENTINEL(chunk) STMT_NIL
@@ -314,7 +315,7 @@ memarea_assert_ok(memarea_t *area)
   }
 }
 
-#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */
+#else /* defined(DISABLE_MEMORY_SENTINELS) */
 
 struct memarea_t {
   smartlist_t *pieces;
diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am
index d1fdde6313..12c1bff72d 100644
--- a/src/lib/meminfo/include.am
+++ b/src/lib/meminfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_meminfo_a_SOURCES =			\
 	src/lib/meminfo/meminfo.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_meminfo_testing_a_SOURCES = \
 src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/meminfo/meminfo.h
diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c
index f4fa45167e..bff71c2f05 100644
--- a/src/lib/meminfo/meminfo.c
+++ b/src/lib/meminfo/meminfo.c
@@ -54,7 +54,7 @@ tor_log_mallinfo(int severity)
       mi.arena, mi.ordblks, mi.smblks, mi.hblks,
       mi.hblkhd, mi.usmblks, mi.fsmblks, mi.uordblks, mi.fordblks,
       mi.keepcost);
-#else /* !(defined(HAVE_MALLINFO)) */
+#else /* !defined(HAVE_MALLINFO) */
   (void)severity;
 #endif /* defined(HAVE_MALLINFO) */
 }
diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h
index 2d64e1ab06..9580640f4d 100644
--- a/src/lib/meminfo/meminfo.h
+++ b/src/lib/meminfo/meminfo.h
@@ -18,4 +18,4 @@
 void tor_log_mallinfo(int severity);
 MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
 
-#endif
+#endif /* !defined(TOR_MEMINFO_H) */
diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include
index 13b209bbed..e4368f799b 100644
--- a/src/lib/net/.may_include
+++ b/src/lib/net/.may_include
@@ -1,8 +1,9 @@
 orconfig.h
-siphash.h
-ht.h
+ext/siphash.h
+ext/ht.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
@@ -11,5 +12,6 @@ lib/lock/*.h
 lib/log/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/malloc/*.h
\ No newline at end of file
diff --git a/src/lib/net/address.c b/src/lib/net/address.c
index 69004ddb0e..41017935e1 100644
--- a/src/lib/net/address.c
+++ b/src/lib/net/address.c
@@ -40,6 +40,7 @@
 
 #include "lib/net/address.h"
 #include "lib/net/socket.h"
+#include "lib/cc/ctassert.h"
 #include "lib/container/smartlist.h"
 #include "lib/ctime/di_ops.h"
 #include "lib/log/log.h"
@@ -52,7 +53,7 @@
 #include "lib/string/printf.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
@@ -98,6 +99,7 @@
 #if AF_UNSPEC != 0
 #error We rely on AF_UNSPEC being 0. Let us know about your platform, please!
 #endif
+CTASSERT(AF_UNSPEC == 0);
 
 /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a
  * sockaddr object in *<b>sa_out</b> of object size <b>len</b>.  If not enough
@@ -337,7 +339,7 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
       break;
     case AF_INET6:
       /* Shortest addr [ :: ] + \0 */
-      if (len < (3 + (decorate ? 2u : 0u)))
+      if (len < (3u + (decorate ? 2 : 0)))
         return NULL;
 
       if (decorate)
@@ -371,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
  *
  * If <b>accept_regular</b> is set and the address is in neither recognized
  * reverse lookup hostname format, try parsing the address as a regular
- * IPv4 or IPv6 address too.
+ * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or
+ * without square brackets.
  */
 int
 tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
@@ -1186,37 +1189,75 @@ fmt_addr32(uint32_t addr)
 }
 
 /** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
- * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
- * square brackets.
+ * may be an IPv4 address, or an IPv6 address surrounded by square brackets.
  *
- *  Return an address family on success, or -1 if an invalid address string is
- *  provided. */
-int
-tor_addr_parse(tor_addr_t *addr, const char *src)
+ * If <b>allow_ipv6_without_brackets</b> is true, also allow IPv6 addresses
+ * without brackets.
+ *
+ * Always rejects IPv4 addresses with brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+static int
+tor_addr_parse_impl(tor_addr_t *addr, const char *src,
+                    bool allow_ipv6_without_brackets)
 {
   /* Holds substring of IPv6 address after removing square brackets */
   char *tmp = NULL;
-  int result;
+  int result = -1;
   struct in_addr in_tmp;
   struct in6_addr in6_tmp;
+  int brackets_detected = 0;
+
   tor_assert(addr && src);
-  if (src[0] == '[' && src[1])
+
+  size_t len = strlen(src);
+
+  if (len && src[0] == '[' && src[len - 1] == ']') {
+    brackets_detected = 1;
     src = tmp = tor_strndup(src+1, strlen(src)-2);
+  }
 
-  if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
-    result = AF_INET6;
-    tor_addr_from_in6(addr, &in6_tmp);
-  } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
-    result = AF_INET;
-    tor_addr_from_in(addr, &in_tmp);
-  } else {
-    result = -1;
+  /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses
+   * without brackets are allowed */
+  if (brackets_detected || allow_ipv6_without_brackets) {
+    if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
+      result = AF_INET6;
+      tor_addr_from_in6(addr, &in6_tmp);
+    }
+  }
+
+  /* Try to parse an IPv4 address without brackets */
+  if (!brackets_detected) {
+    if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
+      result = AF_INET;
+      tor_addr_from_in(addr, &in_tmp);
+    }
+  }
+
+  /* Clear the address on error, to avoid returning uninitialised or partly
+   * parsed data.
+   */
+  if (result == -1) {
+    memset(addr, 0, sizeof(tor_addr_t));
   }
 
   tor_free(tmp);
   return result;
 }
 
+/** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
+ * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
+ * square brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+int
+tor_addr_parse(tor_addr_t *addr, const char *src)
+{
+  return tor_addr_parse_impl(addr, src, 1);
+}
+
 #ifdef HAVE_IFADDRS_TO_SMARTLIST
 /*
  * Convert a linked list consisting of <b>ifaddrs</b> structures
@@ -1709,6 +1750,11 @@ get_interface_address6_list,(int severity,
  * form "ip" or "ip:0".  Otherwise, accept those forms, and set
  * *<b>port_out</b> to <b>default_port</b>.
  *
+ * This function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
  * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_parse(int severity, const char *addrport,
@@ -1718,6 +1764,7 @@ tor_addr_port_parse(int severity, const char *addrport,
   int retval = -1;
   int r;
   char *addr_tmp = NULL;
+  bool has_port;
 
   tor_assert(addrport);
   tor_assert(address_out);
@@ -1727,28 +1774,47 @@ tor_addr_port_parse(int severity, const char *addrport,
   if (r < 0)
     goto done;
 
-  if (!*port_out) {
+  has_port = !! *port_out;
+  /* If there's no port, use the default port, or fail if there is no default
+   */
+  if (!has_port) {
     if (default_port >= 0)
       *port_out = default_port;
     else
       goto done;
   }
 
-  /* make sure that address_out is an IP address */
-  if (tor_addr_parse(address_out, addr_tmp) < 0)
+  /* Make sure that address_out is an IP address.
+   * If there is no port in addrport, allow IPv6 addresses without brackets. */
+  if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0)
     goto done;
 
   retval = 0;
 
  done:
+  /* Clear the address and port on error, to avoid returning uninitialised or
+   * partly parsed data.
+   */
+  if (retval == -1) {
+    memset(address_out, 0, sizeof(tor_addr_t));
+    *port_out = 0;
+  }
   tor_free(addr_tmp);
   return retval;
 }
 
 /** Given an address of the form "host[:port]", try to divide it into its host
- * and port portions, setting *<b>address_out</b> to a newly allocated string
- * holding the address portion and *<b>port_out</b> to the port (or 0 if no
- * port is given).  Return 0 on success, -1 on failure. */
+ * and port portions.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Sets *<b>address_out</b> to a newly allocated string holding the address
+ * portion, and *<b>port_out</b> to the port (or 0 if no port is given).
+ *
+ * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_split(int severity, const char *addrport,
                     char **address_out, uint16_t *port_out)
@@ -1757,8 +1823,11 @@ tor_addr_port_split(int severity, const char *addrport,
   tor_assert(addrport);
   tor_assert(address_out);
   tor_assert(port_out);
+
   /* We need to check for IPv6 manually because the logic below doesn't
-   * do a good job on IPv6 addresses that lack a port. */
+   * do a good job on IPv6 addresses that lack a port.
+   * If an IPv6 address without square brackets is ambiguous, it gets parsed
+   * here as an address, rather than address:port. */
   if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) {
     *port_out = 0;
     *address_out = tor_strdup(addrport);
@@ -1798,8 +1867,7 @@ tor_addr_port_split(int severity, const char *addrport,
     tor_free(address_);
   }
 
-  if (port_out)
-    *port_out = ok ? ((uint16_t) port_) : 0;
+  *port_out = ok ? ((uint16_t) port_) : 0;
 
   return ok ? 0 : -1;
 }
@@ -2018,8 +2086,12 @@ string_is_valid_nonrfc_hostname(const char *string)
 
   smartlist_split_string(components,string,".",0,0);
 
-  if (BUG(smartlist_len(components) == 0))
-    return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks.
+  if (BUG(smartlist_len(components) == 0)) {
+    // LCOV_EXCL_START should be impossible given the earlier checks.
+    smartlist_free(components);
+    return 0;
+    // LCOV_EXCL_STOP
+  }
 
   /* Allow a single terminating '.' used rarely to indicate domains
    * are FQDNs rather than relative. */
diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h
index c45f42be81..4d0d0dd57c 100644
--- a/src/lib/net/alertsock.h
+++ b/src/lib/net/alertsock.h
@@ -42,4 +42,4 @@ typedef struct alert_sockets_t {
 int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags);
 void alert_sockets_close(alert_sockets_t *socks);
 
-#endif
+#endif /* !defined(TOR_ALERTSOCK_H) */
diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c
index 3eb0a033d5..cfe1a7dc26 100644
--- a/src/lib/net/buffers_net.c
+++ b/src/lib/net/buffers_net.c
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/net/buffers_net.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/net/nettypes.h"
@@ -22,6 +22,10 @@
 
 #include <stdlib.h>
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 #ifdef PARANOIA
 /** Helper: If PARANOIA is defined, assert that the buffer in local variable
  * <b>buf</b> is well-formed. */
@@ -30,27 +34,36 @@
 #define check() STMT_NIL
 #endif /* defined(PARANOIA) */
 
-/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into
  * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
- * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
- * and the number of bytes read otherwise. */
+ * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b>
+ * is true, otherwise it uses <b>read()</b>.  Return -1 on error (and sets
+ * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read
+ * otherwise. */
 static inline int
 read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
-              int *reached_eof, int *socket_error)
+              int *reached_eof, int *error, bool is_socket)
 {
   ssize_t read_result;
   if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
     at_most = CHUNK_REMAINING_CAPACITY(chunk);
-  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (is_socket)
+    read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+  else
+    read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most);
 
   if (read_result < 0) {
-    int e = tor_socket_errno(fd);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
-        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+        log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?",
+                 is_socket ? "recv" : "read");
 #endif
-      *socket_error = e;
+      if (error)
+        *error = e;
       return -1;
     }
     return 0; /* would block. */
@@ -68,16 +81,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
   }
 }
 
-/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
- * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
- * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
- * error; else return the number of bytes read.
+/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>.  Read
+ * at most <b>at_most</b> bytes, growing the buffer as necessary.  If recv()
+ * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0.
+ * Return -1 on error; else return the number of bytes read.
  */
 /* XXXX indicate "read blocked" somehow? */
-int
-buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
-                     int *reached_eof,
-                     int *socket_error)
+static int
+buf_read_from_fd(buf_t *buf, int fd, size_t at_most,
+                 int *reached_eof,
+                 int *socket_error,
+                 bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes read" are not mutually exclusive.
@@ -87,7 +101,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 
   check();
   tor_assert(reached_eof);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
 
   if (BUG(buf->datalen >= INT_MAX))
     return -1;
@@ -108,7 +122,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
         readlen = cap;
     }
 
-    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    r = read_to_chunk(buf, chunk, fd, readlen,
+                      reached_eof, socket_error, is_socket);
     check();
     if (r < 0)
       return r; /* Error */
@@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 }
 
 /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
- * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, 0 on blocking, -1 on failure.
+ * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>.  On
+ * success, deduct the bytes written from *<b>buf_flushlen</b>.  Return the
+ * number of bytes written on success, 0 on blocking, -1 on failure.
  */
 static inline int
-flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
-            size_t *buf_flushlen)
+flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen, bool is_socket)
 {
   ssize_t write_result;
 
   if (sz > chunk->datalen)
     sz = chunk->datalen;
-  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (is_socket)
+    write_result = tor_socket_send(fd, chunk->data, sz, 0);
+  else
+    write_result = write(fd, chunk->data, sz);
 
   if (write_result < 0) {
-    int e = tor_socket_errno(s);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
@@ -155,15 +175,15 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
   }
 }
 
-/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+/** Write data from <b>buf</b> to the file descriptor <b>fd</b>.  Write at most
  * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
  * the number of bytes actually written, and remove the written bytes
  * from the buffer.  Return the number of bytes written on success,
  * -1 on failure.  Return 0 if write() would block.
  */
-int
-buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
-                    size_t *buf_flushlen)
+static int
+buf_flush_to_fd(buf_t *buf, int fd, size_t sz,
+                size_t *buf_flushlen, bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes flushed" are not mutually exclusive.
@@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
   int r;
   size_t flushed = 0;
   tor_assert(buf_flushlen);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
   if (BUG(*buf_flushlen > buf->datalen)) {
     *buf_flushlen = buf->datalen;
   }
@@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
     else
       flushlen0 = buf->head->datalen;
 
-    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket);
     check();
     if (r < 0)
       return r;
@@ -200,3 +220,55 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
   tor_assert(flushed < INT_MAX);
   return (int)flushed;
 }
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, s, sz, buf_flushlen, true);
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true);
+}
+
+/** Write data from <b>buf</b> to the pipe <b>fd</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_pipe(buf_t *buf, int fd, size_t sz,
+                  size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false);
+}
+
+/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If read() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_pipe(buf_t *buf, int fd, size_t at_most,
+                   int *reached_eof,
+                   int *socket_error)
+{
+  return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false);
+}
diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h
index 5f69bebedf..5058dd0a26 100644
--- a/src/lib/net/buffers_net.h
+++ b/src/lib/net/buffers_net.h
@@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most,
 int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz,
                         size_t *buf_flushlen);
 
-#endif /* !defined(TOR_BUFFERS_H) */
+int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most,
+                       int *reached_eof,
+                       int *socket_error);
+
+int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz,
+                      size_t *buf_flushlen);
+
+#endif /* !defined(TOR_BUFFERS_NET_H) */
diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h
index 69b0528bc0..b3b77b0589 100644
--- a/src/lib/net/gethostname.h
+++ b/src/lib/net/gethostname.h
@@ -16,4 +16,4 @@
 
 MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
 
-#endif
+#endif /* !defined(TOR_GETHOSTNAME_H) */
diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c
index 1a2406ce5f..d9ae7cd562 100644
--- a/src/lib/net/inaddr.c
+++ b/src/lib/net/inaddr.c
@@ -168,6 +168,13 @@ tor_inet_pton(int af, const char *src, void *dst)
   if (af == AF_INET) {
     return tor_inet_aton(src, dst);
   } else if (af == AF_INET6) {
+    ssize_t len = strlen(src);
+
+    /* Reject if src has needless trailing ':'. */
+    if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') {
+      return 0;
+    }
+
     struct in6_addr *out = dst;
     uint16_t words[8];
     int gapPos = -1, i, setWords=0;
@@ -207,7 +214,6 @@ tor_inet_pton(int af, const char *src, void *dst)
         return 0;
       if (TOR_ISXDIGIT(*src)) {
         char *next;
-        ssize_t len;
         long r = strtol(src, &next, 16);
         if (next == NULL || next == src) {
           /* The 'next == src' error case can happen on versions of openbsd
diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h
index 36352b65ea..602573944c 100644
--- a/src/lib/net/inaddr.h
+++ b/src/lib/net/inaddr.h
@@ -24,4 +24,4 @@ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
 const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len);
 int tor_inet_pton(int af, const char *src, void *dst);
 
-#endif
+#endif /* !defined(TOR_INADDR_H) */
diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h
index 806f2c096a..230f29a63a 100644
--- a/src/lib/net/inaddr_st.h
+++ b/src/lib/net/inaddr_st.h
@@ -104,4 +104,4 @@ struct sockaddr_in6 {
 };
 #endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */
 
-#endif /* TOR_INADDR_ST_H */
+#endif /* !defined(TOR_INADDR_ST_H) */
diff --git a/src/lib/net/include.am b/src/lib/net/include.am
index ff0967e786..485019f4b7 100644
--- a/src/lib/net/include.am
+++ b/src/lib/net/include.am
@@ -5,12 +5,14 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-net-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_net_a_SOURCES =			\
 	src/lib/net/address.c			\
 	src/lib/net/alertsock.c                 \
 	src/lib/net/buffers_net.c		\
 	src/lib/net/gethostname.c		\
 	src/lib/net/inaddr.c			\
+	src/lib/net/network_sys.c		\
 	src/lib/net/resolve.c			\
 	src/lib/net/socket.c			\
 	src/lib/net/socketpair.c
@@ -20,6 +22,7 @@ src_lib_libtor_net_testing_a_SOURCES = \
 src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/net/address.h			\
 	src/lib/net/alertsock.h                 \
@@ -28,6 +31,7 @@ noinst_HEADERS +=				\
 	src/lib/net/inaddr.h			\
 	src/lib/net/inaddr_st.h			\
 	src/lib/net/nettypes.h			\
+	src/lib/net/network_sys.h		\
 	src/lib/net/resolve.h			\
 	src/lib/net/socket.h			\
 	src/lib/net/socketpair.h		\
diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h
index 6209bbe18a..60039bac09 100644
--- a/src/lib/net/nettypes.h
+++ b/src/lib/net/nettypes.h
@@ -31,7 +31,7 @@ typedef int socklen_t;
 #define TOR_SOCKET_T_FORMAT "%"PRIuPTR
 #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
 #define TOR_INVALID_SOCKET INVALID_SOCKET
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 /** Type used for a network socket. */
 #define tor_socket_t int
 #define TOR_SOCKET_T_FORMAT "%d"
@@ -41,4 +41,4 @@ typedef int socklen_t;
 #define TOR_INVALID_SOCKET (-1)
 #endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_NET_TYPES_H) */
diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c
new file mode 100644
index 0000000000..e0a2625d73
--- /dev/null
+++ b/src/lib/net/network_sys.c
@@ -0,0 +1,46 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file network_sys.c
+ * \brief Subsystem object for networking setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/net/network_sys.h"
+#include "lib/net/resolve.h"
+#include "lib/net/socket.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#endif
+
+static int
+subsys_network_initialize(void)
+{
+  if (network_init() < 0)
+    return -1;
+
+  return 0;
+}
+
+static void
+subsys_network_shutdown(void)
+{
+#ifdef _WIN32
+  WSACleanup();
+#endif
+  tor_free_getaddrinfo_cache();
+}
+
+const subsys_fns_t sys_network = {
+  .name = "network",
+  /* Network depends on logging, and a lot of other modules depend on network.
+   */
+  .level = -80,
+  .supported = true,
+  .initialize = subsys_network_initialize,
+  .shutdown = subsys_network_shutdown,
+};
diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h
new file mode 100644
index 0000000000..43e62592ca
--- /dev/null
+++ b/src/lib/net/network_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_network.h
+ * \brief Declare subsystem object for the network module.
+ **/
+
+#ifndef TOR_NETWORK_SYS_H
+#define TOR_NETWORK_SYS_H
+
+extern const struct subsys_fns_t sys_network;
+
+#endif /* !defined(TOR_NETWORK_SYS_H) */
diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c
index 8cee29df37..442bc4a6b3 100644
--- a/src/lib/net/resolve.c
+++ b/src/lib/net/resolve.c
@@ -8,6 +8,7 @@
  * \brief Use the libc DNS resolver to convert hostnames into addresses.
  **/
 
+#define RESOLVE_PRIVATE
 #include "lib/net/resolve.h"
 
 #include "lib/net/address.h"
@@ -16,8 +17,8 @@
 #include "lib/string/parse_int.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
-#include "ht.h"
+#include "ext/siphash.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
@@ -35,6 +36,8 @@
  * *<b>addr</b> to the proper IP address, in host byte order.  Returns 0
  * on success, -1 on failure; 1 on transient failure.
  *
+ * This function only accepts IPv4 addresses.
+ *
  * (This function exists because standard windows gethostbyname
  * doesn't treat raw IP addresses properly.)
  */
@@ -45,6 +48,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   tor_addr_t myaddr;
   int ret;
 
+  if (BUG(!addr))
+    return -1;
+
+  *addr = 0;
+
   if ((ret = tor_addr_lookup(name, AF_INET, &myaddr)))
     return ret;
 
@@ -56,183 +64,257 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   return -1;
 }
 
-/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
- * *<b>addr</b> to the proper IP address and family. The <b>family</b>
- * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
- * <i>preferred</i> family, though another one may be returned if only one
- * family is implemented for this address.
+#ifdef HAVE_GETADDRINFO
+
+/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is
+ * available on this system.
  *
- * Return 0 on success, -1 on failure; 1 on transient failure.
+ * See tor_addr_lookup() for details.
  */
-MOCK_IMPL(int,
-tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                          tor_addr_t *addr))
 {
-  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
-   * something.
-   */
-  struct in_addr iaddr;
-  struct in6_addr iaddr6;
-  tor_assert(name);
-  tor_assert(addr);
-  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
-  if (!*name) {
-    /* Empty address is an error. */
-    return -1;
-  } else if (tor_inet_pton(AF_INET, name, &iaddr)) {
-    /* It's an IPv4 IP. */
-    if (family == AF_INET6)
-      return -1;
-    tor_addr_from_in(addr, &iaddr);
-    return 0;
-  } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) {
-    if (family == AF_INET)
-      return -1;
-    tor_addr_from_in6(addr, &iaddr6);
-    return 0;
-  } else {
-#ifdef HAVE_GETADDRINFO
-    int err;
-    struct addrinfo *res=NULL, *res_p;
-    struct addrinfo *best=NULL;
-    struct addrinfo hints;
-    int result = -1;
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = family;
-    hints.ai_socktype = SOCK_STREAM;
-    err = tor_getaddrinfo(name, NULL, &hints, &res);
-    /* The check for 'res' here shouldn't be necessary, but it makes static
-     * analysis tools happy. */
-    if (!err && res) {
-      best = NULL;
-      for (res_p = res; res_p; res_p = res_p->ai_next) {
-        if (family == AF_UNSPEC) {
-          if (res_p->ai_family == AF_INET) {
-            best = res_p;
-            break;
-          } else if (res_p->ai_family == AF_INET6 && !best) {
-            best = res_p;
-          }
-        } else if (family == res_p->ai_family) {
+  int err;
+  struct addrinfo *res=NULL, *res_p;
+  struct addrinfo *best=NULL;
+  struct addrinfo hints;
+  int result = -1;
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = family;
+  hints.ai_socktype = SOCK_STREAM;
+  err = tor_getaddrinfo(name, NULL, &hints, &res);
+  /* The check for 'res' here shouldn't be necessary, but it makes static
+   * analysis tools happy. */
+  if (!err && res) {
+    best = NULL;
+    for (res_p = res; res_p; res_p = res_p->ai_next) {
+      if (family == AF_UNSPEC) {
+        if (res_p->ai_family == AF_INET) {
           best = res_p;
           break;
+        } else if (res_p->ai_family == AF_INET6 && !best) {
+          best = res_p;
         }
+      } else if (family == res_p->ai_family) {
+        best = res_p;
+        break;
       }
-      if (!best)
-        best = res;
-      if (best->ai_family == AF_INET) {
-        tor_addr_from_in(addr,
-                         &((struct sockaddr_in*)best->ai_addr)->sin_addr);
-        result = 0;
-      } else if (best->ai_family == AF_INET6) {
-        tor_addr_from_in6(addr,
-                          &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
-        result = 0;
-      }
-      tor_freeaddrinfo(res);
-      return result;
     }
-    return (err == EAI_AGAIN) ? 1 : -1;
-#else /* !(defined(HAVE_GETADDRINFO)) */
-    struct hostent *ent;
-    int err;
+    if (!best)
+      best = res;
+    if (best->ai_family == AF_INET) {
+      tor_addr_from_in(addr,
+                       &((struct sockaddr_in*)best->ai_addr)->sin_addr);
+      result = 0;
+    } else if (best->ai_family == AF_INET6) {
+      tor_addr_from_in6(addr,
+                        &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
+      result = 0;
+    }
+    tor_freeaddrinfo(res);
+    return result;
+  }
+  return (err == EAI_AGAIN) ? 1 : -1;
+}
+
+#else /* !defined(HAVE_GETADDRINFO) */
+
+/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname().
+ * Used when getaddrinfo() is not available on this system.
+ *
+ * See tor_addr_lookup() for details.
+ */
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                           tor_addr_t *addr))
+{
+  (void) family;
+  struct hostent *ent;
+  int err;
 #ifdef HAVE_GETHOSTBYNAME_R_6_ARG
-    char buf[2048];
-    struct hostent hostent;
-    int r;
-    r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
+  char buf[2048];
+  struct hostent hostent;
+  int r;
+  r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_5_ARG)
-    char buf[2048];
-    struct hostent hostent;
-    ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
+  char buf[2048];
+  struct hostent hostent;
+  ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_3_ARG)
-    struct hostent_data data;
-    struct hostent hent;
-    memset(&data, 0, sizeof(data));
-    err = gethostbyname_r(name, &hent, &data);
-    ent = err ? NULL : &hent;
+  struct hostent_data data;
+  struct hostent hent;
+  memset(&data, 0, sizeof(data));
+  err = gethostbyname_r(name, &hent, &data);
+  ent = err ? NULL : &hent;
 #else
-    ent = gethostbyname(name);
+  ent = gethostbyname(name);
 #ifdef _WIN32
-    err = WSAGetLastError();
+  err = WSAGetLastError();
 #else
-    err = h_errno;
-#endif
+  err = h_errno;
+#endif /* defined(_WIN32) */
 #endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */
-    if (ent) {
-      if (ent->h_addrtype == AF_INET) {
-        tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
-      } else if (ent->h_addrtype == AF_INET6) {
-        tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
-      } else {
-        tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
-      }
-      return 0;
+  if (ent) {
+    if (ent->h_addrtype == AF_INET) {
+      tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
+    } else if (ent->h_addrtype == AF_INET6) {
+      tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
+    } else {
+      tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
     }
+    return 0;
+  }
 #ifdef _WIN32
-    return (err == WSATRY_AGAIN) ? 1 : -1;
+  return (err == WSATRY_AGAIN) ? 1 : -1;
 #else
-    return (err == TRY_AGAIN) ? 1 : -1;
+  return (err == TRY_AGAIN) ? 1 : -1;
 #endif
+}
 #endif /* defined(HAVE_GETADDRINFO) */
+
+/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
+ * *<b>addr</b> to the proper IP address and family. The <b>family</b>
+ * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
+ * <i>preferred</i> family, though another one may be returned if only one
+ * family is implemented for this address.
+ *
+ * Like tor_addr_parse(), this function accepts IPv6 addresses with or without
+ * square brackets.
+ *
+ * Return 0 on success, -1 on failure; 1 on transient failure.
+ */
+MOCK_IMPL(int,
+tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+{
+  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
+   * something.
+   */
+  int parsed_family = 0;
+  int result = -1;
+
+  tor_assert(name);
+  tor_assert(addr);
+  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
+
+  if (!*name) {
+    /* Empty address is an error. */
+    goto permfail;
+  }
+
+  /* Is it an IP address? */
+  parsed_family = tor_addr_parse(addr, name);
+
+  if (parsed_family >= 0) {
+    /* If the IP address family matches, or was unspecified */
+    if (parsed_family == family || family == AF_UNSPEC) {
+      goto success;
+    } else {
+      goto permfail;
+    }
+  } else {
+    /* Clear the address after a failed tor_addr_parse(). */
+    memset(addr, 0, sizeof(tor_addr_t));
+    result = tor_addr_lookup_host_impl(name, family, addr);
+    goto done;
   }
+
+ /* If we weren't successful, and haven't already set the result,
+  * assume it's a permanent failure */
+ permfail:
+  result = -1;
+  goto done;
+ success:
+  result = 0;
+
+ /* We have set the result, now it's time to clean up */
+ done:
+  if (result) {
+    /* Clear the address on error */
+    memset(addr, 0, sizeof(tor_addr_t));
+  }
+  return result;
 }
 
 /** Parse an address or address-port combination from <b>s</b>, resolve the
  * address as needed, and put the result in <b>addr_out</b> and (optionally)
- * <b>port_out</b>.  Return 0 on success, negative on failure. */
+ * <b>port_out</b>.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Return 0 on success, negative on failure. */
 int
 tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out)
 {
-  const char *port;
   tor_addr_t addr;
-  uint16_t portval;
+  uint16_t portval = 0;
   char *tmp = NULL;
+  int rv = 0;
+  int result;
 
   tor_assert(s);
   tor_assert(addr_out);
 
   s = eat_whitespace(s);
 
-  if (*s == '[') {
-    port = strstr(s, "]");
-    if (!port)
-      goto err;
-    tmp = tor_strndup(s+1, port-(s+1));
-    port = port+1;
-    if (*port == ':')
-      port++;
-    else
-      port = NULL;
-  } else {
-    port = strchr(s, ':');
-    if (port)
-      tmp = tor_strndup(s, port-s);
-    else
-      tmp = tor_strdup(s);
-    if (port)
-      ++port;
+  /* Try parsing s as an address:port first, so we don't have to duplicate
+   * the logic that rejects IPv6:Port with no square brackets. */
+  rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0);
+  /* That was easy, no DNS required. */
+  if (rv == 0)
+    goto success;
+
+  /* Now let's check for malformed IPv6 addresses and ports:
+   * tor_addr_port_parse() requires squared brackes if there is a port,
+   * and we want tor_addr_port_lookup() to have the same requirement.
+   * But we strip the port using tor_addr_port_split(), so tor_addr_lookup()
+   * only sees the address, and will accept it without square brackets. */
+  int family = tor_addr_parse(&addr, s);
+  /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need
+   * to reject this address as malformed. */
+  if (family >= 0) {
+    /* Double-check it's an IPv6 address. If not, we have a parsing bug.
+     */
+    tor_assertf_nonfatal(family == AF_INET6,
+                         "Wrong family: %d (should be IPv6: %d) which "
+                         "failed IP:port parsing, but passed IP parsing. "
+                         "input string: '%s'; parsed address: '%s'.",
+                         family, AF_INET6, s, fmt_addr(&addr));
+    goto err;
   }
 
-  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+  /* Now we have a hostname. Let's split off the port, if any. */
+  rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval);
+  if (rv < 0)
     goto err;
-  tor_free(tmp);
 
-  if (port) {
-    portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL);
-    if (!portval)
-      goto err;
-  } else {
-    portval = 0;
-  }
+  /* And feed the hostname to the lookup function. */
+  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+    goto err;
 
+ success:
   if (port_out)
     *port_out = portval;
   tor_addr_copy(addr_out, &addr);
+  result = 0;
+  goto done;
 
-  return 0;
  err:
+  /* Clear the address and port on error */
+  memset(addr_out, 0, sizeof(tor_addr_t));
+  if (port_out)
+    *port_out = 0;
+  result = -1;
+
+ /* We have set the result, now it's time to clean up */
+ done:
   tor_free(tmp);
-  return -1;
+  return result;
 }
 
 #ifdef USE_SANDBOX_GETADDRINFO
@@ -421,4 +503,13 @@ tor_make_getaddrinfo_cache_active(void)
 {
   sandbox_getaddrinfo_is_active = 1;
 }
-#endif
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
+void
+sandbox_disable_getaddrinfo_cache(void)
+{
+}
+void
+tor_make_getaddrinfo_cache_active(void)
+{
+}
+#endif /* defined(USE_SANDBOX_GETADDRINFO) */
diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h
index 47a283c81c..b979b2fb41 100644
--- a/src/lib/net/resolve.h
+++ b/src/lib/net/resolve.h
@@ -24,12 +24,18 @@
 
 struct tor_addr_t;
 
+/*
+ * Primary lookup functions.
+ */
 MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr));
 MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
                                 struct tor_addr_t *addr_out));
 int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out,
                          uint16_t *port_out);
 
+/*
+ * Sandbox helpers
+ */
 struct addrinfo;
 #ifdef USE_SANDBOX_GETADDRINFO
 /** Pre-calls getaddrinfo in order to pre-record result. */
@@ -42,8 +48,7 @@ int tor_getaddrinfo(const char *name, const char *servname,
                         struct addrinfo **res);
 void tor_freeaddrinfo(struct addrinfo *addrinfo);
 void tor_free_getaddrinfo_cache(void);
-void tor_make_getaddrinfo_cache_active(void);
-#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
 #define tor_getaddrinfo(name, servname, hints, res)  \
   getaddrinfo((name),(servname), (hints),(res))
 #define tor_add_addrinfo(name) \
@@ -54,5 +59,15 @@ void tor_make_getaddrinfo_cache_active(void);
 #endif /* defined(USE_SANDBOX_GETADDRINFO) */
 
 void sandbox_disable_getaddrinfo_cache(void);
+void tor_make_getaddrinfo_cache_active(void);
 
+/*
+ * Internal resolver wrapper; exposed for mocking.
+ */
+#ifdef RESOLVE_PRIVATE
+MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name,
+                                                  uint16_t family,
+                                                  struct tor_addr_t *addr));
 #endif
+
+#endif /* !defined(TOR_RESOLVE_H) */
diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c
index fba90b7506..e1b82251ed 100644
--- a/src/lib/net/socket.c
+++ b/src/lib/net/socket.c
@@ -31,6 +31,9 @@
 #endif
 #include <stddef.h>
 #include <string.h>
+#ifdef __FreeBSD__
+#include <sys/sysctl.h>
+#endif
 
 /** Called before we make any calls to network-related functions.
  * (Some operating systems require their network libraries to be
@@ -60,6 +63,32 @@ network_init(void)
   return 0;
 }
 
+/**
+ * Warn the user if any system network parameters should be changed.
+ */
+void
+check_network_configuration(bool server_mode)
+{
+#ifdef __FreeBSD__
+  if (server_mode) {
+    int random_id_state;
+    size_t state_size = sizeof(random_id_state);
+
+    if (sysctlbyname("net.inet.ip.random_id", &random_id_state,
+                     &state_size, NULL, 0)) {
+      log_warn(LD_CONFIG,
+               "Failed to figure out if IP ids are randomized.");
+    } else if (random_id_state == 0) {
+      log_warn(LD_CONFIG, "Looks like IP ids are not randomized. "
+               "Please consider setting the net.inet.ip.random_id sysctl, "
+               "so your relay makes it harder to figure out how busy it is.");
+    }
+  }
+#else /* !defined(__FreeBSD__) */
+  (void) server_mode;
+#endif /* defined(__FreeBSD__) */
+}
+
 /* When set_max_file_sockets() is called, update this with the max file
  * descriptor value so we can use it to check the limit when opening a new
  * socket. Default value is what Debian sets as the default hard limit. */
@@ -177,7 +206,7 @@ mark_socket_closed(tor_socket_t s)
     bitarray_clear(open_sockets, s);
   }
 }
-#else /* !(defined(DEBUG_SOCKET_COUNTING)) */
+#else /* !defined(DEBUG_SOCKET_COUNTING) */
 #define mark_socket_open(s) ((void) (s))
 #define mark_socket_closed(s) ((void) (s))
 #endif /* defined(DEBUG_SOCKET_COUNTING) */
@@ -279,7 +308,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -389,7 +418,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -429,7 +458,9 @@ get_n_open_sockets(void)
  * localhost is inaccessible (for example, if the networking
  * stack is down). And even if it succeeds, the socket pair will not
  * be able to read while localhost is down later (the socket pair may
- * even close, depending on OS-specific timeouts).
+ * even close, depending on OS-specific timeouts). The socket pair
+ * should work on IPv4-only, IPv6-only, and dual-stack systems, as long
+ * as they have the standard localhost addresses.
  *
  * Returns 0 on success and -errno on failure; do not rely on the value
  * of errno or WSAGetLastError().
@@ -456,11 +487,11 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
   r = socketpair(family, type, protocol, fd);
   if (r < 0)
     return -errno;
-#else
+#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */
   r = tor_ersatz_socketpair(family, type, protocol, fd);
   if (r < 0)
     return -r;
-#endif
+#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */
 
 #if defined(FD_CLOEXEC)
   if (SOCKET_OK(fd[0])) {
diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h
index 0909619510..53a9f1bb92 100644
--- a/src/lib/net/socket.h
+++ b/src/lib/net/socket.h
@@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
 int set_socket_nonblocking(tor_socket_t socket);
 int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]);
 int network_init(void);
+void check_network_configuration(bool server_mode);
 
 int get_max_sockets(void);
 void set_max_sockets(int);
@@ -91,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count);
 #define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
 int tor_socket_errno(tor_socket_t sock);
 const char *tor_socket_strerror(int e);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 #define SOCK_ERRNO(e) e
 #if EAGAIN == EWOULDBLOCK
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
@@ -115,4 +116,4 @@ const char *tor_socket_strerror(int e);
 #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKET_H) */
diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c
index 10eb749735..f3a0c3770a 100644
--- a/src/lib/net/socketpair.c
+++ b/src/lib/net/socketpair.c
@@ -22,11 +22,11 @@
 #include <windows.h>
 #define socket_errno() (WSAGetLastError())
 #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT
-#else
+#else /* !defined(_WIN32) */
 #define closesocket(x) close(x)
 #define socket_errno() (errno)
 #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef NEED_ERSATZ_SOCKETPAIR
 
@@ -105,7 +105,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2)
 /**
  * Helper used to implement socketpair on systems that lack it, by
  * making a direct connection to localhost.
- */
+ *
+ * See tor_socketpair() for details.
+ *
+ * The direct connection defaults to IPv4, but falls back to IPv6 if
+ * IPv4 is not supported.
+ **/
 int
 tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
 {
diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h
index 6be0803881..5820606973 100644
--- a/src/lib/net/socketpair.h
+++ b/src/lib/net/socketpair.h
@@ -16,4 +16,4 @@ int tor_ersatz_socketpair(int family, int type, int protocol,
                           tor_socket_t fd[2]);
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKETPAIR_H) */
diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h
index e55242ce66..e55119e0b0 100644
--- a/src/lib/net/socks5_status.h
+++ b/src/lib/net/socks5_status.h
@@ -29,4 +29,4 @@ typedef enum {
   SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08,
 } socks5_reply_status_t;
 
-#endif
+#endif /* !defined(TOR_SOCKS5_STATUS_H) */
diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am
index 16c5812604..84bd7feb00 100644
--- a/src/lib/osinfo/include.am
+++ b/src/lib/osinfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_osinfo_a_SOURCES =			\
 	src/lib/osinfo/uname.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_osinfo_testing_a_SOURCES = \
 src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/osinfo/uname.h
diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c
index 2b37ff136c..34860c407a 100644
--- a/src/lib/osinfo/uname.c
+++ b/src/lib/osinfo/uname.c
@@ -137,7 +137,7 @@ get_uname,(void))
         if (!is_server && !is_client) {
           strlcat(uname_result, " [client or server]", sizeof(uname_result));
         }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
         /* LCOV_EXCL_START -- can't provoke uname failure */
         strlcpy(uname_result, "Unknown platform", sizeof(uname_result));
         /* LCOV_EXCL_STOP */
diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h
index fcce629074..443a30d358 100644
--- a/src/lib/osinfo/uname.h
+++ b/src/lib/osinfo/uname.h
@@ -15,4 +15,4 @@
 
 MOCK_DECL(const char *, get_uname,(void));
 
-#endif
+#endif /* !defined(HAVE_TOR_UNAME_H) */
diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include
index 05414d2a96..ce1b6ecf59 100644
--- a/src/lib/process/.may_include
+++ b/src/lib/process/.may_include
@@ -1,17 +1,20 @@
 orconfig.h
 
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
 lib/err/*.h
-lib/intmath/*.h
+lib/evloop/*.h
 lib/fs/*.h
+lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/process/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 
-ht.h
\ No newline at end of file
+ext/ht.h
diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c
index 3b90bef671..ae34b5bcb8 100644
--- a/src/lib/process/daemon.c
+++ b/src/lib/process/daemon.c
@@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd)
 
   return 0;
 }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
 /* defined(_WIN32) */
 int
 start_daemon(void)
diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h
index 20920e0aae..423c498837 100644
--- a/src/lib/process/daemon.h
+++ b/src/lib/process/daemon.h
@@ -18,4 +18,4 @@ int finish_daemon(const char *desired_cwd);
 
 bool start_daemon_has_been_called(void);
 
-#endif
+#endif /* !defined(TOR_DAEMON_H) */
diff --git a/src/lib/process/env.c b/src/lib/process/env.c
index 0060200ba1..3912ade197 100644
--- a/src/lib/process/env.c
+++ b/src/lib/process/env.c
@@ -47,7 +47,7 @@ get_environment(void)
    * when we do a mostly-static build on OSX 10.7, the resulting binary won't
    * work on OSX 10.6. */
   return *_NSGetEnviron();
-#else /* !(defined(HAVE__NSGETENVIRON)) */
+#else /* !defined(HAVE__NSGETENVIRON) */
   return environ;
 #endif /* defined(HAVE__NSGETENVIRON) */
 }
diff --git a/src/lib/process/env.h b/src/lib/process/env.h
index 15d59351e0..19c2235970 100644
--- a/src/lib/process/env.h
+++ b/src/lib/process/env.h
@@ -38,4 +38,4 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
                                            const char *new_var,
                                            void (*free_old)(void*),
                                            int free_p);
-#endif
+#endif /* !defined(TOR_ENV_H) */
diff --git a/src/lib/process/include.am b/src/lib/process/include.am
index c6cc3a6699..af5f99617b 100644
--- a/src/lib/process/include.am
+++ b/src/lib/process/include.am
@@ -5,25 +5,35 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-process-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_process_a_SOURCES =		\
 	src/lib/process/daemon.c		\
 	src/lib/process/env.c			\
 	src/lib/process/pidfile.c		\
+	src/lib/process/process.c		\
+	src/lib/process/process_sys.c		\
+	src/lib/process/process_unix.c		\
+	src/lib/process/process_win32.c		\
 	src/lib/process/restrict.c		\
 	src/lib/process/setuid.c		\
-	src/lib/process/subprocess.c		\
-	src/lib/process/waitpid.c
+	src/lib/process/waitpid.c		\
+	src/lib/process/winprocess_sys.c
 
 src_lib_libtor_process_testing_a_SOURCES = \
 	$(src_lib_libtor_process_a_SOURCES)
 src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/process/daemon.h		\
 	src/lib/process/env.h			\
 	src/lib/process/pidfile.h		\
+	src/lib/process/process.h		\
+	src/lib/process/process_sys.h		\
+	src/lib/process/process_unix.h		\
+	src/lib/process/process_win32.h		\
 	src/lib/process/restrict.h		\
 	src/lib/process/setuid.h		\
-	src/lib/process/subprocess.h		\
-	src/lib/process/waitpid.h
+	src/lib/process/waitpid.h		\
+	src/lib/process/winprocess_sys.h
diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h
index dfeb39e046..af59041f80 100644
--- a/src/lib/process/pidfile.h
+++ b/src/lib/process/pidfile.h
@@ -13,4 +13,4 @@
 
 int write_pidfile(const char *filename);
 
-#endif
+#endif /* !defined(TOR_PIDFILE_H) */
diff --git a/src/lib/process/process.c b/src/lib/process/process.c
new file mode 100644
index 0000000000..2194a603ff
--- /dev/null
+++ b/src/lib/process/process.c
@@ -0,0 +1,797 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.c
+ * \brief Module for working with other processes.
+ **/
+
+#define PROCESS_PRIVATE
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_STDDEF_H
+#include <stddef.h>
+#endif
+
+/** A list of all <b>process_t</b> instances currently allocated. */
+static smartlist_t *processes;
+
+/**
+ * Boolean.  If true, then Tor may call execve or CreateProcess via
+ * tor_spawn_background.
+ **/
+static int may_spawn_background_process = 1;
+
+/** Structure to represent a child process. */
+struct process_t {
+  /** Process status. */
+  process_status_t status;
+
+  /** Which protocol is the process using? */
+  process_protocol_t protocol;
+
+  /** Which function to call when we have data ready from stdout? */
+  process_read_callback_t stdout_read_callback;
+
+  /** Which function to call when we have data ready from stderr? */
+  process_read_callback_t stderr_read_callback;
+
+  /** Which function call when our process terminated? */
+  process_exit_callback_t exit_callback;
+
+  /** Our exit code when the process have terminated. */
+  process_exit_code_t exit_code;
+
+  /** Name of the command we want to execute (for example: /bin/ls). */
+  char *command;
+
+  /** The arguments used for the new process. The format here is one argument
+   * per element of the smartlist_t. On Windows these arguments are combined
+   * together using the <b>tor_join_win_cmdline</b> function. On Unix the
+   * process name (argv[0]) and the trailing NULL is added automatically before
+   * the process is executed. */
+  smartlist_t *arguments;
+
+  /** The environment used for the new process. */
+  smartlist_t *environment;
+
+  /** Buffer to store data from stdout when it is read. */
+  buf_t *stdout_buffer;
+
+  /** Buffer to store data from stderr when it is read. */
+  buf_t *stderr_buffer;
+
+  /** Buffer to store data to stdin before it is written. */
+  buf_t *stdin_buffer;
+
+  /** Do we need to store some custom data with the process? */
+  void *data;
+
+#ifndef _WIN32
+  /** Our Unix process handle. */
+  process_unix_t *unix_process;
+#else
+  /** Our Win32 process handle. */
+  process_win32_t *win32_process;
+#endif /* !defined(_WIN32) */
+};
+
+/** Convert a given process status in <b>status</b> to its string
+ * representation. */
+const char *
+process_status_to_string(process_status_t status)
+{
+  switch (status) {
+  case PROCESS_STATUS_NOT_RUNNING:
+    return "not running";
+  case PROCESS_STATUS_RUNNING:
+    return "running";
+  case PROCESS_STATUS_ERROR:
+    return "error";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/** Convert a given process protocol in <b>protocol</b> to its string
+ * representation. */
+const char *
+process_protocol_to_string(process_protocol_t protocol)
+{
+  switch (protocol) {
+  case PROCESS_PROTOCOL_LINE:
+    return "Line";
+  case PROCESS_PROTOCOL_RAW:
+    return "Raw";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * Turn off may_spawn_background_process, so that all future calls to
+ * tor_spawn_background are guaranteed to fail.
+ **/
+void
+tor_disable_spawning_background_processes(void)
+{
+  may_spawn_background_process = 0;
+}
+
+/** Initialize the Process subsystem.  This function initializes the Process
+ * subsystem's global state. For cleaning up, <b>process_free_all()</b> should
+ * be called. */
+void
+process_init(void)
+{
+  processes = smartlist_new();
+
+#ifdef _WIN32
+  process_win32_init();
+#endif
+}
+
+/** Free up all resources that is handled by the Process subsystem. Note that
+ * this call does not terminate already running processes. */
+void
+process_free_all(void)
+{
+#ifdef _WIN32
+  process_win32_deinit();
+#endif
+
+  SMARTLIST_FOREACH(processes, process_t *, x, process_free(x));
+  smartlist_free(processes);
+}
+
+/** Get a list of all processes.  This function returns a smartlist of
+ * <b>process_t</b> containing all the currently allocated processes. */
+const smartlist_t *
+process_get_all_processes(void)
+{
+  return processes;
+}
+
+/** Allocate and initialize a new process.  This function returns a newly
+ * allocated and initialized process data, which can be used to configure and
+ * later run a subprocess of Tor. Use the various <b>process_set_*()</b>
+ * methods to configure it and run the process using <b>process_exec()</b>. Use
+ * <b>command</b> to specify the path to the command to run. You can either
+ * specify an absolute path to the command or relative where Tor will use the
+ * underlying operating system's functionality for finding the command to run.
+ * */
+process_t *
+process_new(const char *command)
+{
+  tor_assert(command);
+
+  process_t *process;
+  process = tor_malloc_zero(sizeof(process_t));
+
+  /* Set our command. */
+  process->command = tor_strdup(command);
+
+  /* By default we are not running. */
+  process->status = PROCESS_STATUS_NOT_RUNNING;
+
+  /* Prepare process environment. */
+  process->arguments = smartlist_new();
+  process->environment = smartlist_new();
+
+  /* Prepare the buffers. */
+  process->stdout_buffer = buf_new();
+  process->stderr_buffer = buf_new();
+  process->stdin_buffer = buf_new();
+
+#ifndef _WIN32
+  /* Prepare our Unix process handle. */
+  process->unix_process = process_unix_new();
+#else
+  /* Prepare our Win32 process handle. */
+  process->win32_process = process_win32_new();
+#endif /* !defined(_WIN32) */
+
+  smartlist_add(processes, process);
+
+  return process;
+}
+
+/** Deallocate the given process in <b>process</b>. */
+void
+process_free_(process_t *process)
+{
+  if (! process)
+    return;
+
+  /* Cleanup parameters. */
+  tor_free(process->command);
+
+  /* Cleanup arguments and environment. */
+  SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x));
+  smartlist_free(process->arguments);
+
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+
+  /* Cleanup the buffers. */
+  buf_free(process->stdout_buffer);
+  buf_free(process->stderr_buffer);
+  buf_free(process->stdin_buffer);
+
+#ifndef _WIN32
+  /* Cleanup our Unix process handle. */
+  process_unix_free(process->unix_process);
+#else
+  /* Cleanup our Win32 process handle. */
+  process_win32_free(process->win32_process);
+#endif /* !defined(_WIN32) */
+
+  smartlist_remove(processes, process);
+
+  tor_free(process);
+}
+
+/** Execute the given process.  This function executes the given process as a
+ * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_exec(process_t *process)
+{
+  tor_assert(process);
+
+  if (BUG(may_spawn_background_process == 0))
+    return PROCESS_STATUS_ERROR;
+
+  process_status_t status = PROCESS_STATUS_NOT_RUNNING;
+
+  log_info(LD_PROCESS, "Starting new process: %s", process->command);
+
+#ifndef _WIN32
+  status = process_unix_exec(process);
+#else
+  status = process_win32_exec(process);
+#endif
+
+  /* Update our state. */
+  process_set_status(process, status);
+
+  if (status != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_PROCESS, "Failed to start process: %s",
+             process_get_command(process));
+  }
+
+  return status;
+}
+
+/** Terminate the given process. Returns true on success,
+ * otherwise false. */
+bool
+process_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  /* Terminating a non-running process isn't going to work. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  log_debug(LD_PROCESS, "Terminating process");
+
+#ifndef _WIN32
+  return process_unix_terminate(process);
+#else
+  return process_win32_terminate(process);
+#endif
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+#ifndef _WIN32
+  return process_unix_get_pid(process);
+#else
+  return process_win32_get_pid(process);
+#endif
+}
+
+/** Set the callback function for output from the child process's standard out
+ * handle.  This function sets the callback function which is called every time
+ * the child process have written output to its standard out file handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stdout_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stdout_read_callback = callback;
+}
+
+/** Set the callback function for output from the child process's standard
+ * error handle.  This function sets the callback function which is called
+ * every time the child process have written output to its standard error file
+ * handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stderr_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stderr_read_callback = callback;
+}
+
+/** Set the callback function for process exit notification.  The
+ * <b>callback</b> function will be called every time your child process have
+ * terminated. */
+void
+process_set_exit_callback(process_t *process,
+                          process_exit_callback_t callback)
+{
+  tor_assert(process);
+  process->exit_callback = callback;
+}
+
+/** Get the current command of the given process. */
+const char *
+process_get_command(const process_t *process)
+{
+  tor_assert(process);
+  return process->command;
+}
+
+void
+process_set_protocol(process_t *process, process_protocol_t protocol)
+{
+  tor_assert(process);
+  process->protocol = protocol;
+}
+
+/** Get the currently used protocol of the given process. */
+process_protocol_t
+process_get_protocol(const process_t *process)
+{
+  tor_assert(process);
+  return process->protocol;
+}
+
+/** Set opague pointer to data.  This function allows you to store a pointer to
+ * your own data in the given process. Use <b>process_get_data()</b> in the
+ * various callback functions to retrieve the data again.
+ *
+ * Note that the given process does NOT take ownership of the data and you are
+ * responsible for freeing up any resources allocated by the given data.
+ * */
+void
+process_set_data(process_t *process, void *data)
+{
+  tor_assert(process);
+  process->data = data;
+}
+
+/** Get the opaque pointer to callback data from the given process.  This
+ * function allows you get the data you stored with <b>process_set_data()</b>
+ * in the different callback functions. */
+void *
+process_get_data(const process_t *process)
+{
+  tor_assert(process);
+  return process->data;
+}
+
+/** Set the status of a given process. */
+void
+process_set_status(process_t *process, process_status_t status)
+{
+  tor_assert(process);
+  process->status = status;
+}
+
+/** Get the status of the given process. */
+process_status_t
+process_get_status(const process_t *process)
+{
+  tor_assert(process);
+  return process->status;
+}
+
+/** Append an argument to the list of arguments in the given process. */
+void
+process_append_argument(process_t *process, const char *argument)
+{
+  tor_assert(process);
+  tor_assert(argument);
+
+  smartlist_add(process->arguments, tor_strdup(argument));
+}
+
+/** Returns a list of arguments (excluding the command itself) from the
+ * given process. */
+const smartlist_t *
+process_get_arguments(const process_t *process)
+{
+  tor_assert(process);
+  return process->arguments;
+}
+
+/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b>
+ * to deallocate it after use. */
+char **
+process_get_argv(const process_t *process)
+{
+  tor_assert(process);
+
+  /** Generate a Unix style process argument vector from our process's
+   * arguments smartlist_t. */
+  char **argv = NULL;
+
+  char *filename = process->command;
+  const smartlist_t *arguments = process->arguments;
+  const size_t size = smartlist_len(arguments);
+
+  /* Make space for the process filename as argv[0] and a trailing NULL. */
+  argv = tor_malloc_zero(sizeof(char *) * (size + 2));
+
+  /* Set our filename as first argument. */
+  argv[0] = filename;
+
+  /* Put in the rest of the values from arguments. */
+  SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) {
+    tor_assert(arg_val != NULL);
+
+    argv[arg_val_sl_idx + 1] = arg_val;
+  } SMARTLIST_FOREACH_END(arg_val);
+
+  return argv;
+}
+
+/** This function clears the internal environment and copies over every string
+ * from <b>env</b> as the new environment. */
+void
+process_reset_environment(process_t *process, const smartlist_t *env)
+{
+  tor_assert(process);
+  tor_assert(env);
+
+  /* Cleanup old environment. */
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+  process->environment = smartlist_new();
+
+  SMARTLIST_FOREACH(env, char *, x,
+                    smartlist_add(process->environment, tor_strdup(x)));
+}
+
+/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the
+ * given process. */
+void
+process_set_environment(process_t *process,
+                        const char *key,
+                        const char *value)
+{
+  tor_assert(process);
+  tor_assert(key);
+  tor_assert(value);
+
+  smartlist_add_asprintf(process->environment, "%s=%s", key, value);
+}
+
+/** Returns a newly allocated <b>process_environment_t</b> containing the
+ * environment variables for the given process. */
+process_environment_t *
+process_get_environment(const process_t *process)
+{
+  tor_assert(process);
+  return process_environment_make(process->environment);
+}
+
+#ifndef _WIN32
+/** Get the internal handle for the Unix backend. */
+process_unix_t *
+process_get_unix_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->unix_process);
+  return process->unix_process;
+}
+#else /* defined(_WIN32) */
+/** Get the internal handle for Windows backend. */
+process_win32_t *
+process_get_win32_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->win32_process);
+  return process->win32_process;
+}
+#endif /* !defined(_WIN32) */
+
+/** Write <b>size</b> bytes of <b>data</b> to the given process's standard
+ * input. */
+void
+process_write(process_t *process,
+              const uint8_t *data, size_t size)
+{
+  tor_assert(process);
+  tor_assert(data);
+
+  buf_add(process->stdin_buffer, (char *)data, size);
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** As tor_vsnprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_vprintf(process_t *process,
+                const char *format, va_list args)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  int size;
+  char *data;
+
+  size = tor_vasprintf(&data, format, args);
+  process_write(process, (uint8_t *)data, size);
+  tor_free(data);
+}
+
+/** As tor_snprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_printf(process_t *process,
+               const char *format, ...)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  va_list ap;
+  va_start(ap, format);
+  process_vprintf(process, format, ap);
+  va_end(ap);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard output
+ * handle. */
+void
+process_notify_event_stdout(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stdout(process, process->stdout_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stdout_buffer,
+                      process->stdout_read_callback);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard error
+ * handle. */
+void
+process_notify_event_stderr(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stderr(process, process->stderr_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stderr_buffer,
+                      process->stderr_read_callback);
+}
+
+/** This function is called by the Process backend when a given process is
+ * allowed to begin writing data to the standard input of the child process. */
+void
+process_notify_event_stdin(process_t *process)
+{
+  tor_assert(process);
+
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** This function is called by the Process backend when a given process have
+ * terminated. The exit status code is passed in <b>exit_code</b>. We mark the
+ * process as no longer running and calls the <b>exit_callback</b> with
+ * information about the process termination. The given <b>process</b> is
+ * free'd iff the exit_callback returns true. */
+void
+process_notify_event_exit(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_debug(LD_PROCESS,
+            "Process terminated with exit code: %"PRIu64, exit_code);
+
+  /* Update our state. */
+  process_set_status(process, PROCESS_STATUS_NOT_RUNNING);
+  process->exit_code = exit_code;
+
+  /* Call our exit callback, if it exists. */
+  bool free_process_handle = false;
+
+  /* The exit callback will tell us if we should process_free() our handle. */
+  if (process->exit_callback)
+    free_process_handle = process->exit_callback(process, exit_code);
+
+  if (free_process_handle)
+    process_free(process);
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard out handle. Returns the number of
+ * bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stdout(process, buffer);
+#else
+  return process_win32_read_stdout(process, buffer);
+#endif
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard error handle. Returns the number
+ * of bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stderr(process, buffer);
+#else
+  return process_win32_read_stderr(process, buffer);
+#endif
+}
+
+/** This function calls the backend function for the given process whenever
+ * there is data to be written to the backends' file handles. */
+MOCK_IMPL(STATIC void, process_write_stdin,
+          (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  process_unix_write(process, buffer);
+#else
+  process_win32_write(process, buffer);
+#endif
+}
+
+/** This function calls the protocol handlers based on the value of
+ * <b>process_get_protocol(process)</b>. Currently we call
+ * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and
+ * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */
+STATIC void
+process_read_data(process_t *process,
+                  buf_t *buffer,
+                  process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  switch (process_get_protocol(process)) {
+  case PROCESS_PROTOCOL_RAW:
+    process_read_buffer(process, buffer, callback);
+    break;
+  case PROCESS_PROTOCOL_LINE:
+    process_read_lines(process, buffer, callback);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_unreached();
+    return;
+    /* LCOV_EXCL_STOP */
+  }
+}
+
+/** This function takes the content of the given <b>buffer</b> and passes it to
+ * the given <b>callback</b> function, but ensures that an additional zero byte
+ * is added to the end of the data such that the given callback implementation
+ * can threat the content as a ASCIIZ string. */
+STATIC void
+process_read_buffer(process_t *process,
+                    buf_t *buffer,
+                    process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer);
+
+  /* We allocate an extra byte for the zero byte in the end. */
+  char *data = tor_malloc_zero(size + 1);
+
+  buf_get_bytes(buffer, data, size);
+  log_debug(LD_PROCESS, "Read data from process");
+
+  if (callback)
+    callback(process, data, size);
+
+  tor_free(data);
+}
+
+/** This function tries to extract complete lines from the given <b>buffer</b>
+ * and calls the given <b>callback</b> function whenever it has a complete
+ * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n"
+ * from the line. If we are unable to extract a complete line we leave the data
+ * in the buffer for next call. */
+STATIC void
+process_read_lines(process_t *process,
+                   buf_t *buffer,
+                   process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer) + 1;
+  size_t line_size = 0;
+  char *data = tor_malloc_zero(size);
+  int ret;
+
+  while (true) {
+    line_size = size;
+    ret = buf_get_line(buffer, data, &line_size);
+
+    /* A complete line should always be smaller than the size of our
+     * buffer. */
+    tor_assert(ret != -1);
+
+    /* Remove \n from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\n') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    /* Remove \r from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\r') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    if (ret == 1) {
+      log_debug(LD_PROCESS, "Read line from process: \"%s\"", data);
+
+      if (callback)
+        callback(process, data, line_size);
+
+      /* We have read a whole line, let's see if there is more lines to read.
+       * */
+      continue;
+    }
+
+    /* No complete line for us to read. We are done for now. */
+    tor_assert_nonfatal(ret == 0);
+    break;
+  }
+
+  tor_free(data);
+}
diff --git a/src/lib/process/process.h b/src/lib/process/process.h
new file mode 100644
index 0000000000..05c091a5bf
--- /dev/null
+++ b/src/lib/process/process.h
@@ -0,0 +1,145 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.h
+ * \brief Header for process.c
+ **/
+
+#ifndef TOR_PROCESS_H
+#define TOR_PROCESS_H
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+/** Maximum number of bytes to write to a process' stdin. */
+#define PROCESS_MAX_WRITE (1024)
+
+/** Maximum number of bytes to read from a process' stdout/stderr. */
+#define PROCESS_MAX_READ  (1024)
+
+typedef enum {
+  /** The process is not running. */
+  PROCESS_STATUS_NOT_RUNNING,
+
+  /** The process is running. */
+  PROCESS_STATUS_RUNNING,
+
+  /** The process is in an erroneous state. */
+  PROCESS_STATUS_ERROR
+} process_status_t;
+
+const char *process_status_to_string(process_status_t status);
+
+typedef enum {
+  /** Pass complete \n-terminated lines to the
+   *  callback (with the \n or \r\n removed). */
+  PROCESS_PROTOCOL_LINE,
+
+  /** Pass the raw response from read() to the callback. */
+  PROCESS_PROTOCOL_RAW
+} process_protocol_t;
+
+const char *process_protocol_to_string(process_protocol_t protocol);
+
+void tor_disable_spawning_background_processes(void);
+
+struct smartlist_t;
+
+struct process_t;
+typedef struct process_t process_t;
+
+typedef uint64_t process_exit_code_t;
+typedef uint64_t process_pid_t;
+
+typedef void (*process_read_callback_t)(process_t *,
+                                        const char *,
+                                        size_t);
+typedef bool
+(*process_exit_callback_t)(process_t *, process_exit_code_t);
+
+void process_init(void);
+void process_free_all(void);
+const struct smartlist_t *process_get_all_processes(void);
+
+process_t *process_new(const char *command);
+void process_free_(process_t *process);
+#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s))
+
+process_status_t process_exec(process_t *process);
+bool process_terminate(process_t *process);
+
+process_pid_t process_get_pid(process_t *process);
+
+void process_set_stdout_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_stderr_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_exit_callback(process_t *,
+                               process_exit_callback_t);
+
+const char *process_get_command(const process_t *process);
+
+void process_append_argument(process_t *process, const char *argument);
+const struct smartlist_t *process_get_arguments(const process_t *process);
+char **process_get_argv(const process_t *process);
+
+void process_reset_environment(process_t *process,
+                               const struct smartlist_t *env);
+void process_set_environment(process_t *process,
+                             const char *key,
+                             const char *value);
+
+struct process_environment_t;
+struct process_environment_t *process_get_environment(const process_t *);
+
+void process_set_protocol(process_t *process, process_protocol_t protocol);
+process_protocol_t process_get_protocol(const process_t *process);
+
+void process_set_data(process_t *process, void *data);
+void *process_get_data(const process_t *process);
+
+void process_set_status(process_t *process, process_status_t status);
+process_status_t process_get_status(const process_t *process);
+
+#ifndef _WIN32
+struct process_unix_t;
+struct process_unix_t *process_get_unix_process(const process_t *process);
+#else
+struct process_win32_t;
+struct process_win32_t *process_get_win32_process(const process_t *process);
+#endif /* !defined(_WIN32) */
+
+void process_write(process_t *process,
+                   const uint8_t *data, size_t size);
+void process_vprintf(process_t *process,
+                     const char *format, va_list args) CHECK_PRINTF(2, 0);
+void process_printf(process_t *process,
+                    const char *format, ...) CHECK_PRINTF(2, 3);
+
+void process_notify_event_stdout(process_t *process);
+void process_notify_event_stderr(process_t *process);
+void process_notify_event_stdin(process_t *process);
+void process_notify_event_exit(process_t *process,
+                               process_exit_code_t);
+
+#ifdef PROCESS_PRIVATE
+MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *));
+MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *));
+MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *));
+
+STATIC void process_read_data(process_t *process,
+                              buf_t *buffer,
+                              process_read_callback_t callback);
+STATIC void process_read_buffer(process_t *process,
+                                buf_t *buffer,
+                                process_read_callback_t callback);
+STATIC void process_read_lines(process_t *process,
+                               buf_t *buffer,
+                               process_read_callback_t callback);
+#endif /* defined(PROCESS_PRIVATE) */
+
+#endif /* !defined(TOR_PROCESS_H) */
diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c
new file mode 100644
index 0000000000..3c809a00e8
--- /dev/null
+++ b/src/lib/process/process_sys.c
@@ -0,0 +1,33 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.c
+ * \brief Subsystem object for process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/process/process_sys.h"
+#include "lib/process/process.h"
+
+static int
+subsys_process_initialize(void)
+{
+  process_init();
+  return 0;
+}
+
+static void
+subsys_process_shutdown(void)
+{
+  process_free_all();
+}
+
+const subsys_fns_t sys_process = {
+  .name = "process",
+  .level = -35,
+  .supported = true,
+  .initialize = subsys_process_initialize,
+  .shutdown = subsys_process_shutdown
+};
diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h
new file mode 100644
index 0000000000..b7a116d838
--- /dev/null
+++ b/src/lib/process/process_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.h
+ * \brief Declare subsystem object for the process module.
+ **/
+
+#ifndef TOR_PROCESS_SYS_H
+#define TOR_PROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_process;
+
+#endif /* !defined(TOR_PROCESS_SYS_H) */
diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c
new file mode 100644
index 0000000000..8191bdc1f0
--- /dev/null
+++ b/src/lib/process/process_unix.c
@@ -0,0 +1,698 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.c
+ * \brief Module for working with Unix processes.
+ **/
+
+#define PROCESS_UNIX_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/waitpid.h"
+#include "lib/process/env.h"
+
+#include <stdio.h>
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+#include <sys/prctl.h>
+#endif
+
+#if HAVE_SIGNAL_H
+#include <signal.h>
+#endif
+
+#ifndef _WIN32
+
+/** Maximum number of file descriptors, if we cannot get it via sysconf() */
+#define DEFAULT_MAX_FD 256
+
+/** Internal state for Unix handles. */
+struct process_unix_handle_t {
+  /** Unix File Descriptor. */
+  int fd;
+
+  /** Have we reached end of file? */
+  bool reached_eof;
+
+  /** Event structure for libevent. */
+  struct event *event;
+
+  /** Are we writing? */
+  bool is_writing;
+};
+
+/** Internal state for our Unix process. */
+struct process_unix_t {
+  /** Standard in handle. */
+  process_unix_handle_t stdin_handle;
+
+  /** Standard out handle. */
+  process_unix_handle_t stdout_handle;
+
+  /** Standard error handle. */
+  process_unix_handle_t stderr_handle;
+
+  /** The process identifier of our process. */
+  pid_t pid;
+
+  /** Waitpid Callback structure. */
+  waitpid_callback_t *waitpid;
+};
+
+/** Returns a newly allocated <b>process_unix_t</b>. */
+process_unix_t *
+process_unix_new(void)
+{
+  process_unix_t *unix_process;
+  unix_process = tor_malloc_zero(sizeof(process_unix_t));
+
+  unix_process->stdin_handle.fd = -1;
+  unix_process->stderr_handle.fd = -1;
+  unix_process->stdout_handle.fd = -1;
+
+  return unix_process;
+}
+
+/** Deallocates the given <b>unix_process</b>. */
+void
+process_unix_free_(process_unix_t *unix_process)
+{
+  if (! unix_process)
+    return;
+
+  /* Clean up our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+
+  /* FIXME(ahf): Refactor waitpid code? */
+  unix_process->waitpid = NULL;
+
+  /* Close all our file descriptors. */
+  process_unix_close_file_descriptors(unix_process);
+
+  tor_event_free(unix_process->stdout_handle.event);
+  tor_event_free(unix_process->stderr_handle.event);
+  tor_event_free(unix_process->stdin_handle.event);
+
+  tor_free(unix_process);
+}
+
+/** Executes the given process as a child process of Tor.  This function is
+ * responsible for setting up the child process and run it. This includes
+ * setting up pipes for interprocess communication, initialize the waitpid
+ * callbacks, and finally run fork() followed by execve(). Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_unix_exec(process_t *process)
+{
+  static int max_fd = -1;
+
+  process_unix_t *unix_process;
+  pid_t pid;
+  int stdin_pipe[2];
+  int stdout_pipe[2];
+  int stderr_pipe[2];
+  int retval, fd;
+
+  unix_process = process_get_unix_process(process);
+
+  /* Create standard in pipe. */
+  retval = pipe(stdin_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdin "
+             "communication with process: %s",
+             strerror(errno));
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard out pipe. */
+  retval = pipe(stdout_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdout "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard error pipe. */
+  retval = pipe(stderr_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stderr "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+#ifdef _SC_OPEN_MAX
+  if (-1 == max_fd) {
+    max_fd = (int)sysconf(_SC_OPEN_MAX);
+
+    if (max_fd == -1) {
+      max_fd = DEFAULT_MAX_FD;
+      log_warn(LD_PROCESS,
+               "Cannot find maximum file descriptor, assuming: %d", max_fd);
+    }
+  }
+#else /* !defined(_SC_OPEN_MAX) */
+  max_fd = DEFAULT_MAX_FD;
+#endif /* defined(_SC_OPEN_MAX) */
+
+  pid = fork();
+
+  if (0 == pid) {
+    /* This code is running in the child process context. */
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+    /* Attempt to have the kernel issue a SIGTERM if the parent
+     * goes away. Certain attributes of the binary being execve()ed
+     * will clear this during the execve() call, but it's better
+     * than nothing.
+     */
+    prctl(PR_SET_PDEATHSIG, SIGTERM);
+#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
+
+    /* Link process stdout to the write end of the pipe. */
+    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stderr to the write end of the pipe. */
+    retval = dup2(stderr_pipe[1], STDERR_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stdin to the read end of the pipe */
+    retval = dup2(stdin_pipe[0], STDIN_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Close our pipes now after they have been dup2()'ed. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /* Close all other fds, including the read end of the pipe.  XXX: We should
+     * now be doing enough FD_CLOEXEC setting to make this needless.
+     */
+    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++)
+      close(fd);
+
+    /* Create the argv value for our new process. */
+    char **argv = process_get_argv(process);
+
+    /* Create the env value for our new process. */
+    process_environment_t *env = process_get_environment(process);
+
+    /* Call the requested program. */
+    execve(argv[0], argv, env->unixoid_environment_block);
+
+    /* If we made it here it is because execve failed :-( */
+    tor_free(argv);
+    process_environment_free(env);
+
+ error:
+    fprintf(stderr, "Error from child process: %s", strerror(errno));
+    _exit(1);
+  }
+
+  /* We are in the parent process. */
+  if (-1 == pid) {
+    log_warn(LD_PROCESS,
+             "Failed to create child process: %s", strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    /** Cleanup standard error pipe. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Register our PID. */
+  unix_process->pid = pid;
+
+  /* Setup waitpid callbacks. */
+  unix_process->waitpid = set_waitpid_callback(pid,
+                                               process_unix_waitpid_callback,
+                                               process);
+
+  /* Handle standard out. */
+  unix_process->stdout_handle.fd = stdout_pipe[0];
+  retval = close(stdout_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard error. */
+  unix_process->stderr_handle.fd = stderr_pipe[0];
+  retval = close(stderr_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Failed to close write end of standard error pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard in. */
+  unix_process->stdin_handle.fd = stdin_pipe[1];
+  retval = close(stdin_pipe[0]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s",
+             strerror(errno));
+  }
+
+  /* Setup our handles. */
+  process_unix_setup_handle(process,
+                            &unix_process->stdout_handle,
+                            EV_READ|EV_PERSIST,
+                            stdout_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stderr_handle,
+                            EV_READ|EV_PERSIST,
+                            stderr_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stdin_handle,
+                            EV_WRITE|EV_PERSIST,
+                            stdin_write_callback);
+
+  /* Start reading from standard out and standard error. */
+  process_unix_start_reading(&unix_process->stdout_handle);
+  process_unix_start_reading(&unix_process->stderr_handle);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_unix_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* All running processes should have a waitpid. */
+  if (BUG(unix_process->waitpid == NULL))
+    return false;
+
+  bool success = true;
+
+  /* Send a SIGTERM to our child process. */
+  int ret;
+
+  ret = kill(unix_process->pid, SIGTERM);
+
+  if (ret == -1) {
+    log_warn(LD_PROCESS, "Unable to terminate process: %s",
+             strerror(errno));
+    success = false;
+  }
+
+  /* Close all our FD's. */
+  if (! process_unix_close_file_descriptors(unix_process))
+    success = false;
+
+  return success;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_unix_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+  return (process_pid_t)unix_process->pid;
+}
+
+/** Write the given <b>buffer</b> as input to the given <b>process</b>'s
+ * standard input. Returns the number of bytes written. */
+int
+process_unix_write(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  size_t buffer_flush_len = buf_datalen(buffer);
+  const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len);
+
+  /* If we have data to write (when buffer_flush_len > 0) and we are not
+   * currently getting file descriptor events from the kernel, we tell the
+   * kernel to start notifying us about when we can write to our file
+   * descriptor and return. */
+  if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) {
+    process_unix_start_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We don't have any data to write, but the kernel is currently notifying us
+   * about whether we are able to write or not. Tell the kernel to stop
+   * notifying us until we have data to write. */
+  if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) {
+    process_unix_stop_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We have data to write and the kernel have told us to write it. */
+  return buf_flush_to_pipe(buffer,
+                           process_get_unix_process(process)->stdin_handle.fd,
+                           max_to_write, &buffer_flush_len);
+}
+
+/** Read data from the given process's standard output and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stdout(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stdout_handle,
+                                  buffer);
+}
+
+/** Read data from the given process's standard error and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stderr(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stderr_handle,
+                                  buffer);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard output. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stdout_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdout(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard error. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stderr_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stderr(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * written the child process's standard input. We notify the Process subsystem,
+ * which is then responsible for calling back to us for doing the actual write
+ * of the data. */
+STATIC void
+stdin_write_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdin(process);
+}
+
+/** This function tells libevent that we are interested in receiving read
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * read events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+}
+
+/** This function tells libevent that we are interested in receiving write
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+
+  handle->is_writing = true;
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * write events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+
+  handle->is_writing = false;
+}
+
+/** This function is called when the waitpid system have detected that our
+ * process have terminated. We disable the waitpid system and notify the
+ * Process subsystem that we have terminated. */
+STATIC void
+process_unix_waitpid_callback(int status, void *data)
+{
+  tor_assert(data);
+
+  process_t *process = data;
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* Remove our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+  unix_process->waitpid = NULL;
+
+  /* Notify our process. */
+  process_notify_event_exit(process, status);
+
+  /* Make sure you don't modify the process after we have called
+   * process_notify_event_exit() on it, to allow users to process_free() it in
+   * the exit callback. */
+}
+
+/** This function sets the file descriptor in the <b>handle</b> as non-blocking
+ * and configures the libevent event structure based on the given <b>flags</b>
+ * to ensure that <b>callback</b> is called whenever we have events on the
+ * given <b>handle</b>. */
+STATIC void
+process_unix_setup_handle(process_t *process,
+                          process_unix_handle_t *handle,
+                          short flags,
+                          event_callback_fn callback)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(callback);
+
+  /* Put our file descriptor into non-blocking mode. */
+  if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) {
+    log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s",
+             strerror(errno));
+  }
+
+  /* Setup libevent event. */
+  handle->event = tor_event_new(tor_libevent_get_base(),
+                                handle->fd,
+                                flags,
+                                callback,
+                                process);
+}
+
+/** This function reads data from the given <b>handle</b> and puts it into
+ * <b>buffer</b>. Returns the number of bytes read this way. */
+STATIC int
+process_unix_read_handle(process_t *process,
+                         process_unix_handle_t *handle,
+                         buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(buffer);
+
+  int ret = 0;
+  int eof = 0;
+  int error = 0;
+
+  ret = buf_read_from_pipe(buffer,
+                           handle->fd,
+                           PROCESS_MAX_READ,
+                           &eof,
+                           &error);
+
+  if (error)
+    log_warn(LD_PROCESS,
+             "Unable to read data: %s", strerror(error));
+
+  if (eof) {
+    handle->reached_eof = true;
+    process_unix_stop_reading(handle);
+  }
+
+  return ret;
+}
+
+/** Close the standard in, out, and error handles of the given
+ * <b>unix_process</b>. */
+STATIC bool
+process_unix_close_file_descriptors(process_unix_t *unix_process)
+{
+  tor_assert(unix_process);
+
+  int ret;
+  bool success = true;
+
+  /* Stop reading and writing before we close() our
+   * file descriptors. */
+  if (! unix_process->stdout_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stdout_handle);
+
+  if (! unix_process->stderr_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stderr_handle);
+
+  if (unix_process->stdin_handle.is_writing)
+    process_unix_stop_writing(&unix_process->stdin_handle);
+
+  if (unix_process->stdin_handle.fd != -1) {
+    ret = close(unix_process->stdin_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard in");
+      success = false;
+    }
+
+    unix_process->stdin_handle.fd = -1;
+  }
+
+  if (unix_process->stdout_handle.fd != -1) {
+    ret = close(unix_process->stdout_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard out");
+      success = false;
+    }
+
+    unix_process->stdout_handle.fd = -1;
+  }
+
+  if (unix_process->stderr_handle.fd != -1) {
+    ret = close(unix_process->stderr_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard error");
+      success = false;
+    }
+
+    unix_process->stderr_handle.fd = -1;
+  }
+
+  return success;
+}
+
+#endif /* !defined(_WIN32) */
diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h
new file mode 100644
index 0000000000..da40b3e567
--- /dev/null
+++ b/src/lib/process/process_unix.h
@@ -0,0 +1,68 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.h
+ * \brief Header for process_unix.c
+ **/
+
+#ifndef TOR_PROCESS_UNIX_H
+#define TOR_PROCESS_UNIX_H
+
+#ifndef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+
+#include <event2/event.h>
+
+struct process_t;
+
+struct process_unix_t;
+typedef struct process_unix_t process_unix_t;
+
+process_unix_t *process_unix_new(void);
+void process_unix_free_(process_unix_t *unix_process);
+#define process_unix_free(s) \
+  FREE_AND_NULL(process_unix_t, process_unix_free_, (s))
+
+process_status_t process_unix_exec(struct process_t *process);
+bool process_unix_terminate(struct process_t *process);
+
+process_pid_t process_unix_get_pid(struct process_t *process);
+
+int process_unix_write(struct process_t *process, buf_t *buffer);
+int process_unix_read_stdout(struct process_t *process, buf_t *buffer);
+int process_unix_read_stderr(struct process_t *process, buf_t *buffer);
+
+#ifdef PROCESS_UNIX_PRIVATE
+struct process_unix_handle_t;
+typedef struct process_unix_handle_t process_unix_handle_t;
+
+STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data);
+
+STATIC void process_unix_start_reading(process_unix_handle_t *);
+STATIC void process_unix_stop_reading(process_unix_handle_t *);
+
+STATIC void process_unix_start_writing(process_unix_handle_t *);
+STATIC void process_unix_stop_writing(process_unix_handle_t *);
+
+STATIC void process_unix_waitpid_callback(int status, void *data);
+
+STATIC void process_unix_setup_handle(process_t *process,
+                                      process_unix_handle_t *handle,
+                                      short flags,
+                                      event_callback_fn callback);
+STATIC int process_unix_read_handle(process_t *,
+                                    process_unix_handle_t *,
+                                    buf_t *);
+STATIC bool process_unix_close_file_descriptors(process_unix_t *);
+#endif /* defined(PROCESS_UNIX_PRIVATE) */
+
+#endif /* !defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_UNIX_H) */
diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c
new file mode 100644
index 0000000000..7e4082ad13
--- /dev/null
+++ b/src/lib/process/process_win32.c
@@ -0,0 +1,1105 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.c
+ * \brief Module for working with Windows processes.
+ **/
+
+#define PROCESS_WIN32_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/process/process.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef _WIN32
+
+/** The size of our intermediate buffers. */
+#define BUFFER_SIZE (1024)
+
+/** Timer that ticks once a second and calls the process_win32_timer_callback()
+ * function. */
+static periodic_timer_t *periodic_timer;
+
+/** Structure to represent the state around the pipe HANDLE.
+ *
+ * This structure is used to store state about a given HANDLE, including
+ * whether we have reached end of file, its intermediate buffers, and how much
+ * data that is available in the intermediate buffer. */
+struct process_win32_handle_t {
+  /** Standard out pipe handle. */
+  HANDLE pipe;
+
+  /** True iff we have reached EOF from the pipe. */
+  bool reached_eof;
+
+  /** How much data is available in buffer. */
+  size_t data_available;
+
+  /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */
+  char buffer[BUFFER_SIZE];
+
+  /** Overlapped structure for ReadFileEx() and WriteFileEx(). */
+  OVERLAPPED overlapped;
+
+  /** Are we waiting for another I/O operation to complete? */
+  bool busy;
+};
+
+/** Structure to represent the Windows specific implementation details of this
+ * Process backend.
+ *
+ * This structure is attached to <b>process_t</b> (see process.h) and is
+ * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b>
+ * method. */
+struct process_win32_t {
+  /** Standard in state. */
+  process_win32_handle_t stdin_handle;
+
+  /** Standard out state. */
+  process_win32_handle_t stdout_handle;
+
+  /** Standard error state. */
+  process_win32_handle_t stderr_handle;
+
+  /** Process Information. */
+  PROCESS_INFORMATION process_information;
+};
+
+/** Create a new <b>process_win32_t</b>.
+ *
+ * This function constructs a new <b>process_win32_t</b> and initializes the
+ * default values. */
+process_win32_t *
+process_win32_new(void)
+{
+  process_win32_t *win32_process;
+  win32_process = tor_malloc_zero(sizeof(process_win32_t));
+
+  win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE;
+
+  return win32_process;
+}
+
+/** Free a given <b>process_win32_t</b>.
+ *
+ * This function deinitializes and frees up the resources allocated for the
+ * given <b>process_win32_t</b>. */
+void
+process_win32_free_(process_win32_t *win32_process)
+{
+  if (! win32_process)
+    return;
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  tor_free(win32_process);
+}
+
+/** Initialize the Windows backend of the Process subsystem. */
+void
+process_win32_init(void)
+{
+  /* We don't start the periodic timer here because it makes no sense to have
+   * the timer running until we have some processes that benefits from the
+   * timer timer ticks. */
+}
+
+/** Deinitialize the Windows backend of the Process subsystem. */
+void
+process_win32_deinit(void)
+{
+  /* Stop our timer, but only if it's running. */
+  if (process_win32_timer_running())
+    process_win32_timer_stop();
+}
+
+/** Execute the given process. This function is responsible for setting up
+ * named pipes for I/O between the child process and the Tor process. Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_win32_exec(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  HANDLE stdout_pipe_read = NULL;
+  HANDLE stdout_pipe_write = NULL;
+  HANDLE stderr_pipe_read = NULL;
+  HANDLE stderr_pipe_write = NULL;
+  HANDLE stdin_pipe_read = NULL;
+  HANDLE stdin_pipe_write = NULL;
+  BOOL ret = FALSE;
+
+  /* Setup our security attributes. */
+  SECURITY_ATTRIBUTES security_attributes;
+  security_attributes.nLength = sizeof(security_attributes);
+  security_attributes.bInheritHandle = TRUE;
+  /* FIXME: should we set explicit security attributes?
+   * (See Ticket #2046, comment 5) */
+  security_attributes.lpSecurityDescriptor = NULL;
+
+  /* Create our standard out pipe. */
+  if (! process_win32_create_pipe(&stdout_pipe_read,
+                                  &stdout_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create our standard error pipe. */
+  if (! process_win32_create_pipe(&stderr_pipe_read,
+                                  &stderr_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create out standard in pipe. */
+  if (! process_win32_create_pipe(&stdin_pipe_read,
+                                  &stdin_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_WRITER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Configure startup info for our child process. */
+  STARTUPINFOA startup_info;
+
+  memset(&startup_info, 0, sizeof(startup_info));
+  startup_info.cb = sizeof(startup_info);
+  startup_info.hStdError = stderr_pipe_write;
+  startup_info.hStdOutput = stdout_pipe_write;
+  startup_info.hStdInput = stdin_pipe_read;
+  startup_info.dwFlags |= STARTF_USESTDHANDLES;
+
+  /* Create the env value for our new process. */
+  process_environment_t *env = process_get_environment(process);
+
+  /* Create the argv value for our new process. */
+  char **argv = process_get_argv(process);
+
+  /* Windows expects argv to be a whitespace delimited string, so join argv up
+   */
+  char *joined_argv = tor_join_win_cmdline((const char **)argv);
+
+  /* Create the child process */
+  ret = CreateProcessA(NULL,
+                       joined_argv,
+                       NULL,
+                       NULL,
+                       TRUE,
+                       CREATE_NO_WINDOW,
+                       env->windows_environment_block[0] == '\0' ?
+                         NULL : env->windows_environment_block,
+                       NULL,
+                       &startup_info,
+                       &win32_process->process_information);
+
+  tor_free(argv);
+  tor_free(joined_argv);
+  process_environment_free(env);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "CreateProcessA() failed: %s",
+      format_win32_error(GetLastError()));
+
+    /* Cleanup our handles. */
+    CloseHandle(stdout_pipe_read);
+    CloseHandle(stdout_pipe_write);
+    CloseHandle(stderr_pipe_read);
+    CloseHandle(stderr_pipe_write);
+    CloseHandle(stdin_pipe_read);
+    CloseHandle(stdin_pipe_write);
+
+    /* In the Unix backend, we do not get an error in the Tor process when a
+     * child process fails to spawn its target executable since we need to
+     * first do the fork() call in the Tor process and then the child process
+     * is responsible for doing the call to execve().
+     *
+     * This means that the user of the process_exec() API must check for
+     * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on
+     * Unix, but will happen for error cases on Windows where it does not
+     * happen on Unix. For example: when the target executable does not exist
+     * on the file system.
+     *
+     * To have somewhat feature compatibility between the Unix and the Windows
+     * backend, we here notify the process_t owner that the process have exited
+     * (even though it never managed to run) to ensure that the exit callback
+     * is executed.
+     */
+    process_notify_event_exit(process, 0);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* TODO: Should we close hProcess and hThread in
+   * process_handle->process_information? */
+  win32_process->stdout_handle.pipe = stdout_pipe_read;
+  win32_process->stderr_handle.pipe = stderr_pipe_read;
+  win32_process->stdin_handle.pipe = stdin_pipe_write;
+
+  /* Close our ends of the pipes that is now owned by the child process. */
+  CloseHandle(stdout_pipe_write);
+  CloseHandle(stderr_pipe_write);
+  CloseHandle(stdin_pipe_read);
+
+  /* Used by the callback functions from ReadFileEx() and WriteFileEx() such
+   * that we can figure out which process_t that was responsible for the event.
+   *
+   * Warning, here be dragons:
+   *
+   *   MSDN says that the hEvent member of the overlapped structure is unused
+   *   for ReadFileEx() and WriteFileEx, which allows us to store a pointer to
+   *   our process state there.
+   */
+  win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process;
+
+  /* Start our timer if it is not already running. */
+  if (! process_win32_timer_running())
+    process_win32_timer_start();
+
+  /* We use Windows Extended I/O functions, so our completion callbacks are
+   * called automatically for us when there is data to read. Because of this
+   * we start the read of standard out and error right away. */
+  process_notify_event_stdout(process);
+  process_notify_event_stderr(process);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_win32_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Terminate our process. */
+  BOOL ret;
+
+  ret = TerminateProcess(win32_process->process_information.hProcess, 0);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "TerminateProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  return true;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_win32_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  return (process_pid_t)win32_process->process_information.dwProcessId;
+}
+
+/** Schedule an async write of the data found in <b>buffer</b> for the given
+ * process.  This function runs an async write operation of the content of
+ * buffer, if we are not already waiting for a pending I/O request. Returns the
+ * number of bytes that Windows will hopefully write for us in the background.
+ * */
+int
+process_win32_write(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD error_code = 0;
+  const size_t buffer_size = buf_datalen(buffer);
+
+  /* Windows is still writing our buffer. */
+  if (win32_process->stdin_handle.busy)
+    return 0;
+
+  /* Nothing for us to do right now. */
+  if (buffer_size == 0)
+    return 0;
+
+  /* We have reached end of file already? */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return 0;
+
+  /* Figure out how much data we should read. */
+  const size_t write_size = MIN(buffer_size,
+                                sizeof(win32_process->stdin_handle.buffer));
+
+  /* Read data from the process_t buffer into our intermediate buffer. */
+  buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size);
+
+  /* Because of the slightly weird API for WriteFileEx() we must set this to 0
+   * before we call WriteFileEx() because WriteFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Schedule our write. */
+  ret = WriteFileEx(win32_process->stdin_handle.pipe,
+                    win32_process->stdin_handle.buffer,
+                    write_size,
+                    &win32_process->stdin_handle.overlapped,
+                    process_win32_stdin_write_done);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "WriteFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for WriteFileEx() we
+   * should check GetLastError() after a call to WriteFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* This cast should be safe since our buffer can maximum be BUFFER_SIZE
+   * large. */
+  return (int)write_size;
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard output of the child process
+ * and asks Windows to call process_win32_stdout_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stdout(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stdout_handle,
+                                        buffer,
+                                        process_win32_stdout_read_done);
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard error of the child process
+ * and asks Windows to call process_win32_stderr_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stderr(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stderr_handle,
+                                        buffer,
+                                        process_win32_stderr_read_done);
+}
+
+/** This function is responsible for moving the Tor process into what Microsoft
+ * calls an "alertable" state. Once the process is in an alertable state the
+ * Windows kernel will notify us when our background I/O requests have finished
+ * and the callbacks will be executed. */
+void
+process_win32_trigger_completion_callbacks(void)
+{
+  DWORD ret;
+
+  /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep
+   * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause
+   * the process to enter alertable state, where the Windows kernel will notify
+   * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which
+   * will cause our completion callbacks to be executed.
+   *
+   * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION
+   * if one or more I/O callbacks were executed. */
+  ret = SleepEx(0, TRUE);
+
+  /* Warn us if the function returned something we did not anticipate. */
+  if (ret != 0 && ret != WAIT_IO_COMPLETION) {
+    log_warn(LD_PROCESS, "SleepEx() returned %lu", ret);
+  }
+}
+
+/** Start the periodic timer which is reponsible for checking whether processes
+ * are still alive and to make sure that the Tor process is periodically being
+ * moved into an alertable state. */
+void
+process_win32_timer_start(void)
+{
+  /* Make sure we never start our timer if it's already running. */
+  if (BUG(process_win32_timer_running()))
+    return;
+
+  /* Wake up once a second. */
+  static const struct timeval interval = {1, 0};
+
+  log_info(LD_PROCESS, "Starting Windows Process I/O timer");
+  periodic_timer = periodic_timer_new(tor_libevent_get_base(),
+                                      &interval,
+                                      process_win32_timer_callback,
+                                      NULL);
+}
+
+/** Stops the periodic timer. */
+void
+process_win32_timer_stop(void)
+{
+  if (BUG(periodic_timer == NULL))
+    return;
+
+  log_info(LD_PROCESS, "Stopping Windows Process I/O timer");
+  periodic_timer_free(periodic_timer);
+}
+
+/** Returns true iff the periodic timer is running. */
+bool
+process_win32_timer_running(void)
+{
+  return periodic_timer != NULL;
+}
+
+/** This function is called whenever the periodic_timer ticks. The function is
+ * responsible for moving the Tor process into an alertable state once a second
+ * and checking for whether our child processes have terminated since the last
+ * tick. */
+STATIC void
+process_win32_timer_callback(periodic_timer_t *timer, void *data)
+{
+  tor_assert(timer == periodic_timer);
+  tor_assert(data == NULL);
+
+  /* Move the process into an alertable state. */
+  process_win32_trigger_completion_callbacks();
+
+  /* Check if our processes are still alive. */
+
+  /* Since the call to process_win32_timer_test_process() might call
+   * process_notify_event_exit() which again might call process_free() which
+   * updates the list of processes returned by process_get_all_processes() it
+   * is important here that we make sure to not touch the list of processes if
+   * the call to process_win32_timer_test_process() returns true. */
+  bool done;
+
+  do {
+    const smartlist_t *processes = process_get_all_processes();
+    done = true;
+
+    SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) {
+      /* If process_win32_timer_test_process() returns true, it means that
+       * smartlist_remove() might have been called on the list returned by
+       * process_get_all_processes(). We start the loop over again until we
+       * have a succesful run over the entire list where the list was not
+       * modified. */
+      if (process_win32_timer_test_process(process)) {
+        done = false;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(process);
+  } while (! done);
+}
+
+/** Test whether a given process is still alive. Notify the Process subsystem
+ * if our process have died. Returns true iff the given process have
+ * terminated. */
+STATIC bool
+process_win32_timer_test_process(process_t *process)
+{
+  tor_assert(process);
+
+  /* No need to look at processes that don't claim they are running. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD exit_code = 0;
+
+  /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error
+   * message until some time after the process have actually terminated. We
+   * make sure that our ReadFileEx() calls for the process have *all* returned
+   * and both standard out and error have been marked as EOF before we try to
+   * see if the process terminated.
+   *
+   * This ensures that we *never* call the exit callback of the `process_t`,
+   * which potentially ends up calling `process_free()` on our `process_t`,
+   * before all data have been received from the process.
+   *
+   * We do NOT have a check here for whether standard in reached EOF since
+   * standard in's WriteFileEx() function is only called on-demand when we have
+   * something to write and is thus usually not awaiting to finish any
+   * operations. If we WriteFileEx() to a file that has terminated we'll simply
+   * get an error from ReadFileEx() or its completion routine and move on with
+   * life.  */
+  if (! win32_process->stdout_handle.reached_eof)
+    return false;
+
+  if (! win32_process->stderr_handle.reached_eof)
+    return false;
+
+  /* We start by testing whether our process is still running. */
+  ret = GetExitCodeProcess(win32_process->process_information.hProcess,
+                           &exit_code);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Notify our process_t that our process have terminated. Since our
+   * exit_callback might decide to process_free() our process handle it is very
+   * important that we do not touch the process_t after the call to
+   * process_notify_event_exit(). */
+  if (exit_code != STILL_ACTIVE) {
+    process_notify_event_exit(process, exit_code);
+    return true;
+  }
+
+  return false;
+}
+
+/** Create a new overlapped named pipe. This function creates a new connected,
+ * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is
+ * succesful. Returns true on sucess, false on failure. */
+STATIC bool
+process_win32_create_pipe(HANDLE *read_pipe,
+                          HANDLE *write_pipe,
+                          SECURITY_ATTRIBUTES *attributes,
+                          process_win32_pipe_type_t pipe_type)
+{
+  tor_assert(read_pipe);
+  tor_assert(write_pipe);
+  tor_assert(attributes);
+
+  BOOL ret = FALSE;
+
+  /* Buffer size. */
+  const size_t size = 4096;
+
+  /* Our additional read/write modes that depends on which pipe type we are
+   * creating. */
+  DWORD read_mode = 0;
+  DWORD write_mode = 0;
+
+  /* Generate the unique pipe name. */
+  char pipe_name[MAX_PATH];
+  static DWORD process_id = 0;
+  static DWORD counter = 0;
+
+  if (process_id == 0)
+    process_id = GetCurrentProcessId();
+
+  tor_snprintf(pipe_name, sizeof(pipe_name),
+               "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu",
+               process_id, counter++);
+
+  /* Only one of our handles can be overlapped. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    read_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    write_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Setup our read and write handles. */
+  HANDLE read_handle;
+  HANDLE write_handle;
+
+  /* Create our named pipe. */
+  read_handle = CreateNamedPipeA(pipe_name,
+                                 (PIPE_ACCESS_INBOUND|read_mode),
+                                 (PIPE_TYPE_BYTE|PIPE_WAIT),
+                                 1,
+                                 size,
+                                 size,
+                                 1000,
+                                 attributes);
+
+  if (read_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Create our file in the pipe namespace. */
+  write_handle = CreateFileA(pipe_name,
+                             GENERIC_WRITE,
+                             0,
+                             attributes,
+                             OPEN_EXISTING,
+                             (FILE_ATTRIBUTE_NORMAL|write_mode),
+                             NULL);
+
+  if (write_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateFileA() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+
+    return false;
+  }
+
+  /* Set the inherit flag for our pipe. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "SetHandleInformation() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+    CloseHandle(write_handle);
+
+    return false;
+  }
+
+  /* Everything is good. */
+  *read_pipe = read_handle;
+  *write_pipe = write_handle;
+
+  return true;
+}
+
+/** Cleanup a given <b>handle</b>. */
+STATIC void
+process_win32_cleanup_handle(process_win32_handle_t *handle)
+{
+  tor_assert(handle);
+
+#if 0
+  BOOL ret;
+  DWORD error_code;
+
+  /* Cancel any pending I/O requests: This means that instead of getting
+   * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem
+   * like this is needed. */
+  ret = CancelIo(handle->pipe);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* There was no pending I/O requests for our handle. */
+    if (error_code != ERROR_NOT_FOUND) {
+      log_warn(LD_PROCESS, "CancelIo() failed: %s",
+               format_win32_error(error_code));
+    }
+  }
+#endif /* 0 */
+
+  /* Close our handle. */
+  if (handle->pipe != INVALID_HANDLE_VALUE) {
+    CloseHandle(handle->pipe);
+    handle->pipe = INVALID_HANDLE_VALUE;
+    handle->reached_eof = true;
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard output. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stdout_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stdout_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stdout(process);
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard error. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stderr_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stderr_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stderr(process);
+  }
+}
+
+/** This function is called when WriteFileEx() completes its background write
+ * to the child process's standard input. We notify the Process subsystem that
+ * it can write data to us again. */
+STATIC VOID WINAPI
+process_win32_stdin_write_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  (void)byte_count;
+
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  win32_process->stdin_handle.busy = false;
+
+  /* Check if we have been asked to write to the handle that have been marked
+   * as having reached EOF. */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return;
+
+  if (error_code == 0) {
+    /** Our data have been succesfully written. Clear our state and schedule
+     * the next write. */
+    win32_process->stdin_handle.data_available = 0;
+    memset(win32_process->stdin_handle.buffer, 0,
+           sizeof(win32_process->stdin_handle.buffer));
+
+    /* Schedule the next write. */
+    process_notify_event_stdin(process);
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our WriteFileEx() call was succesful, but we reached the end of our
+     * file.  We mark our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    win32_process->stdin_handle.reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from WriteFileEx(): %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+  }
+}
+
+/** This function reads data from the given <b>handle</b>'s internal buffer and
+ * moves it into the given <b>buffer</b>. Additionally, we start the next
+ * ReadFileEx() background operation with the given <b>callback</b> as
+ * completion callback. Returns the number of bytes written to the buffer. */
+STATIC int
+process_win32_read_from_handle(process_win32_handle_t *handle,
+                               buf_t *buffer,
+                               LPOVERLAPPED_COMPLETION_ROUTINE callback)
+{
+  tor_assert(handle);
+  tor_assert(buffer);
+  tor_assert(callback);
+
+  BOOL ret = FALSE;
+  int bytes_available = 0;
+  DWORD error_code = 0;
+
+  /* We already have a request to read data that isn't complete yet. */
+  if (BUG(handle->busy))
+    return 0;
+
+  /* Check if we have been asked to read from a handle that have already told
+   * us that we have reached the end of the file. */
+  if (BUG(handle->reached_eof))
+    return 0;
+
+  /* This cast should be safe since our buffer can be at maximum up to
+   * BUFFER_SIZE in size. */
+  bytes_available = (int)handle->data_available;
+
+  if (handle->data_available > 0) {
+    /* Read data from our intermediate buffer into the process_t buffer. */
+    buf_add(buffer, handle->buffer, handle->data_available);
+
+    /* Reset our read state. */
+    handle->data_available = 0;
+    memset(handle->buffer, 0, sizeof(handle->buffer));
+  }
+
+  /* Because of the slightly weird API for ReadFileEx() we must set this to 0
+   * before we call ReadFileEx() because ReadFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Ask the Windows kernel to read data from our pipe into our buffer and call
+   * the callback function when it is done. */
+  ret = ReadFileEx(handle->pipe,
+                   handle->buffer,
+                   sizeof(handle->buffer),
+                   &handle->overlapped,
+                   callback);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "ReadFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    handle->reached_eof = true;
+    return bytes_available;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for ReadFileEx() we
+   * should check GetLastError() after a call to ReadFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    handle->reached_eof = true;
+    return bytes_available;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* We mark our handle as having a pending I/O request. */
+  handle->busy = true;
+
+  return bytes_available;
+}
+
+/** This function checks the callback values from ReadFileEx() in
+ * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true
+ * iff our caller should request more data from ReadFileEx(). */
+STATIC bool
+process_win32_handle_read_completion(process_win32_handle_t *handle,
+                                     DWORD error_code,
+                                     DWORD byte_count)
+{
+  tor_assert(handle);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  handle->busy = false;
+
+  if (error_code == 0) {
+    /* Our ReadFileEx() call was succesful and there is data for us. */
+
+    /* This cast should be safe since byte_count should never be larger than
+     * BUFFER_SIZE. */
+    tor_assert(byte_count <= BUFFER_SIZE);
+    handle->data_available = (size_t)byte_count;
+
+    /* Tell our caller to schedule the next read. */
+    return true;
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our ReadFileEx() finished, but we reached the end of our file.  We mark
+     * our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    handle->reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from ReadFileEx(): %s",
+             format_win32_error(error_code));
+
+    handle->reached_eof = true;
+  }
+
+  /* Our caller should NOT schedule the next read. */
+  return false;
+}
+
+/** Format a single argument for being put on a Windows command line.
+ * Returns a newly allocated string */
+STATIC char *
+format_win_cmdline_argument(const char *arg)
+{
+  char *formatted_arg;
+  char need_quotes;
+  const char *c;
+  int i;
+  int bs_counter = 0;
+  /* Backslash we can point to when one is inserted into the string */
+  const char backslash = '\\';
+
+  /* Smartlist of *char */
+  smartlist_t *arg_chars;
+  arg_chars = smartlist_new();
+
+  /* Quote string if it contains whitespace or is empty */
+  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
+
+  /* Build up smartlist of *chars */
+  for (c=arg; *c != '\0'; c++) {
+    if ('"' == *c) {
+      /* Double up backslashes preceding a quote */
+      for (i=0; i<(bs_counter*2); i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      /* Escape the quote */
+      smartlist_add(arg_chars, (void*)&backslash);
+      smartlist_add(arg_chars, (void*)c);
+    } else if ('\\' == *c) {
+      /* Count backslashes until we know whether to double up */
+      bs_counter++;
+    } else {
+      /* Don't double up slashes preceding a non-quote */
+      for (i=0; i<bs_counter; i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      smartlist_add(arg_chars, (void*)c);
+    }
+  }
+  /* Don't double up trailing backslashes */
+  for (i=0; i<bs_counter; i++)
+    smartlist_add(arg_chars, (void*)&backslash);
+
+  /* Allocate space for argument, quotes (if needed), and terminator */
+  const size_t formatted_arg_len = smartlist_len(arg_chars) +
+    (need_quotes ? 2 : 0) + 1;
+  formatted_arg = tor_malloc_zero(formatted_arg_len);
+
+  /* Add leading quote */
+  i=0;
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+
+  /* Add characters */
+  SMARTLIST_FOREACH(arg_chars, char*, ch,
+  {
+    formatted_arg[i++] = *ch;
+  });
+
+  /* Add trailing quote */
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+  formatted_arg[i] = '\0';
+
+  smartlist_free(arg_chars);
+  return formatted_arg;
+}
+
+/** Format a command line for use on Windows, which takes the command as a
+ * string rather than string array. Follows the rules from "Parsing C++
+ * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
+ * Python subprocess module. Returns a newly allocated string */
+STATIC char *
+tor_join_win_cmdline(const char *argv[])
+{
+  smartlist_t *argv_list;
+  char *joined_argv;
+  int i;
+
+  /* Format each argument and put the result in a smartlist */
+  argv_list = smartlist_new();
+  for (i=0; argv[i] != NULL; i++) {
+    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
+  }
+
+  /* Join the arguments with whitespace */
+  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
+
+  /* Free the newly allocated arguments, and the smartlist */
+  SMARTLIST_FOREACH(argv_list, char *, arg,
+  {
+    tor_free(arg);
+  });
+  smartlist_free(argv_list);
+
+  return joined_argv;
+}
+
+#endif /* defined(_WIN32) */
diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h
new file mode 100644
index 0000000000..a50d86df5b
--- /dev/null
+++ b/src/lib/process/process_win32.h
@@ -0,0 +1,97 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.h
+ * \brief Header for process_win32.c
+ **/
+
+#ifndef TOR_PROCESS_WIN32_H
+#define TOR_PROCESS_WIN32_H
+
+#ifdef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include <windows.h>
+
+struct process_t;
+
+struct process_win32_t;
+typedef struct process_win32_t process_win32_t;
+
+process_win32_t *process_win32_new(void);
+void process_win32_free_(process_win32_t *win32_process);
+#define process_win32_free(s) \
+  FREE_AND_NULL(process_win32_t, process_win32_free_, (s))
+
+void process_win32_init(void);
+void process_win32_deinit(void);
+
+process_status_t process_win32_exec(struct process_t *process);
+bool process_win32_terminate(struct process_t *process);
+
+process_pid_t process_win32_get_pid(struct process_t *process);
+
+int process_win32_write(struct process_t *process, buf_t *buffer);
+int process_win32_read_stdout(struct process_t *process, buf_t *buffer);
+int process_win32_read_stderr(struct process_t *process, buf_t *buffer);
+
+void process_win32_trigger_completion_callbacks(void);
+
+/* Timer handling. */
+void process_win32_timer_start(void);
+void process_win32_timer_stop(void);
+bool process_win32_timer_running(void);
+
+#ifdef PROCESS_WIN32_PRIVATE
+STATIC void process_win32_timer_callback(periodic_timer_t *, void *);
+STATIC bool process_win32_timer_test_process(process_t *);
+
+/* I/O pipe handling. */
+struct process_win32_handle_t;
+typedef struct process_win32_handle_t process_win32_handle_t;
+
+typedef enum process_win32_pipe_type_t {
+  /** This pipe is used for reading. */
+  PROCESS_WIN32_PIPE_TYPE_READER,
+
+  /** This pipe is used for writing. */
+  PROCESS_WIN32_PIPE_TYPE_WRITER
+} process_win32_pipe_type_t;
+
+STATIC bool process_win32_create_pipe(HANDLE *,
+                                      HANDLE *,
+                                      SECURITY_ATTRIBUTES *,
+                                      process_win32_pipe_type_t);
+
+STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle);
+
+STATIC VOID WINAPI process_win32_stdout_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stderr_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stdin_write_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+
+STATIC int process_win32_read_from_handle(process_win32_handle_t *,
+                                          buf_t *,
+                                          LPOVERLAPPED_COMPLETION_ROUTINE);
+STATIC bool process_win32_handle_read_completion(process_win32_handle_t *,
+                                                 DWORD,
+                                                 DWORD);
+
+STATIC char *format_win_cmdline_argument(const char *arg);
+STATIC char *tor_join_win_cmdline(const char *argv[]);
+#endif /* defined(PROCESS_WIN32_PRIVATE) */
+
+#endif /* defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_WIN32_H) */
diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c
index 534b39d101..fda284f3d9 100644
--- a/src/lib/process/restrict.c
+++ b/src/lib/process/restrict.c
@@ -152,7 +152,7 @@ tor_mlockall(void)
                            "pages: %s", strerror(errno));
     return -1;
   }
-#else /* !(defined(HAVE_UNIX_MLOCKALL)) */
+#else /* !defined(HAVE_UNIX_MLOCKALL) */
   log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?");
   return -1;
 #endif /* defined(HAVE_UNIX_MLOCKALL) */
@@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
     return -1;
   }
   limit = MAX_CONNECTIONS;
-#else /* !(!defined(HAVE_GETRLIMIT)) */
+#else /* defined(HAVE_GETRLIMIT) */
   struct rlimit rlim;
 
   if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c
index 6e8258f279..3c94ce4bac 100644
--- a/src/lib/process/setuid.c
+++ b/src/lib/process/setuid.c
@@ -72,7 +72,7 @@ log_credential_status(void)
            "UID is %u (real), %u (effective), %u (saved)",
            (unsigned)ruid, (unsigned)euid, (unsigned)suid);
   }
-#else /* !(defined(HAVE_GETRESUID)) */
+#else /* !defined(HAVE_GETRESUID) */
   /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */
   ruid = getuid();
   euid = geteuid();
@@ -93,7 +93,7 @@ log_credential_status(void)
            "GID is %u (real), %u (effective), %u (saved)",
            (unsigned)rgid, (unsigned)egid, (unsigned)sgid);
   }
-#else /* !(defined(HAVE_GETRESGID)) */
+#else /* !defined(HAVE_GETRESGID) */
   /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */
   rgid = getgid();
   egid = getegid();
@@ -154,7 +154,7 @@ have_capability_support(void)
     return 0;
   cap_free(caps);
   return 1;
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   return 0;
 #endif /* defined(HAVE_LINUX_CAPABILITIES) */
 }
@@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags)
     if (drop_capabilities(1))
       return -1;
   }
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   (void) keep_bindlow;
   if (warn_if_no_caps) {
     log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support "
@@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags)
 #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */
   return 0;
 
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void)user;
   (void)flags;
 
diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h
index 7d03e1f025..a2125d2d06 100644
--- a/src/lib/process/setuid.h
+++ b/src/lib/process/setuid.h
@@ -19,4 +19,4 @@ int have_capability_support(void);
 #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1)
 int switch_id(const char *user, unsigned flags);
 
-#endif
+#endif /* !defined(TOR_SETUID_H) */
diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c
deleted file mode 100644
index f4429d7f76..0000000000
--- a/src/lib/process/subprocess.c
+++ /dev/null
@@ -1,1236 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.c
- * \brief Launch and monitor other processes.
- **/
-
-#define SUBPROCESS_PRIVATE
-#include "lib/process/subprocess.h"
-
-#include "lib/container/smartlist.h"
-#include "lib/err/torerr.h"
-#include "lib/log/log.h"
-#include "lib/log/util_bug.h"
-#include "lib/log/win32err.h"
-#include "lib/malloc/malloc.h"
-#include "lib/process/env.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/compat_ctype.h"
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_PRCTL_H
-#include <sys/prctl.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-#include <errno.h>
-#include <string.h>
-
-/** Format a single argument for being put on a Windows command line.
- * Returns a newly allocated string */
-static char *
-format_win_cmdline_argument(const char *arg)
-{
-  char *formatted_arg;
-  char need_quotes;
-  const char *c;
-  int i;
-  int bs_counter = 0;
-  /* Backslash we can point to when one is inserted into the string */
-  const char backslash = '\\';
-
-  /* Smartlist of *char */
-  smartlist_t *arg_chars;
-  arg_chars = smartlist_new();
-
-  /* Quote string if it contains whitespace or is empty */
-  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
-
-  /* Build up smartlist of *chars */
-  for (c=arg; *c != '\0'; c++) {
-    if ('"' == *c) {
-      /* Double up backslashes preceding a quote */
-      for (i=0; i<(bs_counter*2); i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      /* Escape the quote */
-      smartlist_add(arg_chars, (void*)&backslash);
-      smartlist_add(arg_chars, (void*)c);
-    } else if ('\\' == *c) {
-      /* Count backslashes until we know whether to double up */
-      bs_counter++;
-    } else {
-      /* Don't double up slashes preceding a non-quote */
-      for (i=0; i<bs_counter; i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      smartlist_add(arg_chars, (void*)c);
-    }
-  }
-  /* Don't double up trailing backslashes */
-  for (i=0; i<bs_counter; i++)
-    smartlist_add(arg_chars, (void*)&backslash);
-
-  /* Allocate space for argument, quotes (if needed), and terminator */
-  const size_t formatted_arg_len = smartlist_len(arg_chars) +
-    (need_quotes ? 2 : 0) + 1;
-  formatted_arg = tor_malloc_zero(formatted_arg_len);
-
-  /* Add leading quote */
-  i=0;
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-
-  /* Add characters */
-  SMARTLIST_FOREACH(arg_chars, char*, ch,
-  {
-    formatted_arg[i++] = *ch;
-  });
-
-  /* Add trailing quote */
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-  formatted_arg[i] = '\0';
-
-  smartlist_free(arg_chars);
-  return formatted_arg;
-}
-
-/** Format a command line for use on Windows, which takes the command as a
- * string rather than string array. Follows the rules from "Parsing C++
- * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
- * Python subprocess module. Returns a newly allocated string */
-char *
-tor_join_win_cmdline(const char *argv[])
-{
-  smartlist_t *argv_list;
-  char *joined_argv;
-  int i;
-
-  /* Format each argument and put the result in a smartlist */
-  argv_list = smartlist_new();
-  for (i=0; argv[i] != NULL; i++) {
-    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
-  }
-
-  /* Join the arguments with whitespace */
-  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
-
-  /* Free the newly allocated arguments, and the smartlist */
-  SMARTLIST_FOREACH(argv_list, char *, arg,
-  {
-    tor_free(arg);
-  });
-  smartlist_free(argv_list);
-
-  return joined_argv;
-}
-
-#ifndef _WIN32
-/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in
- * <b>hex_errno</b>.  Called between fork and _exit, so must be signal-handler
- * safe.
- *
- * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available.
- *
- * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded
- * with spaces. CHILD_STATE indicates where
- * in the process of starting the child process did the failure occur (see
- * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of
- * errno when the failure occurred.
- *
- * On success return the number of characters added to hex_errno, not counting
- * the terminating NUL; return -1 on error.
- */
-STATIC int
-format_helper_exit_status(unsigned char child_state, int saved_errno,
-                          char *hex_errno)
-{
-  unsigned int unsigned_errno;
-  int written, left;
-  char *cur;
-  size_t i;
-  int res = -1;
-
-  /* Fill hex_errno with spaces, and a trailing newline (memset may
-     not be signal handler safe, so we can't use it) */
-  for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++)
-    hex_errno[i] = ' ';
-  hex_errno[HEX_ERRNO_SIZE - 1] = '\n';
-
-  /* Convert errno to be unsigned for hex conversion */
-  if (saved_errno < 0) {
-    // Avoid overflow on the cast to unsigned int when result is INT_MIN
-    // by adding 1 to the signed int negative value,
-    // then, after it has been negated and cast to unsigned,
-    // adding the original 1 back (the double-addition is intentional).
-    // Otherwise, the cast to signed could cause a temporary int
-    // to equal INT_MAX + 1, which is undefined.
-    unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1;
-  } else {
-    unsigned_errno = (unsigned int) saved_errno;
-  }
-
-  /*
-   * Count how many chars of space we have left, and keep a pointer into the
-   * current point in the buffer.
-   */
-  left = HEX_ERRNO_SIZE+1;
-  cur = hex_errno;
-
-  /* Emit child_state */
-  written = format_hex_number_sigsafe(child_state, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-  if (left <= 0)
-    goto err;
-
-  /* Now the '/' */
-  *cur = '/';
-
-  /* Adjust left and cur */
-  ++cur;
-  --left;
-  if (left <= 0)
-    goto err;
-
-  /* Need minus? */
-  if (saved_errno < 0) {
-    *cur = '-';
-    ++cur;
-    --left;
-    if (left <= 0)
-      goto err;
-  }
-
-  /* Emit unsigned_errno */
-  written = format_hex_number_sigsafe(unsigned_errno, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-
-  /* Check that we have enough space left for a newline and a NUL */
-  if (left <= 1)
-    goto err;
-
-  /* Emit the newline and NUL */
-  *cur++ = '\n';
-  *cur++ = '\0';
-
-  res = (int)(cur - hex_errno - 1);
-
-  goto done;
-
- err:
-  /*
-   * In error exit, just write a '\0' in the first char so whatever called
-   * this at least won't fall off the end.
-   */
-  *hex_errno = '\0';
-
- done:
-  return res;
-}
-#endif /* !defined(_WIN32) */
-
-/* Maximum number of file descriptors, if we cannot get it via sysconf() */
-#define DEFAULT_MAX_FD 256
-
-/** Terminate the process of <b>process_handle</b>, if that process has not
- * already exited.
- *
- * Return 0 if we succeeded in terminating the process (or if the process
- * already exited), and -1 if we tried to kill the process but failed.
- *
- * Based on code originally borrowed from Python's os.kill. */
-int
-tor_terminate_process(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) {
-    HANDLE handle = process_handle->pid.hProcess;
-
-    if (!TerminateProcess(handle, 0))
-      return -1;
-    else
-      return 0;
-  }
-#else /* !(defined(_WIN32)) */
-  if (process_handle->waitpid_cb) {
-    /* We haven't got a waitpid yet, so we can just kill off the process. */
-    return kill(process_handle->pid, SIGTERM);
-  }
-#endif /* defined(_WIN32) */
-
-  return 0; /* We didn't need to kill the process, so report success */
-}
-
-/** Return the Process ID of <b>process_handle</b>. */
-int
-tor_process_get_pid(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  return (int) process_handle->pid.dwProcessId;
-#else
-  return (int) process_handle->pid;
-#endif
-}
-
-#ifdef _WIN32
-HANDLE
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#else /* !(defined(_WIN32)) */
-/* DOCDOC tor_process_get_stdout_pipe */
-int
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#endif /* defined(_WIN32) */
-
-/* DOCDOC process_handle_new */
-static process_handle_t *
-process_handle_new(void)
-{
-  process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t));
-
-#ifdef _WIN32
-  out->stdin_pipe = INVALID_HANDLE_VALUE;
-  out->stdout_pipe = INVALID_HANDLE_VALUE;
-  out->stderr_pipe = INVALID_HANDLE_VALUE;
-#else
-  out->stdin_pipe = -1;
-  out->stdout_pipe = -1;
-  out->stderr_pipe = -1;
-#endif /* defined(_WIN32) */
-
-  return out;
-}
-
-#ifndef _WIN32
-/** Invoked when a process that we've launched via tor_spawn_background() has
- * been found to have terminated.
- */
-static void
-process_handle_waitpid_cb(int status, void *arg)
-{
-  process_handle_t *process_handle = arg;
-
-  process_handle->waitpid_exit_status = status;
-  clear_waitpid_callback(process_handle->waitpid_cb);
-  if (process_handle->status == PROCESS_STATUS_RUNNING)
-    process_handle->status = PROCESS_STATUS_NOTRUNNING;
-  process_handle->waitpid_cb = 0;
-}
-#endif /* !defined(_WIN32) */
-
-/**
- * @name child-process states
- *
- * Each of these values represents a possible state that a child process can
- * be in.  They're used to determine what to say when telling the parent how
- * far along we were before failure.
- *
- * @{
- */
-#define CHILD_STATE_INIT 0
-#define CHILD_STATE_PIPE 1
-#define CHILD_STATE_MAXFD 2
-#define CHILD_STATE_FORK 3
-#define CHILD_STATE_DUPOUT 4
-#define CHILD_STATE_DUPERR 5
-#define CHILD_STATE_DUPIN 6
-#define CHILD_STATE_CLOSEFD 7
-#define CHILD_STATE_EXEC 8
-#define CHILD_STATE_FAILEXEC 9
-/** @} */
-/**
- * Boolean.  If true, then Tor may call execve or CreateProcess via
- * tor_spawn_background.
- **/
-static int may_spawn_background_process = 1;
-/**
- * Turn off may_spawn_background_process, so that all future calls to
- * tor_spawn_background are guaranteed to fail.
- **/
-void
-tor_disable_spawning_background_processes(void)
-{
-  may_spawn_background_process = 0;
-}
-/** Start a program in the background. If <b>filename</b> contains a '/', then
- * it will be treated as an absolute or relative path.  Otherwise, on
- * non-Windows systems, the system path will be searched for <b>filename</b>.
- * On Windows, only the current directory will be searched. Here, to search the
- * system path (as well as the application directory, current working
- * directory, and system directories), set filename to NULL.
- *
- * The strings in <b>argv</b> will be passed as the command line arguments of
- * the child program (following convention, argv[0] should normally be the
- * filename of the executable, and this must be the case if <b>filename</b> is
- * NULL). The last element of argv must be NULL. A handle to the child process
- * will be returned in process_handle (which must be non-NULL). Read
- * process_handle.status to find out if the process was successfully launched.
- * For convenience, process_handle.status is returned by this function.
- *
- * Some parts of this code are based on the POSIX subprocess module from
- * Python, and example code from
- * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx.
- */
-int
-tor_spawn_background(const char *const filename, const char **argv,
-                     process_environment_t *env,
-                     process_handle_t **process_handle_out)
-{
-  if (BUG(may_spawn_background_process == 0)) {
-    /* We should never reach this point if we're forbidden to spawn
-     * processes. Instead we should have caught the attempt earlier. */
-    return PROCESS_STATUS_ERROR;
-  }
-
-#ifdef _WIN32
-  HANDLE stdout_pipe_read = NULL;
-  HANDLE stdout_pipe_write = NULL;
-  HANDLE stderr_pipe_read = NULL;
-  HANDLE stderr_pipe_write = NULL;
-  HANDLE stdin_pipe_read = NULL;
-  HANDLE stdin_pipe_write = NULL;
-  process_handle_t *process_handle;
-  int status;
-
-  STARTUPINFOA siStartInfo;
-  BOOL retval = FALSE;
-
-  SECURITY_ATTRIBUTES saAttr;
-  char *joined_argv;
-
-  saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
-  saAttr.bInheritHandle = TRUE;
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-  saAttr.lpSecurityDescriptor = NULL;
-
-  /* Assume failure to start process */
-  status = PROCESS_STATUS_ERROR;
-
-  /* Set up pipe for stdout */
-  if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdout communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdout communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stderr */
-  if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stderr communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stderr communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stdin */
-  if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdin communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdin communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Create the child process */
-
-  /* Windows expects argv to be a whitespace delimited string, so join argv up
-   */
-  joined_argv = tor_join_win_cmdline(argv);
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-
-  ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION));
-  ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
-  siStartInfo.cb = sizeof(STARTUPINFO);
-  siStartInfo.hStdError = stderr_pipe_write;
-  siStartInfo.hStdOutput = stdout_pipe_write;
-  siStartInfo.hStdInput = stdin_pipe_read;
-  siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
-
-  /* Create the child process */
-
-  retval = CreateProcessA(filename,      // module name
-                 joined_argv,   // command line
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-                 NULL,          // process security attributes
-                 NULL,          // primary thread security attributes
-                 TRUE,          // handles are inherited
-  /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess()
-   * work?) */
-                 CREATE_NO_WINDOW,             // creation flags
-                 (env==NULL) ? NULL : env->windows_environment_block,
-                 NULL,          // use parent's current directory
-                 &siStartInfo,  // STARTUPINFO pointer
-                 &(process_handle->pid));  // receives PROCESS_INFORMATION
-
-  tor_free(joined_argv);
-
-  if (!retval) {
-    log_warn(LD_GENERAL,
-      "Failed to create child process %s: %s", filename?filename:argv[0],
-      format_win32_error(GetLastError()));
-    tor_free(process_handle);
-  } else  {
-    /* TODO: Close hProcess and hThread in process_handle->pid? */
-    process_handle->stdout_pipe = stdout_pipe_read;
-    process_handle->stderr_pipe = stderr_pipe_read;
-    process_handle->stdin_pipe = stdin_pipe_write;
-    status = process_handle->status = PROCESS_STATUS_RUNNING;
-  }
-
-  /* TODO: Close pipes on exit */
-  *process_handle_out = process_handle;
-  return status;
-#else /* !(defined(_WIN32)) */
-  pid_t pid;
-  int stdout_pipe[2];
-  int stderr_pipe[2];
-  int stdin_pipe[2];
-  int fd, retval;
-  process_handle_t *process_handle;
-  int status;
-
-  const char *error_message = SPAWN_ERROR_MESSAGE;
-  size_t error_message_length;
-
-  /* Represents where in the process of spawning the program is;
-     this is used for printing out the error message */
-  unsigned char child_state = CHILD_STATE_INIT;
-
-  char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */
-
-  static int max_fd = -1;
-
-  status = PROCESS_STATUS_ERROR;
-
-  /* We do the strlen here because strlen() is not signal handler safe,
-     and we are not allowed to use unsafe functions between fork and exec */
-  error_message_length = strlen(error_message);
-
-  // child_state = CHILD_STATE_PIPE;
-
-  /* Set up pipe for redirecting stdout, stderr, and stdin of child */
-  retval = pipe(stdout_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdout communication with child process: %s",
-       strerror(errno));
-    return status;
-  }
-
-  retval = pipe(stderr_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stderr communication with child process: %s",
-      strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-
-    return status;
-  }
-
-  retval = pipe(stdin_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdin communication with child process: %s",
-       strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-
-    return status;
-  }
-
-  // child_state = CHILD_STATE_MAXFD;
-
-#ifdef _SC_OPEN_MAX
-  if (-1 == max_fd) {
-    max_fd = (int) sysconf(_SC_OPEN_MAX);
-    if (max_fd == -1) {
-      max_fd = DEFAULT_MAX_FD;
-      log_warn(LD_GENERAL,
-               "Cannot find maximum file descriptor, assuming %d", max_fd);
-    }
-  }
-#else /* !(defined(_SC_OPEN_MAX)) */
-  max_fd = DEFAULT_MAX_FD;
-#endif /* defined(_SC_OPEN_MAX) */
-
-  // child_state = CHILD_STATE_FORK;
-
-  pid = fork();
-  if (0 == pid) {
-    /* In child */
-
-#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
-    /* Attempt to have the kernel issue a SIGTERM if the parent
-     * goes away. Certain attributes of the binary being execve()ed
-     * will clear this during the execve() call, but it's better
-     * than nothing.
-     */
-    prctl(PR_SET_PDEATHSIG, SIGTERM);
-#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
-
-    child_state = CHILD_STATE_DUPOUT;
-
-    /* Link child stdout to the write end of the pipe */
-    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPERR;
-
-    /* Link child stderr to the write end of the pipe */
-    retval = dup2(stderr_pipe[1], STDERR_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPIN;
-
-    /* Link child stdin to the read end of the pipe */
-    retval = dup2(stdin_pipe[0], STDIN_FILENO);
-    if (-1 == retval)
-      goto error;
-
-    // child_state = CHILD_STATE_CLOSEFD;
-
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-
-    /* Close all other fds, including the read end of the pipe */
-    /* XXX: We should now be doing enough FD_CLOEXEC setting to make
-     * this needless. */
-    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) {
-      close(fd);
-    }
-
-    // child_state = CHILD_STATE_EXEC;
-
-    /* Call the requested program. We need the cast because
-       execvp doesn't define argv as const, even though it
-       does not modify the arguments */
-    if (env)
-      execve(filename, (char *const *) argv, env->unixoid_environment_block);
-    else {
-      static char *new_env[] = { NULL };
-      execve(filename, (char *const *) argv, new_env);
-    }
-
-    /* If we got here, the exec or open(/dev/null) failed */
-
-    child_state = CHILD_STATE_FAILEXEC;
-
-  error:
-    {
-      /* XXX: are we leaking fds from the pipe? */
-      int n, err=0;
-      ssize_t nbytes;
-
-      n = format_helper_exit_status(child_state, errno, hex_errno);
-
-      if (n >= 0) {
-        /* Write the error message. GCC requires that we check the return
-           value, but there is nothing we can do if it fails */
-        /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */
-        nbytes = write(STDOUT_FILENO, error_message, error_message_length);
-        err = (nbytes < 0);
-        nbytes = write(STDOUT_FILENO, hex_errno, n);
-        err += (nbytes < 0);
-      }
-
-      _exit(err?254:255); // exit ok: in child.
-    }
-
-    /* Never reached, but avoids compiler warning */
-    return status; // LCOV_EXCL_LINE
-  }
-
-  /* In parent */
-
-  if (-1 == pid) {
-    log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno));
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    return status;
-  }
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-  process_handle->pid = pid;
-
-  /* TODO: If the child process forked but failed to exec, waitpid it */
-
-  /* Return read end of the pipes to caller, and close write end */
-  process_handle->stdout_pipe = stdout_pipe[0];
-  retval = close(stdout_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stdout pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  process_handle->waitpid_cb = set_waitpid_callback(pid,
-                                                    process_handle_waitpid_cb,
-                                                    process_handle);
-
-  process_handle->stderr_pipe = stderr_pipe[0];
-  retval = close(stderr_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stderr pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  /* Return write end of the stdin pipe to caller, and close the read end */
-  process_handle->stdin_pipe = stdin_pipe[1];
-  retval = close(stdin_pipe[0]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close read end of stdin pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  status = process_handle->status = PROCESS_STATUS_RUNNING;
-  /* Set stdin/stdout/stderr pipes to be non-blocking */
-  if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) {
-    log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
-             "nonblocking in parent process: %s", strerror(errno));
-  }
-
-  *process_handle_out = process_handle;
-  return status;
-#endif /* defined(_WIN32) */
-}
-
-/** Destroy all resources allocated by the process handle in
- *  <b>process_handle</b>.
- *  If <b>also_terminate_process</b> is true, also terminate the
- *  process of the process handle. */
-MOCK_IMPL(void,
-tor_process_handle_destroy,(process_handle_t *process_handle,
-                            int also_terminate_process))
-{
-  if (!process_handle)
-    return;
-
-  if (also_terminate_process) {
-    if (tor_terminate_process(process_handle) < 0) {
-      const char *errstr =
-#ifdef _WIN32
-        format_win32_error(GetLastError());
-#else
-        strerror(errno);
-#endif
-      log_notice(LD_GENERAL, "Failed to terminate process with "
-                 "PID '%d' ('%s').", tor_process_get_pid(process_handle),
-                 errstr);
-    } else {
-      log_info(LD_GENERAL, "Terminated process with PID '%d'.",
-               tor_process_get_pid(process_handle));
-    }
-  }
-
-  process_handle->status = PROCESS_STATUS_NOTRUNNING;
-
-#ifdef _WIN32
-  if (process_handle->stdout_pipe)
-    CloseHandle(process_handle->stdout_pipe);
-
-  if (process_handle->stderr_pipe)
-    CloseHandle(process_handle->stderr_pipe);
-
-  if (process_handle->stdin_pipe)
-    CloseHandle(process_handle->stdin_pipe);
-#else /* !(defined(_WIN32)) */
-  close(process_handle->stdout_pipe);
-  close(process_handle->stderr_pipe);
-  close(process_handle->stdin_pipe);
-
-  clear_waitpid_callback(process_handle->waitpid_cb);
-#endif /* defined(_WIN32) */
-
-  memset(process_handle, 0x0f, sizeof(process_handle_t));
-  tor_free(process_handle);
-}
-
-/** Get the exit code of a process specified by <b>process_handle</b> and store
- * it in <b>exit_code</b>, if set to a non-NULL value.  If <b>block</b> is set
- * to true, the call will block until the process has exited.  Otherwise if
- * the process is still running, the function will return
- * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns
- * PROCESS_EXIT_EXITED if the process did exit. If there is a failure,
- * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if
- * non-NULL) will be undefined. N.B. Under *nix operating systems, this will
- * probably not work in Tor, because waitpid() is called in main.c to reap any
- * terminated child processes.*/
-int
-tor_get_exit_code(process_handle_t *process_handle,
-                  int block, int *exit_code)
-{
-#ifdef _WIN32
-  DWORD retval;
-  BOOL success;
-
-  if (block) {
-    /* Wait for the process to exit */
-    retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE);
-    if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-              (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  } else {
-    retval = WaitForSingleObject(process_handle->pid.hProcess, 0);
-    if (WAIT_TIMEOUT == retval) {
-      /* Process has not exited */
-      return PROCESS_EXIT_RUNNING;
-    } else if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-               (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-
-  if (exit_code != NULL) {
-    success = GetExitCodeProcess(process_handle->pid.hProcess,
-                                 (PDWORD)exit_code);
-    if (!success) {
-      log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s",
-               format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-#else /* !(defined(_WIN32)) */
-  int stat_loc;
-  int retval;
-
-  if (process_handle->waitpid_cb) {
-    /* We haven't processed a SIGCHLD yet. */
-    retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG);
-    if (retval == process_handle->pid) {
-      clear_waitpid_callback(process_handle->waitpid_cb);
-      process_handle->waitpid_cb = NULL;
-      process_handle->waitpid_exit_status = stat_loc;
-    }
-  } else {
-    /* We already got a SIGCHLD for this process, and handled it. */
-    retval = process_handle->pid;
-    stat_loc = process_handle->waitpid_exit_status;
-  }
-
-  if (!block && 0 == retval) {
-    /* Process has not exited */
-    return PROCESS_EXIT_RUNNING;
-  } else if (retval != process_handle->pid) {
-    log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s",
-             (int)process_handle->pid, strerror(errno));
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (!WIFEXITED(stat_loc)) {
-    log_warn(LD_GENERAL, "Process %d did not exit normally",
-             (int)process_handle->pid);
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (exit_code != NULL)
-    *exit_code = WEXITSTATUS(stat_loc);
-#endif /* defined(_WIN32) */
-
-  return PROCESS_EXIT_EXITED;
-}
-
-#ifdef _WIN32
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>hProcess</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle
- * to the process owning the <b>h</b>. In this case, the function will exit
- * only once the process has exited, or <b>count</b> bytes are read. Returns
- * the number of bytes read, or -1 on error. */
-ssize_t
-tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                    const process_handle_t *process)
-{
-  size_t numread = 0;
-  BOOL retval;
-  DWORD byte_count;
-  BOOL process_exited = FALSE;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    /* Check if there is anything to read */
-    retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
-    if (!retval) {
-      log_warn(LD_GENERAL,
-        "Failed to peek from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* Nothing available: process exited or it is busy */
-
-      /* Exit if we don't know whether the process is running */
-      if (NULL == process)
-        break;
-
-      /* The process exited and there's nothing left to read from it */
-      if (process_exited)
-        break;
-
-      /* If process is not running, check for output one more time in case
-         it wrote something after the peek was performed. Otherwise keep on
-         waiting for output */
-      tor_assert(process != NULL);
-      byte_count = WaitForSingleObject(process->pid.hProcess, 0);
-      if (WAIT_TIMEOUT != byte_count)
-        process_exited = TRUE;
-
-      continue;
-    }
-
-    /* There is data to read; read it */
-    retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL);
-    tor_assert(byte_count + numread <= count);
-    if (!retval) {
-      log_warn(LD_GENERAL, "Failed to read from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* End of file */
-      break;
-    }
-    numread += byte_count;
-  }
-  return (ssize_t)numread;
-}
-#else /* !(defined(_WIN32)) */
-/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>process</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise data will be read until end of file, or
- * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
- * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
- * file has been reached. */
-ssize_t
-tor_read_all_handle(int fd, char *buf, size_t count,
-                    const process_handle_t *process,
-                    int *eof)
-{
-  size_t numread = 0;
-  ssize_t result;
-
-  if (eof)
-    *eof = 0;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    result = read(fd, buf+numread, count-numread);
-
-    if (result == 0) {
-      log_debug(LD_GENERAL, "read() reached end of file");
-      if (eof)
-        *eof = 1;
-      break;
-    } else if (result < 0 && errno == EAGAIN) {
-      if (process)
-        continue;
-      else
-        break;
-    } else if (result < 0) {
-      log_warn(LD_GENERAL, "read() failed: %s", strerror(errno));
-      return -1;
-    }
-
-    numread += result;
-  }
-
-  log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread);
-  return (ssize_t)numread;
-}
-#endif /* defined(_WIN32) */
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stdout(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stderr(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Return a string corresponding to <b>stream_status</b>. */
-const char *
-stream_status_to_string(enum stream_status stream_status)
-{
-  switch (stream_status) {
-    case IO_STREAM_OKAY:
-      return "okay";
-    case IO_STREAM_EAGAIN:
-      return "temporarily unavailable";
-    case IO_STREAM_TERM:
-      return "terminated";
-    case IO_STREAM_CLOSED:
-      return "closed";
-    default:
-      tor_fragile_assert();
-      return "unknown";
-  }
-}
-
-/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be
- * modified. The resulting smartlist will consist of pointers to buf, so there
- * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated
- * string. <b>len</b> should be set to the length of the buffer excluding the
- * NUL. Non-printable characters (including NUL) will be replaced with "." */
-int
-tor_split_lines(smartlist_t *sl, char *buf, int len)
-{
-  /* Index in buf of the start of the current line */
-  int start = 0;
-  /* Index in buf of the current character being processed */
-  int cur = 0;
-  /* Are we currently in a line */
-  char in_line = 0;
-
-  /* Loop over string */
-  while (cur < len) {
-    /* Loop until end of line or end of string */
-    for (; cur < len; cur++) {
-      if (in_line) {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* End of line */
-          buf[cur] = '\0';
-          /* Point cur to the next line */
-          cur++;
-          /* Line starts at start and ends with a nul */
-          break;
-        } else {
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      } else {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* Skip leading vertical space */
-          ;
-        } else {
-          in_line = 1;
-          start = cur;
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      }
-    }
-    /* We are at the end of the line or end of string. If in_line is true there
-     * is a line which starts at buf+start and ends at a NUL. cur points to
-     * the character after the NUL. */
-    if (in_line)
-      smartlist_add(sl, (void *)(buf+start));
-    in_line = 0;
-  }
-  return smartlist_len(sl);
-}
-
-#ifdef _WIN32
-
-/** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (HANDLE *handle,
-                            enum stream_status *stream_status_out))
-{
-  int pos;
-  char stdout_buf[600] = {0};
-  smartlist_t *lines = NULL;
-
-  tor_assert(stream_status_out);
-
-  *stream_status_out = IO_STREAM_TERM;
-
-  pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL);
-  if (pos < 0) {
-    *stream_status_out = IO_STREAM_TERM;
-    return NULL;
-  }
-  if (pos == 0) {
-    *stream_status_out = IO_STREAM_EAGAIN;
-    return NULL;
-  }
-
-  /* End with a null even if there isn't a \r\n at the end */
-  /* TODO: What if this is a partial line? */
-  stdout_buf[pos] = '\0';
-
-  /* Split up the buffer */
-  lines = smartlist_new();
-  tor_split_lines(lines, stdout_buf, pos);
-
-  /* Currently 'lines' is populated with strings residing on the
-     stack. Replace them with their exact copies on the heap: */
-  SMARTLIST_FOREACH(lines, char *, line,
-                    SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line)));
-
-  *stream_status_out = IO_STREAM_OKAY;
-
-  return lines;
-}
-
-#else /* !(defined(_WIN32)) */
-
-/** Return a smartlist containing lines outputted from
- *  <b>fd</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
-{
-  enum stream_status stream_status;
-  char stdout_buf[400];
-  smartlist_t *lines = NULL;
-
-  while (1) {
-    memset(stdout_buf, 0, sizeof(stdout_buf));
-
-    stream_status = get_string_from_pipe(fd,
-                                         stdout_buf, sizeof(stdout_buf) - 1);
-    if (stream_status != IO_STREAM_OKAY)
-      goto done;
-
-    if (!lines) lines = smartlist_new();
-    smartlist_split_string(lines, stdout_buf, "\n", 0, 0);
-  }
-
- done:
-  *stream_status_out = stream_status;
-  return lines;
-}
-
-#endif /* defined(_WIN32) */
-
-/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
- *  sure it's below <b>count</b> bytes.
- *  If the string has a trailing newline, we strip it off.
- *
- * This function is specifically created to handle input from managed
- * proxies, according to the pluggable transports spec. Make sure it
- * fits your needs before using it.
- *
- * Returns:
- * IO_STREAM_CLOSED: If the stream is closed.
- * IO_STREAM_EAGAIN: If there is nothing to read and we should check back
- *  later.
- * IO_STREAM_TERM: If something is wrong with the stream.
- * IO_STREAM_OKAY: If everything went okay and we got a string
- *  in <b>buf_out</b>. */
-enum stream_status
-get_string_from_pipe(int fd, char *buf_out, size_t count)
-{
-  ssize_t ret;
-
-  tor_assert(count <= INT_MAX);
-
-  ret = read(fd, buf_out, count);
-
-  if (ret == 0)
-    return IO_STREAM_CLOSED;
-  else if (ret < 0 && errno == EAGAIN)
-    return IO_STREAM_EAGAIN;
-  else if (ret < 0)
-    return IO_STREAM_TERM;
-
-  if (buf_out[ret - 1] == '\n') {
-    /* Remove the trailing newline */
-    buf_out[ret - 1] = '\0';
-  } else
-    buf_out[ret] = '\0';
-
-  return IO_STREAM_OKAY;
-}
diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h
deleted file mode 100644
index aa3127d62d..0000000000
--- a/src/lib/process/subprocess.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.h
- * \brief Header for subprocess.c
- **/
-
-#ifndef TOR_SUBPROCESS_H
-#define TOR_SUBPROCESS_H
-
-#include "lib/cc/torint.h"
-#include "lib/testsupport/testsupport.h"
-#include <stddef.h>
-#ifdef _WIN32
-#include <windows.h>
-#endif
-
-struct smartlist_t;
-
-void tor_disable_spawning_background_processes(void);
-
-typedef struct process_handle_t process_handle_t;
-struct process_environment_t;
-int tor_spawn_background(const char *const filename, const char **argv,
-                         struct process_environment_t *env,
-                         process_handle_t **process_handle_out);
-
-#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code "
-
-/** Status of an I/O stream. */
-enum stream_status {
-  IO_STREAM_OKAY,
-  IO_STREAM_EAGAIN,
-  IO_STREAM_TERM,
-  IO_STREAM_CLOSED
-};
-
-const char *stream_status_to_string(enum stream_status stream_status);
-
-enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
-
-/* Values of process_handle_t.status. */
-#define PROCESS_STATUS_NOTRUNNING 0
-#define PROCESS_STATUS_RUNNING 1
-#define PROCESS_STATUS_ERROR -1
-
-#ifdef SUBPROCESS_PRIVATE
-struct waitpid_callback_t;
-
-/** Structure to represent the state of a process with which Tor is
- * communicating. The contents of this structure are private to util.c */
-struct process_handle_t {
-  /** One of the PROCESS_STATUS_* values */
-  int status;
-#ifdef _WIN32
-  HANDLE stdin_pipe;
-  HANDLE stdout_pipe;
-  HANDLE stderr_pipe;
-  PROCESS_INFORMATION pid;
-#else /* !(defined(_WIN32)) */
-  int stdin_pipe;
-  int stdout_pipe;
-  int stderr_pipe;
-  pid_t pid;
-  /** If the process has not given us a SIGCHLD yet, this has the
-   * waitpid_callback_t that gets invoked once it has. Otherwise this
-   * contains NULL. */
-  struct waitpid_callback_t *waitpid_cb;
-  /** The exit status reported by waitpid. */
-  int waitpid_exit_status;
-#endif /* defined(_WIN32) */
-};
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-/* Return values of tor_get_exit_code() */
-#define PROCESS_EXIT_RUNNING 1
-#define PROCESS_EXIT_EXITED 0
-#define PROCESS_EXIT_ERROR -1
-int tor_get_exit_code(process_handle_t *process_handle,
-                      int block, int *exit_code);
-int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
-#ifdef _WIN32
-ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                            const process_handle_t *process);
-#else
-ssize_t tor_read_all_handle(int fd, char *buf, size_t count,
-                            const process_handle_t *process,
-                            int *eof);
-#endif /* defined(_WIN32) */
-ssize_t tor_read_all_from_process_stdout(
-    const process_handle_t *process_handle, char *buf, size_t count);
-ssize_t tor_read_all_from_process_stderr(
-    const process_handle_t *process_handle, char *buf, size_t count);
-char *tor_join_win_cmdline(const char *argv[]);
-
-int tor_process_get_pid(process_handle_t *process_handle);
-#ifdef _WIN32
-HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#else
-int tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#endif
-
-#ifdef _WIN32
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle,
-                           enum stream_status *stream_status));
-#else
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd,
-                                       enum stream_status *stream_status));
-#endif /* defined(_WIN32) */
-
-int tor_terminate_process(process_handle_t *process_handle);
-
-MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle,
-                                            int also_terminate_process));
-
-#ifdef SUBPROCESS_PRIVATE
-/* Prototypes for private functions only used by util.c (and unit tests) */
-
-#ifndef _WIN32
-STATIC int format_helper_exit_status(unsigned char child_state,
-                              int saved_errno, char *hex_errno);
-
-/* Space for hex values of child state, a slash, saved_errno (with
-   leading minus) and newline (no null) */
-#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \
-                        1 + sizeof(int) * 2 + 1)
-#endif /* !defined(_WIN32) */
-
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-#endif
diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c
index 9b626394d2..2b38481aeb 100644
--- a/src/lib/process/waitpid.c
+++ b/src/lib/process/waitpid.c
@@ -16,7 +16,7 @@
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
-#include "ht.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
diff --git a/src/lib/process/winprocess_sys.c b/src/lib/process/winprocess_sys.c
new file mode 100644
index 0000000000..ad65886422
--- /dev/null
+++ b/src/lib/process/winprocess_sys.c
@@ -0,0 +1,66 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.c
+ * \brief Subsystem object for windows process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/process/winprocess_sys.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef _WIN32
+#include <windows.h>
+
+#define WINPROCESS_SYS_ENABLED true
+
+static int
+subsys_winprocess_initialize(void)
+{
+#ifndef HeapEnableTerminationOnCorruption
+#define HeapEnableTerminationOnCorruption 1
+#endif
+
+  /* On heap corruption, just give up; don't try to play along. */
+  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
+
+  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
+   * (On 64-bit Windows it always fails, and some compilers don't like the
+   * PSETDEP cast.)
+   * 32-bit Windows defines _WIN32.
+   * 64-bit Windows defines _WIN32 and _WIN64. */
+#ifndef _WIN64
+  /* Call SetProcessDEPPolicy to permanently enable DEP.
+     The function will not resolve on earlier versions of Windows,
+     and failure is not dangerous. */
+  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
+  if (hMod) {
+    typedef BOOL (WINAPI *PSETDEP)(DWORD);
+    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
+                           "SetProcessDEPPolicy");
+    if (setdeppolicy) {
+      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
+      setdeppolicy(3);
+    }
+  }
+#endif /* !defined(_WIN64) */
+
+  return 0;
+}
+#else /* !defined(_WIN32) */
+#define WINPROCESS_SYS_ENABLED false
+#define subsys_winprocess_initialize NULL
+#endif /* defined(_WIN32) */
+
+const subsys_fns_t sys_winprocess = {
+  .name = "winprocess",
+  /* HeapEnableTerminationOnCorruption and setdeppolicy() are security
+   * features, we want them to run first. */
+  .level = -100,
+  .supported = WINPROCESS_SYS_ENABLED,
+  .initialize = subsys_winprocess_initialize,
+};
diff --git a/src/lib/process/winprocess_sys.h b/src/lib/process/winprocess_sys.h
new file mode 100644
index 0000000000..7ab2aa04a6
--- /dev/null
+++ b/src/lib/process/winprocess_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.h
+ * \brief Declare subsystem object for winprocess.c
+ **/
+
+#ifndef TOR_WINPROCESS_SYS_H
+#define TOR_WINPROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_winprocess;
+
+#endif /* !defined(TOR_WINPROCESS_SYS_H) */
diff --git a/src/lib/pubsub/.may_include b/src/lib/pubsub/.may_include
new file mode 100644
index 0000000000..5623492f00
--- /dev/null
+++ b/src/lib/pubsub/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/pubsub/*.h
+lib/string/*.h
diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am
new file mode 100644
index 0000000000..e2abebcd40
--- /dev/null
+++ b/src/lib/pubsub/include.am
@@ -0,0 +1,28 @@
+
+noinst_LIBRARIES += src/lib/libtor-pubsub.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_pubsub_a_SOURCES =			\
+	src/lib/pubsub/pubsub_build.c			\
+	src/lib/pubsub/pubsub_check.c			\
+	src/lib/pubsub/pubsub_publish.c
+
+src_lib_libtor_pubsub_testing_a_SOURCES = \
+	$(src_lib_libtor_pubsub_a_SOURCES)
+src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/pubsub/pub_binding_st.h			\
+	src/lib/pubsub/pubsub.h				\
+	src/lib/pubsub/pubsub_build.h			\
+	src/lib/pubsub/pubsub_builder_st.h		\
+	src/lib/pubsub/pubsub_connect.h			\
+	src/lib/pubsub/pubsub_flags.h			\
+	src/lib/pubsub/pubsub_macros.h			\
+	src/lib/pubsub/pubsub_publish.h
diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h
new file mode 100644
index 0000000000..d841bf3f54
--- /dev/null
+++ b/src/lib/pubsub/pub_binding_st.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pub_binding_st.h
+ * @brief Declaration of pub_binding_t.
+ *
+ * This is an internal type for the pubsub implementation.
+ */
+
+#ifndef TOR_PUB_BINDING_ST_H
+#define TOR_PUB_BINDING_ST_H
+
+#include "lib/dispatch/msgtypes.h"
+struct dispatch_t;
+
+/**
+ * A pub_binding_t is an opaque object that subsystems use to publish
+ * messages.  The DISPATCH_ADD_PUB*() macros set it up.
+ **/
+typedef struct pub_binding_t {
+  /**
+   * A pointer to a configured dispatch_t object.  This is filled in
+   * when the dispatch_t is finally constructed.
+   **/
+  struct dispatch_t *dispatch_ptr;
+  /**
+   * A template for the msg_t fields that are filled in for this message.
+   * This is copied into outgoing messages, ensuring that their fields are set
+   * corretly.
+   **/
+  msg_t msg_template;
+} pub_binding_t;
+
+#endif /* !defined(TOR_PUB_BINDING_ST_H) */
diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h
new file mode 100644
index 0000000000..5346b07517
--- /dev/null
+++ b/src/lib/pubsub/pubsub.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub.h
+ * @brief Header for OO publish-subscribe functionality.
+ *
+ * This module provides a wrapper around the "dispatch" module,
+ * ensuring type-safety and allowing us to do static analysis on
+ * publication and subscriptions.
+ *
+ * With this module, we enforce:
+ *  <ul>
+ *   <li>that every message has (potential) publishers and subscribers;
+ *   <li>that every message is published and subscribed from the correct
+ *       channels, with the correct type ID, every time it is published.
+ *   <li>that type IDs correspond to a single C type, and that the C types are
+ *       used correctly.
+ *   <li>that when a message is published or subscribed, it is done with
+ *       a correct subsystem identifier
+ * </ul>
+ *
+ * We do this by making "publication requests" and "subscription requests"
+ * into objects, and doing some computation on them before we create
+ * a dispatch_t with them.
+ *
+ * Rather than using the dispatch module directly, a publishing module
+ * receives a "binding" object that it uses to send messages with the right
+ * settings.
+ *
+ * Most users of this module will want to use this header, and the
+ * pubsub_macros.h header for convenience.
+ */
+
+/*
+ *
+ * Overview: Messages are sent over channels.  Before sending a message on a
+ * channel, or receiving a message on a channel, a subsystem needs to register
+ * that it publishes, or subscribes, to that message, on that channel.
+ *
+ * Messages, channels, and subsystems are represented internally as short
+ * integers, though they are associated with human-readable strings for
+ * initialization and debugging.
+ *
+ * When registering for a message, a subsystem must say whether it is an
+ * exclusive publisher/subscriber to that message type, or whether other
+ * subsystems may also publish/subscribe to it.
+ *
+ * All messages and their publishers/subscribers must be registered early in
+ * the initialization process.
+ *
+ * By default, it is an error for a message type to have publishers and no
+ * subscribers on a channel, or subscribers and no publishers on a channel.
+ *
+ * A subsystem may register for a message with a note that delivery or
+ * production is disabled -- for example, because the subsystem is
+ * disabled at compile-time. It is not an error for a message type to
+ * have all of its publishers or subscribers disabled.
+ *
+ * After a message is sent, it is delivered to every recipient.  This
+ * delivery happens from the top level of the event loop; it may be
+ * interleaved with network events, timers, etc.
+ *
+ * Messages may have associated data.  This data is typed, and is owned
+ * by the message.  Strings, byte-arrays, and integers have built-in
+ * support.  Other types may be added.  If objects are to be sent,
+ * they should be identified by handle.  If an object requires cleanup,
+ * it should be declared with an associated free function.
+ *
+ * Semantically, if two subsystems communicate only by this kind of
+ * message passing, neither is considered to depend on the other, though
+ * both are considered to have a dependency on the message and on any
+ * types it contains.
+ *
+ * (Or generational index?)
+ */
+#ifndef TOR_PUBSUB_PUBSUB_H
+#define TOR_PUBSUB_PUBSUB_H
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#endif /* !defined(TOR_PUBSUB_PUBSUB_H) */
diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c
new file mode 100644
index 0000000000..e44b7d76ec
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.c
@@ -0,0 +1,307 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.c
+ * @brief Construct a dispatch_t in safer, more OO way.
+ **/
+
+#define PUBSUB_PRIVATE
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+ #include <string.h>
+
+/** Construct and return a new empty pubsub_items_t. */
+static pubsub_items_t *
+pubsub_items_new(void)
+{
+  pubsub_items_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->items = smartlist_new();
+  cfg->type_items = smartlist_new();
+  return cfg;
+}
+
+/** Release all storage held in a pubsub_items_t. */
+void
+pubsub_items_free_(pubsub_items_t *cfg)
+{
+  if (! cfg)
+    return;
+  SMARTLIST_FOREACH(cfg->items, pubsub_cfg_t *, item, tor_free(item));
+  SMARTLIST_FOREACH(cfg->type_items,
+                    pubsub_type_cfg_t *, item, tor_free(item));
+  smartlist_free(cfg->items);
+  smartlist_free(cfg->type_items);
+  tor_free(cfg);
+}
+
+/** Construct and return a new pubsub_builder_t. */
+pubsub_builder_t *
+pubsub_builder_new(void)
+{
+  dispatch_naming_init();
+
+  pubsub_builder_t *pb = tor_malloc_zero(sizeof(*pb));
+  pb->cfg = dcfg_new();
+  pb->items = pubsub_items_new();
+  return pb;
+}
+
+/**
+ * Release all storage held by a pubsub_builder_t.
+ *
+ * You'll (mostly) only want to call this function on an error case: if you're
+ * constructing a dispatch_t instead, you should call
+ * pubsub_builder_finalize() to consume the pubsub_builder_t.
+ */
+void
+pubsub_builder_free_(pubsub_builder_t *pb)
+{
+  if (pb == NULL)
+    return;
+  pubsub_items_free(pb->items);
+  dcfg_free(pb->cfg);
+  tor_free(pb);
+}
+
+/**
+ * Create and return a pubsub_connector_t for the subsystem with ID
+ * <b>subsys</b> to use in adding publications, subscriptions, and types to
+ * <b>builder</b>.
+ **/
+pubsub_connector_t *
+pubsub_connector_for_subsystem(pubsub_builder_t *builder,
+                               subsys_id_t subsys)
+{
+  tor_assert(builder);
+  ++builder->n_connectors;
+
+  pubsub_connector_t *con = tor_malloc_zero(sizeof(*con));
+
+  con->builder = builder;
+  con->subsys_id = subsys;
+
+  return con;
+}
+
+/**
+ * Release all storage held by a pubsub_connector_t.
+ **/
+void
+pubsub_connector_free_(pubsub_connector_t *con)
+{
+  if (!con)
+    return;
+
+  if (con->builder) {
+    --con->builder->n_connectors;
+    tor_assert(con->builder->n_connectors >= 0);
+  }
+  tor_free(con);
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>.
+ **/
+int
+pubsub_add_pub_(pubsub_connector_t *con,
+                pub_binding_t *out,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  memset(out, 0, sizeof(*out));
+  cfg->is_publish = true;
+
+  out->msg_template.sender = cfg->subsys = con->subsys_id;
+  out->msg_template.channel = cfg->channel = channel;
+  out->msg_template.msg = cfg->msg = msg;
+  out->msg_template.type = cfg->type = type;
+
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  /* We're grabbing a pointer to the pub_binding_t so we can tell it about
+   * the dispatcher later on.
+   */
+  cfg->pub_binding = out;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>,
+ * passing them to the callback in <b>recv_fn</b>.
+ **/
+int
+pubsub_add_sub_(pubsub_connector_t *con,
+                recv_fn_t recv_fn,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  cfg->is_publish = false;
+  cfg->subsys = con->subsys_id;
+  cfg->channel = channel;
+  cfg->msg = msg;
+  cfg->type = type;
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  cfg->recv_fn = recv_fn;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+  if (! (flags & DISP_FLAG_STUB)) {
+    if (dcfg_add_recv(con->builder->cfg, msg, cfg->subsys, recv_fn) < 0)
+      goto err;
+  }
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to define the functions to use for manipulating the type
+ * <b>type</b>.  Any function pointers left as NULL will be implemented as
+ * no-ops.
+ **/
+int
+pubsub_connector_register_type_(pubsub_connector_t *con,
+                                msg_type_id_t type,
+                                dispatch_typefns_t *fns,
+                                const char *file,
+                                unsigned line)
+{
+  pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->type = type;
+  memcpy(&cfg->fns, fns, sizeof(*fns));
+  cfg->subsys = con->subsys_id;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  smartlist_add(con->builder->items->type_items, cfg);
+
+  if (dcfg_type_set_fns(con->builder->cfg, type, fns) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Initialize the dispatch_ptr field in every relevant publish binding
+ * for <b>d</b>.
+ */
+static void
+pubsub_items_install_bindings(pubsub_items_t *items,
+                              dispatch_t *d)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      // XXXX we could skip this for STUB publishers, and for any publishers
+      // XXXX where all subscribers are STUB.
+      cfg->pub_binding->dispatch_ptr = d;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Remove the dispatch_ptr fields for all the relevant publish bindings
+ * in <b>items</b>.  The prevents subsequent dispatch_pub_() calls from
+ * sending messages to a dispatcher that has been freed.
+ **/
+void
+pubsub_items_clear_bindings(pubsub_items_t *items)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      cfg->pub_binding->dispatch_ptr = NULL;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Create a new dispatcher as configured in a pubsub_builder_t.
+ *
+ * Consumes and frees its input.
+ **/
+dispatch_t *
+pubsub_builder_finalize(pubsub_builder_t *builder,
+                        pubsub_items_t **items_out)
+{
+  dispatch_t *dispatcher = NULL;
+  tor_assert_nonfatal(builder->n_connectors == 0);
+
+  if (pubsub_builder_check(builder) < 0)
+    goto err;
+
+  if (builder->n_errors) {
+    log_warn(LD_GENERAL, "At least one error occurred previously when "
+             "configuring the dispatcher.");
+    goto err;
+  }
+
+  dispatcher = dispatch_new(builder->cfg);
+
+  if (!dispatcher)
+    goto err;
+
+  pubsub_items_install_bindings(builder->items, dispatcher);
+  if (items_out) {
+    *items_out = builder->items;
+    builder->items = NULL; /* Prevent free */
+  }
+
+ err:
+  pubsub_builder_free(builder);
+  return dispatcher;
+}
diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h
new file mode 100644
index 0000000000..5a0c5f5bd3
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.h
@@ -0,0 +1,92 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.h
+ * @brief Header used for constructing the OO publish-subscribe facility.
+ *
+ * (See pubsub.h for more general information on this API.)
+ **/
+
+#ifndef TOR_PUBSUB_BUILD_H
+#define TOR_PUBSUB_BUILD_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct dispatch_t;
+struct pubsub_connector_t;
+
+/**
+ * A "dispatch builder" is an incomplete dispatcher, used when
+ * registering messages.  It does not have the same integrity guarantees
+ * as a dispatcher.  It cannot actually handle messages itself: once all
+ * subsystems have registered, it is converted into a dispatch_t.
+ **/
+typedef struct pubsub_builder_t pubsub_builder_t;
+
+/**
+ * A "pubsub items" holds the configuration items used to configure a
+ * pubsub_builder.  After the builder is finalized, this field is extracted,
+ * and used later to tear down pointers that enable publishing.
+ **/
+typedef struct pubsub_items_t pubsub_items_t;
+
+/**
+ * Create a new pubsub_builder. This should only happen in the
+ * main-init code.
+ */
+pubsub_builder_t *pubsub_builder_new(void);
+
+/** DOCDOC */
+int pubsub_builder_check(pubsub_builder_t *);
+
+/**
+ * Free a pubsub builder.  This should only happen on error paths, where
+ * we have decided not to construct a dispatcher for some reason.
+ */
+#define pubsub_builder_free(db) \
+  FREE_AND_NULL(pubsub_builder_t, pubsub_builder_free_, (db))
+
+/** Internal implementation of pubsub_builder_free(). */
+void pubsub_builder_free_(pubsub_builder_t *);
+
+/**
+ * Create a pubsub connector that a single subsystem will use to
+ * register its messages.  The main-init code does this during susbsystem
+ * initialization.
+ */
+struct pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *,
+                                                          subsys_id_t);
+
+/**
+ * The main-init code does this after subsystem initialization.
+ */
+#define pubsub_connector_free(c) \
+  FREE_AND_NULL(struct pubsub_connector_t, pubsub_connector_free_, (c))
+
+void pubsub_connector_free_(struct pubsub_connector_t *);
+
+/**
+ * Constructs a dispatcher from a dispatch_builder, after checking that the
+ * invariances on the messages, channels, and connections have been
+ * respected.
+ *
+ * This should happen after every subsystem has initialized, and before
+ * entering the mainloop.
+ */
+struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *,
+                                           pubsub_items_t **items_out);
+
+/**
+ * Clear all pub_binding_t backpointers in <b>items</b>.
+ **/
+void pubsub_items_clear_bindings(pubsub_items_t *items);
+
+#define pubsub_items_free(cfg) \
+  FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg))
+void pubsub_items_free_(pubsub_items_t *cfg);
+
+#endif /* !defined(TOR_PUBSUB_BUILD_H) */
diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h
new file mode 100644
index 0000000000..545aa3f3ef
--- /dev/null
+++ b/src/lib/pubsub/pubsub_builder_st.h
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_builder_st.h
+ *
+ * @brief private structures used for configuring dispatchers and messages.
+ */
+
+#ifndef TOR_PUBSUB_BUILDER_ST_H
+#define TOR_PUBSUB_BUILDER_ST_H
+
+#ifdef PUBSUB_PRIVATE
+
+#include <stdbool.h>
+#include <stddef.h>
+
+struct dispatch_cfg_t;
+struct smartlist_t;
+struct pub_binding_t;
+
+/**
+ * Configuration for a single publication or subscription request.
+ *
+ * These can be stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ *
+ * There are various fields in this request describing the message; all of
+ * them must match other descriptions of the message, or a bug has occurred.
+ **/
+typedef struct pubsub_cfg_t {
+  /** True if this is a publishing request; false for a subscribing request. */
+  bool is_publish;
+  /** The system making this request. */
+  subsys_id_t subsys;
+  /** The channel on which the message is to be sent. */
+  channel_id_t channel;
+  /** The message ID to be sent or received. */
+  message_id_t msg;
+  /** The C type associated with the message. */
+  msg_type_id_t type;
+  /** One or more DISP_FLAGS_* items, combined with bitwise OR. */
+  unsigned flags;
+
+  /**
+   * Publishing only: a pub_binding object that will receive the binding for
+   * this request.  We will finish filling this in when the dispatcher is
+   * constructed, so that the subsystem can publish then and not before.
+   */
+  struct pub_binding_t *pub_binding;
+
+  /**
+   * Subscribing only: a function to receive message objects for this request.
+   */
+  recv_fn_t recv_fn;
+
+  /** The file from which this message was configured */
+  const char *added_by_file;
+  /** The line at which this message was configured */
+  unsigned added_by_line;
+} pubsub_cfg_t;
+
+/**
+ * Configuration request for a single C type.
+ *
+ * These are stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ **/
+typedef struct pubsub_type_cfg_t {
+  /**
+   * The identifier for this type.
+   */
+  msg_type_id_t type;
+  /**
+   * Functions to use when manipulating the type.
+   */
+  dispatch_typefns_t fns;
+
+  /** The subsystem that configured this type. */
+  subsys_id_t subsys;
+  /** The file from which this type was configured */
+  const char *added_by_file;
+  /** The line at which this type was configured */
+  unsigned added_by_line;
+} pubsub_type_cfg_t;
+
+/**
+ * The set of configuration requests for a dispatcher, as made by various
+ * subsystems.
+ **/
+struct pubsub_items_t {
+  /** List of pubsub_cfg_t. */
+  struct smartlist_t *items;
+  /** List of pubsub_type_cfg_t. */
+  struct smartlist_t *type_items;
+};
+
+/**
+ * Type used to construct a dispatcher.  We use this type to build up the
+ * configuration for a dispatcher, and then pass ownership of that
+ * configuration to the newly constructed dispatcher.
+ **/
+struct pubsub_builder_t {
+  /** Number of outstanding pubsub_connector_t objects pointing to this
+   * pubsub_builder_t. */
+  int n_connectors;
+  /** Number of errors encountered while constructing this object so far. */
+  int n_errors;
+  /** In-progress configuration that we're constructing, as a list of the
+   * requests that have been made. */
+  struct pubsub_items_t *items;
+  /** In-progress configuration that we're constructing, in a form that can
+   * be converted to a dispatch_t. */
+  struct dispatch_cfg_t *cfg;
+};
+
+/**
+ * Type given to a subsystem when adding connections to a pubsub_builder_t.
+ * We use this type to force each subsystem to get blamed for the
+ * publications, subscriptions, and types that it adds.
+ **/
+struct pubsub_connector_t {
+  /** The pubsub_builder that this connector refers to. */
+  struct pubsub_builder_t *builder;
+  /** The subsystem that has been given this connector. */
+  subsys_id_t subsys_id;
+};
+
+/**
+ * Helper structure used when constructing a dispatcher that sorts the
+ * pubsub_cfg_t objects in various ways.
+ **/
+typedef struct pubsub_adjmap_t {
+  /* XXXX The next three fields are currently constructed but not yet
+   * XXXX used. I believe we'll want them in the future, though. -nickm
+   */
+  /** Number of subsystems; length of the *_by_subsys arrays. */
+  size_t n_subsystems;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **pub_by_subsys;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **sub_by_subsys;
+
+  /** Number of message IDs; length of the *_by_msg arrays. */
+  size_t n_msgs;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **pub_by_msg;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **sub_by_msg;
+} pubsub_adjmap_t;
+
+#endif /* defined(PUBSUB_PRIVATE) */
+
+#endif /* !defined(TOR_PUBSUB_BUILDER_ST_H) */
diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c
new file mode 100644
index 0000000000..bf1196df2c
--- /dev/null
+++ b/src/lib/pubsub/pubsub_check.c
@@ -0,0 +1,412 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_check.c
+ * @brief Enforce various requirements on a pubsub_builder.
+ **/
+
+#define PUBSUB_PRIVATE
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_build.h"
+
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_string.h"
+
+#include <string.h>
+
+static void pubsub_adjmap_add(pubsub_adjmap_t *map,
+                                const pubsub_cfg_t *item);
+
+/**
+ * Helper: contruct and return a new pubsub_adjacency_map from <b>cfg</b>.
+ * Return NULL on error.
+ **/
+static pubsub_adjmap_t *
+pubsub_build_adjacency_map(const pubsub_items_t *cfg)
+{
+  pubsub_adjmap_t *map = tor_malloc_zero(sizeof(*map));
+  const size_t n_subsystems = get_num_subsys_ids();
+  const size_t n_msgs = get_num_message_ids();
+
+  map->n_subsystems = n_subsystems;
+  map->n_msgs = n_msgs;
+
+  map->pub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->sub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->pub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+  map->sub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+
+  SMARTLIST_FOREACH_BEGIN(cfg->items, const pubsub_cfg_t *, item) {
+    pubsub_adjmap_add(map, item);
+  } SMARTLIST_FOREACH_END(item);
+
+  return map;
+}
+
+/**
+ * Helper: add a single pubsub_cfg_t to an adjacency map.
+ **/
+static void
+pubsub_adjmap_add(pubsub_adjmap_t *map,
+                  const pubsub_cfg_t *item)
+{
+  smartlist_t **by_subsys;
+  smartlist_t **by_msg;
+
+  tor_assert(item->subsys < map->n_subsystems);
+  tor_assert(item->msg < map->n_msgs);
+
+  if (item->is_publish) {
+    by_subsys = &map->pub_by_subsys[item->subsys];
+    by_msg = &map->pub_by_msg[item->msg];
+  } else {
+    by_subsys = &map->sub_by_subsys[item->subsys];
+    by_msg = &map->sub_by_msg[item->msg];
+  }
+
+  if (! *by_subsys)
+    *by_subsys = smartlist_new();
+  if (! *by_msg)
+    *by_msg = smartlist_new();
+  smartlist_add(*by_subsys, (void*) item);
+  smartlist_add(*by_msg, (void *) item);
+}
+
+/**
+ * Release all storage held by m and set m to NULL.
+ **/
+#define pubsub_adjmap_free(m) \
+  FREE_AND_NULL(pubsub_adjmap_t, pubsub_adjmap_free_, m)
+
+/**
+ * Free every element of an <b>n</b>-element array of smartlists, then
+ * free the array itself.
+ **/
+static void
+pubsub_adjmap_free_helper(smartlist_t **lsts, size_t n)
+{
+  if (!lsts)
+    return;
+
+  for (unsigned i = 0; i < n; ++i) {
+    smartlist_free(lsts[i]);
+  }
+  tor_free(lsts);
+}
+
+/**
+ * Release all storage held by <b>map</b>.
+ **/
+static void
+pubsub_adjmap_free_(pubsub_adjmap_t *map)
+{
+  if (!map)
+    return;
+  pubsub_adjmap_free_helper(map->pub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->sub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->pub_by_msg, map->n_msgs);
+  pubsub_adjmap_free_helper(map->sub_by_msg, map->n_msgs);
+  tor_free(map);
+}
+
+/**
+ * Helper: return the length of <b>sl</b>, or 0 if sl is NULL.
+ **/
+static int
+smartlist_len_opt(const smartlist_t *sl)
+{
+  if (sl)
+    return smartlist_len(sl);
+  else
+    return 0;
+}
+
+/** Return a pointer to a statically allocated string encoding the
+ * dispatcher flags in <b>flags</b>. */
+static const char *
+format_flags(unsigned flags)
+{
+  static char buf[32];
+  buf[0] = 0;
+  if (flags & DISP_FLAG_EXCL) {
+    strlcat(buf, " EXCL", sizeof(buf));
+  }
+  if (flags & DISP_FLAG_STUB) {
+    strlcat(buf, " STUB", sizeof(buf));
+  }
+  return buf[0] ? buf+1 : buf;
+}
+
+/**
+ * Log a message containing a description of <b>cfg</b> at severity, prefixed
+ * by the string <b>prefix</b>.
+ */
+static void
+pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix)
+{
+  tor_assert(prefix);
+
+  tor_log(severity, LD_MESG,
+          "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]",
+          prefix,
+          get_subsys_id_name(cfg->subsys),
+          cfg->is_publish ? "PUB" : "SUB",
+          get_message_id_name(cfg->msg),
+          get_msg_type_id_name(cfg->type),
+          get_channel_id_name(cfg->channel),
+          format_flags(cfg->flags),
+          cfg->subsys, cfg->msg, cfg->type, cfg->channel, cfg->flags,
+          cfg->added_by_file, cfg->added_by_line);
+}
+
+/**
+ * Helper: fill a bitarray <b>out</b> with entries corresponding to the
+ * subsystems listed in <b>items</b>.
+ **/
+static void
+get_message_bitarray(const pubsub_adjmap_t *map,
+                     const smartlist_t *items,
+                     bitarray_t **out)
+{
+  *out = bitarray_init_zero((unsigned)map->n_subsystems);
+  if (! items)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) {
+    bitarray_set(*out, cfg->subsys);
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists obey our local graph topology rules.
+ *
+ * (Right now this is just a matter of "each subsystem only
+ * publishes/subscribes once; no subsystem is a publisher and subscriber for
+ * the same message.")
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+lint_message_graph(const pubsub_adjmap_t *map,
+                   message_id_t msg,
+                   const smartlist_t *pub,
+                   const smartlist_t *sub)
+{
+  bitarray_t *published_by = NULL;
+  bitarray_t *subscribed_by = NULL;
+  bool ok = true;
+
+  get_message_bitarray(map, pub, &published_by);
+  get_message_bitarray(map, sub, &subscribed_by);
+
+  /* Check whether any subsystem is publishing and subscribing the same
+   * message. [??]
+   */
+  for (unsigned i = 0; i < map->n_subsystems; ++i) {
+    if (bitarray_is_set(published_by, i) &&
+        bitarray_is_set(subscribed_by, i)) {
+      log_warn(LD_MESG|LD_BUG,
+               "Message \"%s\" is published and subscribed by the same "
+               "subsystem \"%s\".",
+               get_message_id_name(msg),
+               get_subsys_id_name(i));
+      ok = false;
+    }
+  }
+
+  bitarray_free(published_by);
+  bitarray_free(subscribed_by);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists have compatible flags, channels, and types.
+ **/
+static int
+lint_message_consistency(message_id_t msg,
+                         const smartlist_t *pub,
+                         const smartlist_t *sub)
+{
+  if (!smartlist_len_opt(pub) && !smartlist_len_opt(sub))
+    return 0; // LCOV_EXCL_LINE -- this was already checked.
+
+  /* The 'all' list has the publishers and the subscribers. */
+  smartlist_t *all = smartlist_new();
+  if (pub)
+    smartlist_add_all(all, pub);
+  if (sub)
+    smartlist_add_all(all, sub);
+
+  const pubsub_cfg_t *item0 = smartlist_get(all, 0);
+
+  /* Indicates which subsystems we've found publishing/subscribing here. */
+  bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true;
+
+  /* Simple message consistency properties across messages.
+   */
+  SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) {
+    chan_same &= (cfg->channel == item0->channel);
+    type_same &= (cfg->type == item0->type);
+    if (cfg->is_publish)
+      pub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+    else
+      sub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+  } SMARTLIST_FOREACH_END(cfg);
+
+  bool ok = true;
+
+  if (! chan_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "channels.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (! type_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "message types.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Enforce exclusive-ness for publishers and subscribers that have asked for
+   * it.
+   */
+  if (pub_excl && smartlist_len_opt(pub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple publishers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (sub_excl && smartlist_len_opt(sub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple subscribers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  smartlist_free(all);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check whether there are any errors or inconsistencies for the message
+ * described by <b>msg</b> in <b>map</b>.  If there are problems, log about
+ * them, and return -1.  Otherwise return 0.
+ **/
+static int
+lint_message(const pubsub_adjmap_t *map, message_id_t msg)
+{
+  /* NOTE: Some of the checks in this function are maybe over-zealous, and we
+   * might not want to have them forever.  I've marked them with [?] below.
+   */
+  if (BUG(msg >= map->n_msgs))
+    return 0; // LCOV_EXCL_LINE
+
+  const smartlist_t *pub = map->pub_by_msg[msg];
+  const smartlist_t *sub = map->sub_by_msg[msg];
+
+  const size_t n_pub = smartlist_len_opt(pub);
+  const size_t n_sub = smartlist_len_opt(sub);
+
+  if (n_pub == 0 && n_sub == 0) {
+    log_info(LD_MESG, "Nobody is publishing or subscribing to message "
+             "\"%s\".",
+             get_message_id_name(msg));
+    return 0; // No publishers or subscribers: nothing to do.
+  }
+  /* We'll set this to false if there are any problems. */
+  bool ok = true;
+
+  /* First make sure that if there are publishers, there are subscribers. */
+  if (n_pub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has subscribers, but no publishers.",
+             get_message_id_name(msg));
+    ok = false;
+  } else if (n_sub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has publishers, but no subscribers.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Check the message graph topology. */
+  if (lint_message_graph(map, msg, pub, sub) < 0)
+    ok = false;
+
+  /* Check whether the messages have the same fields set on them. */
+  if (lint_message_consistency(msg, pub, sub) < 0)
+    ok = false;
+
+  if (!ok) {
+    /* There was a problem -- let's log all the publishers and subscribers on
+     * this message */
+    if (pub) {
+      SMARTLIST_FOREACH(pub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+    if (sub) {
+      SMARTLIST_FOREACH(sub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+  }
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check all the messages in <b>map</b> for consistency.  Return 0 on success,
+ * -1 on problems.
+ **/
+static int
+pubsub_adjmap_check(const pubsub_adjmap_t *map)
+{
+  bool all_ok = true;
+  for (unsigned i = 0; i < map->n_msgs; ++i) {
+    if (lint_message(map, i) < 0) {
+      all_ok = false;
+    }
+  }
+  return all_ok ? 0 : -1;
+}
+
+/**
+ * Check builder for consistency and various constraints. Return 0 on success,
+ * -1 on failure.
+ **/
+int
+pubsub_builder_check(pubsub_builder_t *builder)
+{
+  pubsub_adjmap_t *map = pubsub_build_adjacency_map(builder->items);
+  int rv = -1;
+
+  if (!map)
+    goto err; // should be impossible
+
+  if (pubsub_adjmap_check(map) < 0)
+    goto err;
+
+  rv = 0;
+ err:
+  pubsub_adjmap_free(map);
+  return rv;
+}
diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h
new file mode 100644
index 0000000000..0ad106044e
--- /dev/null
+++ b/src/lib/pubsub/pubsub_connect.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_connect.h
+ * @brief Header for functions that add relationships to a pubsub builder.
+ *
+ * These functions are used by modules that need to add publication and
+ * subscription requests.  Most users will want to call these functions
+ * indirectly, via the macros in pubsub_macros.h.
+ **/
+
+#ifndef TOR_PUBSUB_CONNECT_H
+#define TOR_PUBSUB_CONNECT_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct pub_binding_t;
+/**
+ * A "dispatch connector" is a view of the dispatcher that a subsystem
+ * uses while initializing itself.  It is specific to the subsystem, and
+ * ensures that each subsystem doesn't need to identify itself
+ * repeatedly while registering its messages.
+ **/
+typedef struct pubsub_connector_t pubsub_connector_t;
+
+int pubsub_add_pub_(struct pubsub_connector_t *con,
+                    struct pub_binding_t *out,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_add_sub_(struct pubsub_connector_t *con,
+                    recv_fn_t recv_fn,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_connector_register_type_(struct pubsub_connector_t *,
+                                    msg_type_id_t,
+                                    dispatch_typefns_t *,
+                                    const char *file,
+                                    unsigned line);
+
+#endif /* !defined(TOR_PUBSUB_CONNECT_H) */
diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h
new file mode 100644
index 0000000000..53c6e49565
--- /dev/null
+++ b/src/lib/pubsub/pubsub_flags.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_flags.h
+ * @brief Flags that can be set on publish/subscribe messages.
+ **/
+
+#ifndef TOR_PUBSUB_FLAGS_H
+#define TOR_PUBSUB_FLAGS_H
+
+/**
+ * Flag for registering a message: declare that no other module is allowed to
+ * publish this message if we are publishing it, or subscribe to it if we are
+ * subscribing to it.
+ */
+#define DISP_FLAG_EXCL (1u<<0)
+
+/**
+ * Flag for registering a message: declare that this message is a stub, and we
+ * will not actually publish/subscribe it, but that the dispatcher should
+ * treat us as if we did when typechecking.
+ *
+ * We use this so that messages aren't treated as "dangling" if they are
+ * potentially used by some other build of Tor.
+ */
+#define DISP_FLAG_STUB (1u<<1)
+
+#endif /* !defined(TOR_PUBSUB_FLAGS_H) */
diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h
new file mode 100644
index 0000000000..357e59fd54
--- /dev/null
+++ b/src/lib/pubsub/pubsub_macros.h
@@ -0,0 +1,373 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pubsub_macros.h
+ * \brief Macros to help with the publish/subscribe dispatch API.
+ *
+ * The dispatch API allows different subsystems of Tor to communicate with
+ * another asynchronously via a shared "message" system.  Some subsystems
+ * declare that they publish a given message, and others declare that they
+ * subscribe to it.  Both subsystems depend on the message, but not upon one
+ * another.
+ *
+ * To declare a message, use DECLARE_MESSAGE() (for messages that take their
+ * data as a pointer) or DECLARE_MESSAGE_INT() (for messages that take their
+ * data as an integer.  For example, you might say
+ *
+ *     DECLARE_MESSAGE(new_circuit, circ, circuit_handle_t *);
+ * or
+ *     DECLARE_MESSAGE_INT(shutdown_requested, boolean, bool);
+ *
+ * Every message has a unique name, a "type name" that the dispatch system
+ * uses to manage associated data, and a C type name.  You'll want to put
+ * these declarations in a header, to be included by all publishers and all
+ * subscribers.
+ *
+ * When a subsystem wants to publish a message, it uses DECLARE_PUBLISH() at
+ * file scope to create necessary static functions.  Then, in its subsystem
+ * initialization (in the "bind to dispatcher" callback) (TODO: name this
+ * properly!), it calls DISPATCH_ADD_PUB() to tell the dispatcher about its
+ * intent to publish.  When it actually wants to publish, it uses the
+ * PUBLISH() macro.  For example:
+ *
+ *     // At file scope
+ *     DECLARE_PUBLISH(shutdown_requested);
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_PUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ *     // somewhere in a function
+ *        {
+ *            PUBLISH(shutdown_requested, true);
+ *        }
+ *
+ * When a subsystem wants to subscribe to a message, it uses
+ * DECLARE_SUBSCRIBE() at file scope to declare static functions.  It must
+ * declare a hook function that receives the message type.  Then, in its "bind
+ * to dispatcher" function, it calls DISPATCHER_ADD_SUB() to tell the
+ * dispatcher about its intent to subscribe.  When another module publishes
+ * the message, the dispatcher will call the provided hook function.
+ *
+ *     // At file scope.  The first argument is the message that you're
+ *     // subscribing to; the second argument is the hook function to declare.
+ *     DECLARE_SUBSCRIBE(shutdown_requested, on_shutdown_req_cb);
+ *
+ *     // You need to declare this function.
+ *     static void on_shutdown_req_cb(const msg_t *msg,
+ *                                    bool value)
+ *     {
+ *         // (do something here.)
+ *     }
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_SUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ * Where did these types come from?  Somewhere in the code, you need to call
+ * DISPATCH_REGISTER_TYPE() to make sure that the dispatcher can manage the
+ * message auxiliary data.  It associates a vtbl-like structure with the
+ * type name, so that the dispatcher knows how to manipulate the type you're
+ * giving it.
+ *
+ * For example, the "boolean" type we're using above could be defined as:
+ *
+ *    static char *boolean_fmt(msg_aux_data_t d)
+ *    {
+ *        // This is used for debugging and dumping messages.
+ *        if (d.u64)
+ *            return tor_strdup("true");
+ *        else
+ *            return tor_strdup("false");
+ *    }
+ *
+ *    static void boolean_free(msg_aux_data_t d)
+ *    {
+ *        // We don't actually need to do anything to free a boolean.
+ *        // We could use "NULL" instead of this function, but I'm including
+ *        // it as an example.
+ *    }
+ *
+ *    static void bind_to_dispatcher(pubsub_connector_t *con)
+ *    {
+ *        dispatch_typefns_t boolean_fns = {
+ *            .fmt_fn = boolean_fmt,
+ *            .free_fn = boolean_free,
+ *        };
+ *        DISPATCH_REGISTER_TYPE(con, boolean, &boolean_fns);
+ *    }
+ *
+ *
+ *
+ * So, how does this all work?  (You can stop reading here, unless you're
+ * debugging something.)
+ *
+ * When you declare a message in a header with DECLARE_MESSAGE() or
+ * DECLARE_MESSAGE_INT(), it creates five things:
+ *
+ *    * two typedefs for the message argument (constant and non-constant
+ *      variants).
+ *    * a constant string to hold the declared message type name
+ *    * two inline functions, to coerce the message argument type to and from
+ *      a "msg_aux_data_t" union.
+ *
+ * All of these declarations have names based on the message name.
+ *
+ * Later, when you say DECLARE_PUBLISH() or DECLARE_SUBSCRIBE(), we use the
+ * elements defined by DECLARE_MESSAGE() to make sure that the publish
+ * function takes the correct argument type, and that the subscription hook is
+ * declared with the right argument type.
+ **/
+
+#ifndef TOR_DISPATCH_MSG_H
+#define TOR_DISPATCH_MSG_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+/* Implemenation notes:
+ *
+ * For a messagename "foo", the DECLARE_MESSAGE*() macros must declare:
+ *
+ * msg_arg_type__foo -- a typedef for the argument type of the foo message.
+ * msg_arg_consttype__foo -- a typedef for the const argument type of the
+ *     foo message.
+ * msg_arg_name__foo[] -- a static string constant holding the unique
+ *      identifier for the type of the foo message.
+ * msg_arg_get__foo() -- an inline function taking a msg_aux_data_t and
+ *      returning the C data type.
+ * msg_arg_set__foo() -- an inline function taking a msg_aux_data_t and
+ *      the C type, setting the msg_aux_data_t to hold the C type.
+ *
+ * For a messagename "foo", the DECLARE_PUBLISH() macro must declare:
+ *
+ * pub_binding__foo -- A static pub_binding_t object used to send messages
+ *      from this module.
+ * publish_fn__foo -- A function taking an argument of the appropriate
+ *      C type, to be invoked by PUBLISH().
+ *
+ * For a messagename "foo", the DECLARE_SUBSCRIBE() macro must declare:
+ *
+ * hookfn -- A user-provided function name, with the correct signature.
+ * recv_fn__foo -- A wrapper callback that takes a msg_t *, and calls
+ *      hookfn with the appropriate arguments.
+ */
+
+/* Macro to declare common elements shared by DECLARE_MESSAGE and
+ * DECLARE_MESSAGE_INT.  Don't call this directly.
+ *
+ * Note that the "msg_arg_name" string constant is defined in each
+ * translation unit.  This might be undesirable; we can tweak it in the
+ * future if need be.
+ */
+#define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)         \
+  typedef c_type msg_arg_type__ ##messagename;                          \
+  typedef const c_type msg_arg_consttype__ ##messagename;               \
+  ATTR_UNUSED static const char msg_arg_name__ ##messagename[] = # typename;
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking a pointer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.
+ * It needs to be defined somewhere in Tor, using
+ * "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_ptr_type" is a C pointer type (like "char *" or "struct foo *").
+ * The "*" needs to be included.
+ */
+#define DECLARE_MESSAGE(messagename, typename, c_ptr_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_ptr_type)           \
+  ATTR_UNUSED static inline c_ptr_type                                  \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return m.ptr;                                                       \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_ptr_type v)          \
+  {                                                                     \
+    m->ptr = v;                                                         \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking an integer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.  It
+ * needs to be defined somewhere in Tor, using "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_type" is a C integer type, like "int" or "bool".  It needs to fit inside
+ * a uint64_t.
+ */
+#define DECLARE_MESSAGE_INT(messagename, typename, c_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)               \
+  ATTR_UNUSED static inline c_type                                      \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return (c_type)m.u64;                                               \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v)              \
+  {                                                                     \
+    m->u64 = (uint64_t)v;                                               \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C module declare that we'll be publishing a given
+ * message type from within this module.
+ *
+ * It creates necessary functions and wrappers to publish a message whose
+ * unique identifier is "messagename".
+ *
+ * Before you use this, you need to include the header where DECLARE_MESSAGE*()
+ * was used for this message.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_PUBLISH(messagename)                                    \
+  static pub_binding_t pub_binding__ ##messagename;                     \
+  static void                                                           \
+  publish_fn__ ##messagename(msg_arg_type__ ##messagename arg)          \
+  {                                                                     \
+    msg_aux_data_t data;                                                \
+    msg_arg_set__ ##messagename(&data, arg);                            \
+    pubsub_pub_(&pub_binding__ ##messagename, data);                    \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C file to declare that we're subscribing to a
+ * given message and associating it with a given "hook function".  It
+ * declares the hook function static, and helps with strong typing.
+ *
+ * Before you use this, you need to include the header where
+ * DECLARE_MESSAGE*() was used for the message whose unique identifier is
+ * "messagename".
+ *
+ * You will need to define a function with the name that you provide for
+ * "hookfn".  The type of this function will be:
+ *     static void hookfn(const msg_t *, const c_type)
+ * where c_type is the c type that you declared in the header.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_SUBSCRIBE(messagename, hookfn) \
+  static void hookfn(const msg_t *,                             \
+                     const msg_arg_consttype__ ##messagename);  \
+  static void recv_fn__ ## messagename(const msg_t *m)          \
+  {                                                             \
+    msg_arg_type__ ## messagename arg;                          \
+    arg = msg_arg_get__ ##messagename(m->aux_data__);           \
+    hookfn(m, arg);                                             \
+  }                                                             \
+  EAT_SEMICOLON
+
+/**
+ * Add a fake use of the publish function for 'messagename', so that
+ * the compiler does not call it unused.
+ */
+#define DISPATCH__FAKE_USE_OF_PUBFN_(messagename)                       \
+  ( 0 ? (publish_fn__ ##messagename((msg_arg_type__##messagename)0), 1) \
+    : 1)
+
+/*
+ * This macro is for internal use.  It backs DISPATCH_ADD_PUB*()
+ */
+#define DISPATCH_ADD_PUB_(connector, channel, messagename, flags)       \
+  (                                                                     \
+    DISPATCH__FAKE_USE_OF_PUBFN_(messagename),                          \
+    pubsub_add_pub_((connector),                                        \
+                      &pub_binding__ ##messagename,                     \
+                      get_channel_id(# channel),                        \
+                    get_message_id(# messagename),                      \
+                      get_msg_type_id(msg_arg_name__ ## messagename),   \
+                      (flags),                                          \
+                      __FILE__,                                         \
+                      __LINE__)                                         \
+    )
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB(connector, channel, messagename)       \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, 0)
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type, and that no other subsystem is allowed to.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/*
+ * This macro is for internal use. It backs DISPATCH_ADD_SUB*()
+ */
+#define DISPATCH_ADD_SUB_(connector, channel, messagename, flags)       \
+  pubsub_add_sub_((connector),                                          \
+                    recv_fn__ ##messagename,                            \
+                    get_channel_id(#channel),                           \
+                    get_message_id(# messagename),                      \
+                    get_msg_type_id(msg_arg_name__ ##messagename),      \
+                    (flags),                                            \
+                    __FILE__,                                           \
+                    __LINE__)
+/*
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB(connector, channel, messagename)       \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, 0)
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type, and that no other subsystem is allowed to do
+ * so.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/**
+ * Publish a given message with a given argument.  (Takes ownership of the
+ * argument if it is a pointer.)
+ */
+#define PUBLISH(messagename, arg)               \
+  publish_fn__ ##messagename(arg)
+
+/**
+ * Use a given connector to declare that the functions to be used to manipuate
+ * a certain C type.
+ **/
+#define DISPATCH_REGISTER_TYPE(con, type, fns)                  \
+  pubsub_connector_register_type_((con),                        \
+                                  get_msg_type_id(#type),       \
+                                  (fns),                        \
+                                  __FILE__,                     \
+                                  __LINE__)
+
+#endif /* !defined(TOR_DISPATCH_MSG_H) */
diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c
new file mode 100644
index 0000000000..454a335a78
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.c
@@ -0,0 +1,72 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_publish.c
+ * @brief Header for functions to publish using a pub_binding_t.
+ **/
+
+#define PUBSUB_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Publish a message from the publication binding <b>pub</b> using the
+ * auxiliary data <b>auxdata</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata)
+{
+  dispatch_t *d = pub->dispatch_ptr;
+  if (BUG(! d)) {
+    /* Tried to publish a message before the dispatcher was configured. */
+    /* (Without a dispatcher, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.type >= d->n_types)) {
+    /* The type associated with this message is not known to the dispatcher. */
+    /* (Without a correct type, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.msg >= d->n_msgs) ||
+      BUG(pub->msg_template.channel >= d->n_queues)) {
+    /* The message ID or channel ID was out of bounds. */
+    // LCOV_EXCL_START
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return -1;
+    // LCOV_EXCL_STOP
+  }
+
+  if (! d->table[pub->msg_template.msg]) {
+    /* Fast path: nobody wants this data. */
+
+    // XXXX Faster path: we could store this in the pub_binding_t.
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return 0;
+  }
+
+  /* Construct the message object */
+  msg_t *m = tor_malloc(sizeof(msg_t));
+  memcpy(m, &pub->msg_template, sizeof(msg_t));
+  m->aux_data__ = auxdata;
+
+  return dispatch_send_msg_unchecked(d, m);
+}
diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h
new file mode 100644
index 0000000000..0686a465de
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.h
@@ -0,0 +1,15 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PUBSUB_PUBLISH_H
+#define TOR_PUBSUB_PUBLISH_H
+
+#include "lib/dispatch/msgtypes.h"
+struct pub_binding_t;
+
+int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata);
+
+#endif /* !defined(TOR_PUBSUB_PUBLISH_H) */
diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include
index 84906dfb3d..853dae7880 100644
--- a/src/lib/sandbox/.may_include
+++ b/src/lib/sandbox/.may_include
@@ -5,11 +5,10 @@ lib/container/*.h
 lib/err/*.h
 lib/log/*.h
 lib/malloc/*.h
-lib/net/*.h
 lib/sandbox/*.h
 lib/sandbox/*.inc
 lib/string/*.h
 
-ht.h
-siphash.h
-tor_queue.h
+ext/ht.h
+ext/siphash.h
+ext/tor_queue.h
diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am
index adfda6bde5..e81f14b55f 100644
--- a/src/lib/sandbox/include.am
+++ b/src/lib/sandbox/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_sandbox_a_SOURCES =			\
 	src/lib/sandbox/sandbox.c
 
@@ -13,6 +14,7 @@ src_lib_libtor_sandbox_testing_a_SOURCES = \
 src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/sandbox/linux_syscalls.inc		\
 	src/lib/sandbox/sandbox.h
diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c
index 8f577b0660..3515e8854e 100644
--- a/src/lib/sandbox/sandbox.c
+++ b/src/lib/sandbox/sandbox.c
@@ -38,13 +38,12 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "lib/cc/torint.h"
-#include "lib/net/resolve.h"
 #include "lib/malloc/malloc.h"
 #include "lib/string/scanf.h"
 
-#include "tor_queue.h"
-#include "ht.h"
-#include "siphash.h"
+#include "ext/tor_queue.h"
+#include "ext/ht.h"
+#include "ext/siphash.h"
 
 #define DEBUGGING_CLOSE
 
@@ -144,6 +143,7 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(clock_gettime),
     SCMP_SYS(close),
     SCMP_SYS(clone),
+    SCMP_SYS(dup),
     SCMP_SYS(epoll_create),
     SCMP_SYS(epoll_wait),
 #ifdef __NR_epoll_pwait
@@ -295,6 +295,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   unsigned i;
   int rc;
   int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD,
+                  SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, SIGIO,
 #ifdef SIGXFSZ
       SIGXFSZ
 #endif
@@ -444,7 +445,7 @@ libc_uses_openat_for_everything(void)
     return 1;
   else
     return 0;
-#else /* !(defined(CHECK_LIBC_VERSION)) */
+#else /* !defined(CHECK_LIBC_VERSION) */
   return 0;
 #endif /* defined(CHECK_LIBC_VERSION) */
 }
@@ -798,6 +799,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   if (rc)
     return rc;
 
+  rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
+      SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
+      SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN));
+  if (rc)
+    return rc;
+
 #ifdef HAVE_SYSTEMD
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
       SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
@@ -1517,7 +1524,6 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // marking the sandbox as active
   sandbox_active = 1;
-  tor_make_getaddrinfo_cache_active();
 
  end:
   seccomp_release(ctx);
@@ -1764,9 +1770,4 @@ sandbox_is_active(void)
   return 0;
 }
 
-void
-sandbox_disable_getaddrinfo_cache(void)
-{
-}
-
 #endif /* !defined(USE_LIBSECCOMP) */
diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h
index 5bec09a36a..b4ae6e5c07 100644
--- a/src/lib/sandbox/sandbox.h
+++ b/src/lib/sandbox/sandbox.h
@@ -108,7 +108,7 @@ typedef struct {
  * it matches the parameter.
  */
 const char* sandbox_intern_string(const char *param);
-#else /* !(defined(USE_LIBSECCOMP)) */
+#else /* !defined(USE_LIBSECCOMP) */
 #define sandbox_intern_string(s) (s)
 #endif /* defined(USE_LIBSECCOMP) */
 
diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include
index a8507761a4..2f0c8d341e 100644
--- a/src/lib/smartlist_core/.may_include
+++ b/src/lib/smartlist_core/.may_include
@@ -4,4 +4,4 @@ lib/malloc/*.h
 lib/err/*.h
 lib/string/*.h
 lib/smartlist_core/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am
index 99d65f0b23..548179bc4f 100644
--- a/src/lib/smartlist_core/include.am
+++ b/src/lib/smartlist_core/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_smartlist_core_a_SOURCES =			\
 	src/lib/smartlist_core/smartlist_core.c                 \
 	src/lib/smartlist_core/smartlist_split.c
@@ -15,6 +16,7 @@ src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \
 	$(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/smartlist_core/smartlist_core.h         \
 	src/lib/smartlist_core/smartlist_foreach.h      \
diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c
index ac85a6cc84..6b0a305a93 100644
--- a/src/lib/smartlist_core/smartlist_core.c
+++ b/src/lib/smartlist_core/smartlist_core.c
@@ -88,6 +88,30 @@ smartlist_ensure_capacity(smartlist_t *sl, size_t size)
 #undef MAX_CAPACITY
 }
 
+/** Expand <b>sl</b> so that its length is at least <b>new_size</b>,
+ * filling in previously unused entries with NULL>
+ *
+ * Do nothing if <b>sl</b> already had at least <b>new_size</b> elements.
+ */
+void
+smartlist_grow(smartlist_t *sl, size_t new_size)
+{
+  smartlist_ensure_capacity(sl, new_size);
+
+  if (new_size > (size_t)sl->num_used) {
+    /* This memset() should be a no-op: everything else in the smartlist code
+     * tries to make sure that unused entries are always NULL.  Still, that is
+     * meant as a safety mechanism, so let's clear the memory here.
+     */
+    memset(sl->list + sl->num_used, 0,
+           sizeof(void *) * (new_size - sl->num_used));
+
+    /* This cast is safe, since we already asserted that we were below
+     * MAX_CAPACITY in smartlist_ensure_capacity(). */
+    sl->num_used = (int)new_size;
+  }
+}
+
 /** Append element to the end of the list. */
 void
 smartlist_add(smartlist_t *sl, void *element)
@@ -153,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element)
       sl->list[i++] = sl->list[j];
     }
   }
+  memset(sl->list + sl->num_used, 0,
+         sizeof(void *) * (num_used_orig - sl->num_used));
 }
 
 /** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h
index a7fbaa099b..36f23e2009 100644
--- a/src/lib/smartlist_core/smartlist_core.h
+++ b/src/lib/smartlist_core/smartlist_core.h
@@ -43,6 +43,7 @@ void smartlist_clear(smartlist_t *sl);
 void smartlist_add(smartlist_t *sl, void *element);
 void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
 void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
+void smartlist_grow(smartlist_t *sl, size_t new_size);
 
 void smartlist_remove(smartlist_t *sl, const void *element);
 void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
@@ -76,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
   raw_assert(sl->num_used > idx);
   sl->list[idx] = val;
 }
-#else /* !(defined(DEBUG_SMARTLIST)) */
+#else /* !defined(DEBUG_SMARTLIST) */
 #define smartlist_len(sl) ((sl)->num_used)
 #define smartlist_get(sl, idx) ((sl)->list[idx])
 #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
@@ -97,4 +98,4 @@ void smartlist_del(smartlist_t *sl, int idx);
 void smartlist_del_keeporder(smartlist_t *sl, int idx);
 void smartlist_insert(smartlist_t *sl, int idx, void *val);
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_CORE_H) */
diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h
index 0f6fe30074..a1fbcd444c 100644
--- a/src/lib/smartlist_core/smartlist_foreach.h
+++ b/src/lib/smartlist_core/smartlist_foreach.h
@@ -83,6 +83,19 @@
          ++var ## _sl_idx) {                                    \
       var = (sl)->list[var ## _sl_idx];
 
+/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to
+ *  SMARTLIST_FOREACH_BEGIN
+ *
+ * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT.
+ */
+#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var)  \
+  STMT_BEGIN                                                       \
+    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;             \
+    type var;                                                      \
+    for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0;   \
+         --var ## _sl_idx) {                                       \
+      var = (sl)->list[var ## _sl_idx];
+
 #define SMARTLIST_FOREACH_END(var)              \
     var = NULL;                                 \
     (void) var ## _sl_idx;                      \
diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h
index 4f72376125..e2cc7245b7 100644
--- a/src/lib/smartlist_core/smartlist_split.h
+++ b/src/lib/smartlist_core/smartlist_split.h
@@ -17,4 +17,4 @@
 int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
                            int flags, int max);
 
-#endif
+#endif /* !defined(TOR_SMARTLIST_SPLIT_H) */
diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include
index ec5c769831..1fb9127f19 100644
--- a/src/lib/string/.may_include
+++ b/src/lib/string/.may_include
@@ -6,5 +6,5 @@ lib/malloc/*.h
 lib/ctime/*.h
 lib/string/*.h
 
-strlcat.c
-strlcpy.c
+ext/strlcat.c
+ext/strlcpy.c
diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c
index e125c921a4..187f784be5 100644
--- a/src/lib/string/compat_string.c
+++ b/src/lib/string/compat_string.c
@@ -14,10 +14,10 @@
 
 /* Inline the strl functions if the platform doesn't have them. */
 #ifndef HAVE_STRLCPY
-#include "strlcpy.c"
+#include "ext/strlcpy.c"
 #endif
 #ifndef HAVE_STRLCAT
-#include "strlcat.c"
+#include "ext/strlcat.c"
 #endif
 
 #include <stdlib.h>
diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h
index a0e37bb6dc..ffc892c3e5 100644
--- a/src/lib/string/compat_string.h
+++ b/src/lib/string/compat_string.h
@@ -25,20 +25,23 @@ static inline int strncasecmp(const char *a, const char *b, size_t n);
 static inline int strncasecmp(const char *a, const char *b, size_t n) {
   return _strnicmp(a,b,n);
 }
-#endif
+#endif /* !defined(HAVE_STRNCASECMP) */
 #ifndef HAVE_STRCASECMP
 static inline int strcasecmp(const char *a, const char *b);
 static inline int strcasecmp(const char *a, const char *b) {
   return _stricmp(a,b);
 }
-#endif
-#endif
+#endif /* !defined(HAVE_STRCASECMP) */
+#endif /* defined(_WIN32) */
 
 #if defined __APPLE__
 /* On OSX 10.9 and later, the overlap-checking code for strlcat would
  * appear to have a severe bug that can sometimes cause aborts in Tor.
  * Instead, use the non-checking variants.  This is sad.
  *
+ * (If --enable-fragile-hardening is passed to configure, we use the hardened
+ * variants, which do not suffer from this issue.)
+ *
  * See https://trac.torproject.org/projects/tor/ticket/15205
  */
 #undef strlcat
@@ -59,4 +62,4 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
 #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts)
 #endif
 
-#endif
+#endif /* !defined(TOR_COMPAT_STRING_H) */
diff --git a/src/lib/string/include.am b/src/lib/string/include.am
index edd74b8a3e..82d35cc5af 100644
--- a/src/lib/string/include.am
+++ b/src/lib/string/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-string-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_string_a_SOURCES =			\
 	src/lib/string/compat_ctype.c			\
 	src/lib/string/compat_string.c			\
@@ -18,6 +19,7 @@ src_lib_libtor_string_testing_a_SOURCES = \
 src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/string/compat_ctype.h			\
 	src/lib/string/compat_string.h			\
diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h
index 925547942e..50d48b44c5 100644
--- a/src/lib/string/parse_int.h
+++ b/src/lib/string/parse_int.h
@@ -22,4 +22,4 @@ double tor_parse_double(const char *s, double min, double max, int *ok,
 uint64_t tor_parse_uint64(const char *s, int base, uint64_t min,
                          uint64_t max, int *ok, char **next);
 
-#endif
+#endif /* !defined(TOR_PARSE_INT_H) */
diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c
index a5cb71ce09..26203932e4 100644
--- a/src/lib/string/printf.c
+++ b/src/lib/string/printf.c
@@ -117,8 +117,8 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = NULL;
     return -1;
   }
-  strp_tmp = tor_malloc(len + 1);
-  r = _vsnprintf(strp_tmp, len+1, fmt, args);
+  strp_tmp = tor_malloc((size_t)len + 1);
+  r = _vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
@@ -153,9 +153,9 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = tor_strdup(buf);
     return len;
   }
-  strp_tmp = tor_malloc(len+1);
+  strp_tmp = tor_malloc((size_t)len+1);
   /* use of tor_vsnprintf() will ensure string is null terminated */
-  r = tor_vsnprintf(strp_tmp, len+1, fmt, args);
+  r = tor_vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h
index 2cc13d6bee..6e90770f81 100644
--- a/src/lib/string/printf.h
+++ b/src/lib/string/printf.h
@@ -27,4 +27,4 @@ int tor_asprintf(char **strp, const char *fmt, ...)
 int tor_vasprintf(char **strp, const char *fmt, va_list args)
   CHECK_PRINTF(2,0);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_PRINTF_H) */
diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h
index 6673173de5..b642e242db 100644
--- a/src/lib/string/scanf.h
+++ b/src/lib/string/scanf.h
@@ -21,4 +21,4 @@ int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \
 int tor_sscanf(const char *buf, const char *pattern, ...)
   CHECK_SCANF(2, 3);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_SCANF_H) */
diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c
index f934f66f02..f5061a11d2 100644
--- a/src/lib/string/util_string.c
+++ b/src/lib/string/util_string.c
@@ -71,7 +71,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle)
 
 /** Return true iff the 'len' bytes at 'mem' are all zero. */
 int
-tor_mem_is_zero(const char *mem, size_t len)
+fast_mem_is_zero(const char *mem, size_t len)
 {
   static const char ZERO[] = {
     0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
@@ -95,17 +95,14 @@ tor_mem_is_zero(const char *mem, size_t len)
 int
 tor_digest_is_zero(const char *digest)
 {
-  static const uint8_t ZERO_DIGEST[] = {
-    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
-  };
-  return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN);
+  return safe_mem_is_zero(digest, DIGEST_LEN);
 }
 
 /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
 int
 tor_digest256_is_zero(const char *digest)
 {
-  return tor_mem_is_zero(digest, DIGEST256_LEN);
+  return safe_mem_is_zero(digest, DIGEST256_LEN);
 }
 
 /** Remove from the string <b>s</b> every character which appears in
@@ -212,21 +209,6 @@ strcmpstart(const char *s1, const char *s2)
   return strncmp(s1, s2, n);
 }
 
-/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>,
- * without depending on a terminating nul in s1.  Sorting order is first by
- * length, then lexically; return values are as for strcmp.
- */
-int
-strcmp_len(const char *s1, const char *s2, size_t s1_len)
-{
-  size_t s2_len = strlen(s2);
-  if (s1_len < s2_len)
-    return -1;
-  if (s1_len > s2_len)
-    return 1;
-  return fast_memcmp(s1, s2, s2_len);
-}
-
 /** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
  * strcasecmp.
  */
@@ -541,3 +523,16 @@ string_is_utf8(const char *str, size_t len)
   }
   return true;
 }
+
+/** As string_is_utf8(), but returns false if the string begins with a UTF-8
+ * byte order mark (BOM).
+ */
+int
+string_is_utf8_no_bom(const char *str, size_t len)
+{
+  if (len >= 3 && (!strcmpstart(str, "\uFEFF") ||
+                   !strcmpstart(str, "\uFFFE"))) {
+    return false;
+  }
+  return string_is_utf8(str, len);
+}
diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h
index d9fbf8c61e..b3c6841d41 100644
--- a/src/lib/string/util_string.h
+++ b/src/lib/string/util_string.h
@@ -20,7 +20,10 @@ const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
                        size_t nlen);
 const void *tor_memstr(const void *haystack, size_t hlen,
                        const char *needle);
-int tor_mem_is_zero(const char *mem, size_t len);
+int fast_mem_is_zero(const char *mem, size_t len);
+#define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN)
+#define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN)
+
 int tor_digest_is_zero(const char *digest);
 int tor_digest256_is_zero(const char *digest);
 
@@ -33,7 +36,6 @@ int tor_strisnonupper(const char *s);
 int tor_strisspace(const char *s);
 int strcmp_opt(const char *s1, const char *s2);
 int strcmpstart(const char *s1, const char *s2);
-int strcmp_len(const char *s1, const char *s2, size_t len);
 int strcasecmpstart(const char *s1, const char *s2);
 int strcmpend(const char *s1, const char *s2);
 int strcasecmpend(const char *s1, const char *s2);
@@ -53,5 +55,6 @@ const char *find_str_at_start_of_line(const char *haystack,
 int string_is_C_identifier(const char *string);
 
 int string_is_utf8(const char *str, size_t len);
+int string_is_utf8_no_bom(const char *str, size_t len);
 
 #endif /* !defined(TOR_UTIL_STRING_H) */
diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include
new file mode 100644
index 0000000000..2b06e8519c
--- /dev/null
+++ b/src/lib/subsys/.may_include
@@ -0,0 +1 @@
+orconfig.h
diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am
new file mode 100644
index 0000000000..c9ab54ca73
--- /dev/null
+++ b/src/lib/subsys/include.am
@@ -0,0 +1,4 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS += \
+	src/lib/subsys/subsys.h
diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h
new file mode 100644
index 0000000000..21f984f32d
--- /dev/null
+++ b/src/lib/subsys/subsys.h
@@ -0,0 +1,95 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_SUBSYS_T
+#define TOR_SUBSYS_T
+
+#include <stdbool.h>
+
+struct pubsub_connector_t;
+
+/**
+ * A subsystem is a part of Tor that is initialized, shut down, configured,
+ * and connected to other parts of Tor.
+ *
+ * All callbacks are optional -- if a callback is set to NULL, the subsystem
+ * manager will treat it as a no-op.
+ *
+ * You should use c99 named-field initializers with this structure: we
+ * will be adding more fields, often in the middle of the structure.
+ **/
+typedef struct subsys_fns_t {
+  /**
+   * The name of this subsystem.  It should be a programmer-readable
+   * identifier.
+   **/
+  const char *name;
+
+  /**
+   * Whether this subsystem is supported -- that is, whether it is compiled
+   * into Tor.  For most subsystems, this should be true.
+   **/
+  bool supported;
+
+  /**
+   * The 'initialization level' for the subsystem.  It should run from -100
+   * through +100.  The subsystems are initialized from lowest level to
+   * highest, and shut down from highest level to lowest.
+   **/
+  int level;
+
+  /**
+   * Initialize any global components of this subsystem.
+   *
+   * This function MAY rely on any lower-level subsystem being initialized.
+   *
+   * This function MUST NOT rely on any runtime configuration information;
+   * it is only for global state or pre-configuration state.
+   *
+   * (If you need to do any setup that depends on configuration, you'll need
+   * to declare a configuration callback. (Not yet designed))
+   *
+   * This function MUST NOT have any parts that can fail.
+   **/
+  int (*initialize)(void);
+
+  /**
+   * Connect a subsystem to the message dispatch system.
+   **/
+  int (*add_pubsub)(struct pubsub_connector_t *);
+
+  /**
+   * Perform any necessary pre-fork cleanup.  This function may not fail.
+   */
+  void (*prefork)(void);
+
+  /**
+   * Perform any necessary post-fork setup. This function may not fail.
+   */
+  void (*postfork)(void);
+
+  /**
+   * Free any thread-local resources held by this subsystem. Called before
+   * the thread exits.
+   */
+  void (*thread_cleanup)(void);
+
+  /**
+   * Free all resources held by this subsystem.
+   *
+   * This function is not allowed to fail.
+   **/
+  void (*shutdown)(void);
+
+} subsys_fns_t;
+
+#define MIN_SUBSYS_LEVEL -100
+#define MAX_SUBSYS_LEVEL 100
+
+/* All tor "libraries" (in src/libs) should have a subsystem level equal to or
+ * less than this value. */
+#define SUBSYS_LEVEL_LIBS -10
+
+#endif /* !defined(TOR_SUBSYS_T) */
diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include
index c93a06e59e..306fa57b7a 100644
--- a/src/lib/term/.may_include
+++ b/src/lib/term/.may_include
@@ -5,5 +5,4 @@ lib/log/*.h
 lib/term/*.h
 lib/malloc/*.h
 
-# From src/ext
-tor_readpassphrase.h
+ext/tor_readpassphrase.h
diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c
index c6459f250f..8741344acf 100644
--- a/src/lib/term/getpass.c
+++ b/src/lib/term/getpass.c
@@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #elif defined(HAVE_READPASSPHRASE_H)
 #include <readpassphrase.h>
 #else
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #endif /* defined(_WIN32) || ... */
 
 #include <stdlib.h>
diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h
index a9c146ea8f..aa597ec423 100644
--- a/src/lib/term/getpass.h
+++ b/src/lib/term/getpass.h
@@ -15,4 +15,4 @@
 
 ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
 
-#endif
+#endif /* !defined(TOR_GETPASS_H) */
diff --git a/src/lib/term/include.am b/src/lib/term/include.am
index 55fe548ebc..a120bba0cb 100644
--- a/src/lib/term/include.am
+++ b/src/lib/term/include.am
@@ -11,6 +11,7 @@ else
 readpassphrase_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_term_a_SOURCES =			\
 	src/lib/term/getpass.c			\
 	$(readpassphrase_source)
@@ -20,5 +21,6 @@ src_lib_libtor_term_testing_a_SOURCES = \
 src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/term/getpass.h
diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am
index b2aa620985..a5ed46eb67 100644
--- a/src/lib/testsupport/include.am
+++ b/src/lib/testsupport/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/testsupport/testsupport.h
diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h
index 9363a9ba66..90b7c43b19 100644
--- a/src/lib/testsupport/testsupport.h
+++ b/src/lib/testsupport/testsupport.h
@@ -21,7 +21,7 @@
  * tests. */
 #define STATIC
 #define EXTERN(type, name) extern type name;
-#else
+#else /* !defined(TOR_UNIT_TESTS) */
 #define STATIC static
 #define EXTERN(type, name)
 #endif /* defined(TOR_UNIT_TESTS) */
@@ -90,7 +90,7 @@
   do {                                          \
     func = func ##__real;                       \
   } while (0)
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
 #define MOCK_DECL(rv, funcname, arglist) \
   rv funcname arglist
 #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include
index fc56f46836..02711348c5 100644
--- a/src/lib/thread/.may_include
+++ b/src/lib/thread/.may_include
@@ -2,6 +2,7 @@ orconfig.h
 lib/cc/*.h
 lib/lock/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/wallclock/*.h
diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c
index 94ab021c52..5c8ffa55c6 100644
--- a/src/lib/thread/compat_threads.c
+++ b/src/lib/thread/compat_threads.c
@@ -14,9 +14,11 @@
 #include "orconfig.h"
 #include <stdlib.h>
 #include "lib/thread/threads.h"
+#include "lib/thread/thread_sys.h"
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
+#include "lib/subsys/subsys.h"
 
 #include <string.h>
 
@@ -65,7 +67,15 @@ atomic_counter_init(atomic_counter_t *counter)
   memset(counter, 0, sizeof(*counter));
   tor_mutex_init_nonrecursive(&counter->mutex);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -109,3 +119,19 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return oldval;
 }
 #endif /* !defined(HAVE_WORKING_STDATOMIC) */
+
+static int
+subsys_threads_initialize(void)
+{
+  tor_threads_init();
+  return 0;
+}
+
+const subsys_fns_t sys_threads = {
+  .name = "threads",
+  .supported = true,
+  /* Threads is used by logging, which is a diagnostic feature, we want it to
+   * init right after low-level error handling and approx time. */
+  .level = -95,
+  .initialize = subsys_threads_initialize,
+};
diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am
index 9ec23d166e..cd8016b5df 100644
--- a/src/lib/thread/include.am
+++ b/src/lib/thread/include.am
@@ -12,6 +12,7 @@ if THREADS_WIN32
 threads_impl_source=src/lib/thread/compat_winthreads.c
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_thread_a_SOURCES =			\
 	src/lib/thread/compat_threads.c			\
 	src/lib/thread/numcpus.c			\
@@ -22,6 +23,8 @@ src_lib_libtor_thread_testing_a_SOURCES = \
 src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/thread/threads.h			\
-	src/lib/thread/numcpus.h
+	src/lib/thread/numcpus.h			\
+	src/lib/thread/thread_sys.h			\
+	src/lib/thread/threads.h
diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h
index 3f0a29ce7c..2f1ea16eb9 100644
--- a/src/lib/thread/numcpus.h
+++ b/src/lib/thread/numcpus.h
@@ -13,4 +13,4 @@
 
 int compute_num_cpus(void);
 
-#endif
+#endif /* !defined(TOR_NUMCPUS_H) */
diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h
new file mode 100644
index 0000000000..c0daf2b5e9
--- /dev/null
+++ b/src/lib/thread/thread_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file threads_sys.h
+ * \brief Declare subsystem object for threads library
+ **/
+
+#ifndef TOR_THREADS_SYS_H
+#define TOR_THREADS_SYS_H
+
+extern const struct subsys_fns_t sys_threads;
+
+#endif /* !defined(TOR_THREADS_SYS_H) */
diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h
index ecf60641b5..4b42b9abd9 100644
--- a/src/lib/thread/threads.h
+++ b/src/lib/thread/threads.h
@@ -107,7 +107,7 @@ typedef struct atomic_counter_t {
   atomic_size_t val;
 } atomic_counter_t;
 #define ATOMIC_LINKAGE static
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 typedef struct atomic_counter_t {
   tor_mutex_t mutex;
   size_t val;
@@ -131,7 +131,17 @@ atomic_counter_init(atomic_counter_t *counter)
 {
   atomic_init(&counter->val, 0);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * This usage note applies to the compat_threads implementation of
+ * atomic_counter_destroy():
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 static inline void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -162,7 +172,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return atomic_exchange(&counter->val, newval);
 }
 
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 #endif /* defined(HAVE_WORKING_STDATOMIC) */
 
 #endif /* !defined(TOR_COMPAT_THREADS_H) */
diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include
index 2c7e37a836..ae01431b60 100644
--- a/src/lib/time/.may_include
+++ b/src/lib/time/.may_include
@@ -4,8 +4,10 @@ lib/cc/*.h
 lib/err/*.h
 lib/intmath/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/time/*.h
 lib/wallclock/*.h
+lib/defs/time.h
 
 # For load_windows_system_lib.
 lib/fs/winlib.h
\ No newline at end of file
diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c
index 98854bad2c..ab45224a7f 100644
--- a/src/lib/time/compat_time.c
+++ b/src/lib/time/compat_time.c
@@ -164,6 +164,8 @@ static int64_t last_tick_count = 0;
  * to be monotonic; increments them as appropriate so that they actually
  * _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC int64_t
 ratchet_performance_counter(int64_t count_raw)
@@ -202,6 +204,8 @@ static struct timeval timeofday_offset = { 0, 0 };
  * supposed to be monotonic; increments them as appropriate so that they
  * actually _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC void
 ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
@@ -270,7 +274,9 @@ monotime_init_internal(void)
 }
 
 /**
- * Set "out" to the most recent monotonic time value
+ * Set "out" to the most recent monotonic time value.
+ *
+ * The returned time may be the same as the previous returned time.
  */
 void
 monotime_get(monotime_t *out)
@@ -298,10 +304,12 @@ monotime_coarse_get(monotime_coarse_t *out)
 #endif /* defined(TOR_UNIT_TESTS) */
   out->abstime_ = mach_approximate_time();
 }
-#endif
+#endif /* defined(HAVE_MACH_APPROXIMATE_TIME) */
 
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ *
+ * The returned value may be equal to zero.
  */
 int64_t
 monotime_diff_nsec(const monotime_t *start,
@@ -522,7 +530,9 @@ monotime_init_internal(void)
     GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void))
       GetProcAddress(h, "GetTickCount64");
   }
-  // FreeLibrary(h) ?
+  // We can't call FreeLibrary(h) here, because freeing the handle may
+  // unload the library, and cause future calls to GetTickCount64_fn()
+  // to fail. See 29642 for details.
 }
 
 void
@@ -757,7 +767,7 @@ monotime_coarse_zero(monotime_coarse_t *out)
 {
   memset(out, 0, sizeof(*out));
 }
-#endif
+#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 
 int64_t
 monotime_diff_usec(const monotime_t *start,
@@ -787,8 +797,8 @@ monotime_absolute_nsec(void)
   return monotime_diff_nsec(&initialized_at, &now);
 }
 
-uint64_t
-monotime_absolute_usec(void)
+MOCK_IMPL(uint64_t,
+monotime_absolute_usec,(void))
 {
   return monotime_absolute_nsec() / 1000;
 }
@@ -823,7 +833,7 @@ monotime_coarse_absolute_msec(void)
 {
   return monotime_coarse_absolute_nsec() / ONE_MILLION;
 }
-#else
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define initialized_at_coarse initialized_at
 #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 
@@ -855,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
     mach_time_info.numer;
   return abstime_val >> monotime_shift;
 }
-#else
+#else /* !defined(__APPLE__) */
 uint64_t
 monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
 {
@@ -866,4 +876,4 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
 {
   return (msec * STAMP_TICKS_PER_SECOND) / 1000;
 }
-#endif
+#endif /* defined(__APPLE__) */
diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h
index 480d426ac7..4d16effd29 100644
--- a/src/lib/time/compat_time.h
+++ b/src/lib/time/compat_time.h
@@ -15,6 +15,120 @@
  * of tens of milliseconds.
  */
 
+/* Q: When should I use monotonic time?
+ *
+ * A: If you need a time that never decreases, use monotonic time. If you need
+ * to send a time to a user or another process, or store a time, use the
+ * wall-clock time.
+ *
+ * Q: Should you use monotime or monotime_coarse as your source?
+ *
+ * A: Generally, you get better precision with monotime, but better
+ * performance with monotime_coarse.
+ *
+ * Q: What is a "monotonic" time, exactly?
+ *
+ * A: Monotonic times are strictly non-decreasing. The difference between any
+ * previous monotonic time, and the current monotonic time, is always greater
+ * than *or equal to* zero.
+ * Zero deltas happen more often:
+ *  - on Windows (due to an OS bug),
+ *  - when using monotime_coarse, or on systems with low-resolution timers,
+ *  - on platforms where we emulate monotonic time using wall-clock time, and
+ *  - when using time units that are larger than nanoseconds (due to
+ *    truncation on division).
+ *
+ * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use
+ *    usec? msec? "stamp units?"
+ *
+ * A: Using monotime_t and monotime_coarse_t directly is most time-efficient,
+ * since no conversion needs to happen.  But they can potentially use more
+ * memory than you would need for a usec/msec/"stamp unit" count.
+ *
+ * Converting to usec or msec on some platforms, and working with them in
+ * general, creates a risk of doing a 64-bit division.  64-bit division is
+ * expensive on 32-bit platforms, which still do exist.
+ *
+ * The "stamp unit" type is designed to give a type that is cheap to convert
+ * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a
+ * 32-bit integer.  Its downside is that it does not correspond directly
+ * to a natural unit of time.
+ *
+ * There is not much point in using "coarse usec" or "coarse nsec", since the
+ * current coarse monotime implementations give you on the order of
+ * milliseconds of precision.
+ *
+ * Q: So, what backends is monotime_coarse using?
+ *
+ * A: Generally speaking, it uses "whatever monotonic-ish time implemenation
+ * does not require a context switch."  The various implementations provide
+ * this by having a view of the current time in a read-only memory page that
+ * is updated with a frequency corresponding to the kernel's tick count.
+ *
+ * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on
+ * obsolete systems].  MSDN claims that the resolution is "typically in the
+ * range of 10-16 msec", but it has said that for years.  Storing
+ * monotime_coarse_t uses 8 bytes.
+ *
+ * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where
+ * available, and falls back to regular monotime. The precision is not
+ * documented, but the implementation is open-source: it reads from a page
+ * that the kernel updates. Storing monotime_coarse_t uses 8 bytes.
+ *
+ * On unixy systems, monotime_coarse uses clock_gettime() with
+ * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC.
+ * It typically uses vdso tricks to read from a page that the kernel updates.
+ * Its precision fixed, but you can get it with clock_getres(): on my Linux
+ * desktop, it claims to be 1 msec, but it will depend on the system HZ
+ * setting. Storing monotime_coarse_t uses 16 bytes.
+ *
+ * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.]
+ *
+ * Q: What backends is regular monotonic time using?
+ *
+ * A: In general, regular monotime uses something that requires a system call.
+ * On platforms where system calls are cheap, you win!  Otherwise, you lose.
+ *
+ * On Windows, monotonic time uses QuereyPerformanceCounter.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On OSX/Apple, monotonic time uses mach_absolute_time.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On unixy systems, monotonic time uses CLOCK_MONOTONIC.  Storing
+ * monotime_t costs 16 bytes.
+ *
+ * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec
+ *    count.
+ *
+ * A: Windows, coarse: Cheap, since it's all multiplication.
+ *
+ * Windows, precise: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Apple, all: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Unixy, all: Fairly cheap, since the only division required is dividing
+ * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value.
+ *
+ * All, "timestamp units": Cheap everywhere: it never divides.
+ *
+ * Q: This is only somewhat related, but how much precision could I hope for
+ *    from a libevent time?
+ *
+ * A: Actually, it's _very_ related if you're timing in order to have a
+ * timeout happen.
+ *
+ * On Windows, it uses select: you could in theory have a microsecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On Linux, you have epoll: It has a millisecond resolution.  Some recent
+ * Libevents can also use timerfd for higher resolution if
+ * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag.
+ */
+
 #ifndef TOR_COMPAT_TIME_H
 #define TOR_COMPAT_TIME_H
 
@@ -86,26 +200,36 @@ void monotime_init(void);
 void monotime_get(monotime_t *out);
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
  */
 int64_t monotime_diff_nsec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of microseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of milliseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_msec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of nanoseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
  */
 uint64_t monotime_absolute_nsec(void);
 /**
  * Return the number of microseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
-uint64_t monotime_absolute_usec(void);
+MOCK_DECL(uint64_t, monotime_absolute_usec,(void));
 /**
  * Return the number of milliseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_absolute_msec(void);
 
@@ -129,10 +253,13 @@ void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec);
  * Set <b>out</b> to the current coarse time.
  */
 void monotime_coarse_get(monotime_coarse_t *out);
+/**
+ * Like monotime_absolute_*(), but faster on some platforms.
+ */
 uint64_t monotime_coarse_absolute_nsec(void);
 uint64_t monotime_coarse_absolute_usec(void);
 uint64_t monotime_coarse_absolute_msec(void);
-#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define monotime_coarse_get monotime_get
 #define monotime_coarse_absolute_nsec monotime_absolute_nsec
 #define monotime_coarse_absolute_usec monotime_absolute_usec
@@ -152,23 +279,32 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t);
 /**
  * Convert a difference, expressed in the units of monotime_coarse_to_stamp,
  * into an approximate number of milliseconds.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units);
 uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec);
 uint32_t monotime_coarse_get_stamp(void);
 
 #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)
+/**
+ * Like monotime_diff_*(), but faster on some platforms.
+ */
 int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
+/**
+ * Like monotime_*(), but faster on some platforms.
+ */
 void monotime_coarse_zero(monotime_coarse_t *out);
 int monotime_coarse_is_zero(const monotime_coarse_t *val);
 void monotime_coarse_add_msec(monotime_coarse_t *out,
                               const monotime_coarse_t *val, uint32_t msec);
-#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 #define monotime_coarse_diff_nsec monotime_diff_nsec
 #define monotime_coarse_diff_usec monotime_diff_usec
 #define monotime_coarse_diff_msec monotime_diff_msec
@@ -182,6 +318,9 @@ void monotime_coarse_add_msec(monotime_coarse_t *out,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
                                      const monotime_coarse_t *end);
@@ -191,6 +330,9 @@ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 static inline int32_t
 monotime_coarse_diff_msec32(const monotime_coarse_t *start,
@@ -202,7 +344,7 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start,
 #else
 #define USING_32BIT_MSEC_HACK
   return monotime_coarse_diff_msec32_(start, end);
-#endif
+#endif /* SIZEOF_VOID_P == 8 */
 }
 
 #ifdef TOR_UNIT_TESTS
diff --git a/src/lib/time/include.am b/src/lib/time/include.am
index a3f93a3744..dcb199b142 100644
--- a/src/lib/time/include.am
+++ b/src/lib/time/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-time-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_time_a_SOURCES =	\
 		src/lib/time/compat_time.c	\
+		src/lib/time/time_sys.c		\
 		src/lib/time/tvdiff.c
 
 src_lib_libtor_time_testing_a_SOURCES = \
@@ -14,6 +16,8 @@ src_lib_libtor_time_testing_a_SOURCES = \
 src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/time/compat_time.h	\
+		src/lib/time/time_sys.h		\
 		src/lib/time/tvdiff.h
diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c
new file mode 100644
index 0000000000..8b9aa2856c
--- /dev/null
+++ b/src/lib/time/time_sys.c
@@ -0,0 +1,28 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.c
+ * \brief Subsystem object for monotime setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/time/time_sys.h"
+#include "lib/time/compat_time.h"
+
+static int
+subsys_time_initialize(void)
+{
+  monotime_init();
+  return 0;
+}
+
+const subsys_fns_t sys_time = {
+  .name = "time",
+  /* Monotonic time depends on logging, and a lot of other modules depend on
+   * monotonic time. */
+  .level = -80,
+  .supported = true,
+  .initialize = subsys_time_initialize,
+};
diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h
new file mode 100644
index 0000000000..6a860ffd08
--- /dev/null
+++ b/src/lib/time/time_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.h
+ * \brief Declare subsystem object for the time module.
+ **/
+
+#ifndef TOR_TIME_SYS_H
+#define TOR_TIME_SYS_H
+
+extern const struct subsys_fns_t sys_time;
+
+#endif /* !defined(TOR_TIME_SYS_H) */
diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c
index a87d0d96dc..d7c245f57a 100644
--- a/src/lib/time/tvdiff.c
+++ b/src/lib/time/tvdiff.c
@@ -11,6 +11,7 @@
 #include "lib/time/tvdiff.h"
 
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/time.h"
 #include "lib/log/log.h"
 
 #ifdef _WIN32
@@ -20,8 +21,6 @@
 #include <sys/time.h>
 #endif
 
-#define TOR_USEC_PER_SEC 1000000
-
 /** Return the difference between start->tv_sec and end->tv_sec.
  * Returns INT64_MAX on overflow and underflow.
  */
diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h
index 724af1528a..657cb99553 100644
--- a/src/lib/time/tvdiff.h
+++ b/src/lib/time/tvdiff.h
@@ -20,4 +20,4 @@ int64_t tv_to_msec(const struct timeval *tv);
 
 time_t time_diff(const time_t from, const time_t to);
 
-#endif
+#endif /* !defined(TOR_TVDIFF_H) */
diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include
index 2840e590b8..c550bde024 100644
--- a/src/lib/tls/.may_include
+++ b/src/lib/tls/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/crypt_ops/*.h
@@ -11,7 +12,7 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/tls/*.h
-
-ciphers.inc
+lib/tls/*.inc
diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c
index b570216df0..ed0f821ce8 100644
--- a/src/lib/tls/buffers_tls.c
+++ b/src/lib/tls/buffers_tls.c
@@ -12,7 +12,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am
index a664b29fb2..7e05ef4f8c 100644
--- a/src/lib/tls/include.am
+++ b/src/lib/tls/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-tls-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_tls_a_SOURCES =			\
 	src/lib/tls/buffers_tls.c		\
 	src/lib/tls/tortls.c			\
@@ -29,6 +30,7 @@ src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_tls_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/tls/ciphers.inc			\
 	src/lib/tls/buffers_tls.h		\
@@ -36,5 +38,6 @@ noinst_HEADERS +=				\
 	src/lib/tls/tortls.h			\
 	src/lib/tls/tortls_internal.h		\
 	src/lib/tls/tortls_st.h			\
+	src/lib/tls/tortls_sys.h		\
 	src/lib/tls/x509.h			\
 	src/lib/tls/x509_internal.h
diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h
index 8b31603923..47f220c4c1 100644
--- a/src/lib/tls/nss_countbytes.h
+++ b/src/lib/tls/nss_countbytes.h
@@ -22,4 +22,4 @@ int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd,
                                    uint64_t *n_read_out,
                                    uint64_t *n_written_out);
 
-#endif
+#endif /* !defined(TOR_NSS_COUNTBYTES_H) */
diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c
index 4ca7c7d9d3..1aff40c437 100644
--- a/src/lib/tls/tortls.c
+++ b/src/lib/tls/tortls.c
@@ -7,6 +7,7 @@
 #define TOR_X509_PRIVATE
 #include "lib/tls/x509.h"
 #include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls_sys.h"
 #include "lib/tls/tortls.h"
 #include "lib/tls/tortls_st.h"
 #include "lib/tls/tortls_internal.h"
@@ -15,6 +16,7 @@
 #include "lib/crypt_ops/crypto_rsa.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/net/socket.h"
+#include "lib/subsys/subsys.h"
 
 #ifdef _WIN32
   #include <winsock2.h>
@@ -440,3 +442,15 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity)
 
   return rv;
 }
+
+static void
+subsys_tortls_shutdown(void)
+{
+  tor_tls_free_all();
+}
+
+const subsys_fns_t sys_tortls = {
+  .name = "tortls",
+  .level = -50,
+  .shutdown = subsys_tortls_shutdown
+};
diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h
index 8efc7a1c98..799bd6aaeb 100644
--- a/src/lib/tls/tortls.h
+++ b/src/lib/tls/tortls.h
@@ -25,12 +25,12 @@ struct ssl_ctx_st;
 struct ssl_session_st;
 typedef struct ssl_ctx_st tor_tls_context_impl_t;
 typedef struct ssl_st tor_tls_impl_t;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 struct PRFileDesc;
 typedef struct PRFileDesc tor_tls_context_impl_t;
 typedef struct PRFileDesc tor_tls_impl_t;
-#endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+#endif /* defined(TORTLS_PRIVATE) */
 
 struct tor_x509_cert_t;
 
@@ -144,9 +144,9 @@ void check_no_tls_errors_(const char *fname, int line);
 
 void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
                            int severity, int domain, const char *doing);
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 #define check_no_tls_errors() STMT_NIL
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 int tor_tls_get_my_certs(int server,
                          const struct tor_x509_cert_t **link_cert_out,
diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h
index 071c506561..866483a94c 100644
--- a/src/lib/tls/tortls_internal.h
+++ b/src/lib/tls/tortls_internal.h
@@ -61,8 +61,8 @@ STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret,
                             void *arg);
 STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
                              uint16_t cipher);
-#endif
-#endif
+#endif /* defined(TORTLS_OPENSSL_PRIVATE) */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef TOR_UNIT_TESTS
 extern int tor_tls_object_ex_data_index;
@@ -73,4 +73,4 @@ extern uint64_t total_bytes_written_over_tls;
 extern uint64_t total_bytes_written_by_tls;
 #endif /* defined(TOR_UNIT_TESTS) */
 
-#endif /* defined(TORTLS_INTERNAL_H) */
+#endif /* !defined(TORTLS_INTERNAL_H) */
diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c
index 80b0df301f..5bafcf676d 100644
--- a/src/lib/tls/tortls_openssl.c
+++ b/src/lib/tls/tortls_openssl.c
@@ -25,7 +25,7 @@
    * <winsock.h> and mess things up, in at least some openssl versions. */
   #include <winsock2.h>
   #include <ws2tcpip.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rand.h"
@@ -318,7 +318,7 @@ tor_tls_init(void)
 #else
     SSL_library_init();
     SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))
@@ -383,7 +383,7 @@ static const char SERVER_CIPHER_LIST[] =
    * conclude that it has no valid ciphers if it's running with TLS1.3.
    */
   TLS1_3_TXT_AES_128_GCM_SHA256 ":"
-#endif
+#endif /* defined(TLS1_3_TXT_AES_128_GCM_SHA256) */
   TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":"
   TLS1_TXT_DHE_RSA_WITH_AES_128_SHA;
 
@@ -464,7 +464,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 /** List of ciphers that clients should advertise, omitting items that
  * our OpenSSL doesn't know about. */
 static const char CLIENT_CIPHER_LIST[] =
-#include "ciphers.inc"
+#include "lib/tls/ciphers.inc"
   /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version
    * of any cipher we say. */
   "!SSLv2"
@@ -657,7 +657,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     if (r < 0)
       goto error;
   }
-#else
+#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */
   if (! is_client) {
     int nid;
     EC_KEY *ec_key;
@@ -673,7 +673,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
       SSL_CTX_set_tmp_ecdh(result->ctx, ec_key);
     EC_KEY_free(ec_key);
   }
-#endif
+#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */
   SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER,
                      always_accept_verify_cb);
   /* let us realloc bufs that we're writing from */
@@ -764,7 +764,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher);
     return c != NULL;
   }
-#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */
+#else /* !defined(HAVE_SSL_CIPHER_FIND) */
 
 # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR)
   if (m && m->get_cipher_by_char) {
@@ -1062,7 +1062,7 @@ tor_tls_new(tor_socket_t sock, int isServer)
     /* We can't actually use TLS 1.3 until this bug is fixed. */
     SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION);
   }
-#endif
+#endif /* defined(SSL_CTRL_SET_MAX_PROTO_VERSION) */
 
   if (!SSL_set_cipher_list(result->ssl,
                      isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) {
@@ -1728,7 +1728,7 @@ tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
     else
       return -1;
   }
-#endif
+#endif /* defined(TLS1_3_VERSION) */
 
   return (r == 1) ? 0 : -1;
 }
diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h
index 3f7ea8ac6a..73f6e6ecca 100644
--- a/src/lib/tls/tortls_st.h
+++ b/src/lib/tls/tortls_st.h
@@ -64,7 +64,7 @@ struct tor_tls_t {
   void (*negotiated_callback)(tor_tls_t *tls, void *arg);
   /** Argument to pass to negotiated_callback. */
   void *callback_arg;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
   /** Last values retried from tor_get_prfiledesc_byte_counts(). */
   uint64_t last_write_count;
@@ -72,4 +72,4 @@ struct tor_tls_t {
 #endif
 };
 
-#endif
+#endif /* !defined(TOR_TORTLS_ST_H) */
diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h
new file mode 100644
index 0000000000..4b04f85f0c
--- /dev/null
+++ b/src/lib/tls/tortls_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tortls_sys.h
+ * \brief Declare subsystem object for the tortls module
+ **/
+
+#ifndef TOR_TORTLS_SYS_H
+#define TOR_TORTLS_SYS_H
+
+extern const struct subsys_fns_t sys_tortls;
+
+#endif /* !defined(TOR_TORTLS_SYS_H) */
diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h
index 5e6660de5c..0390a5464d 100644
--- a/src/lib/tls/x509.h
+++ b/src/lib/tls/x509.h
@@ -35,7 +35,7 @@ struct tor_x509_cert_t {
   common_digests_t cert_digests;
   common_digests_t pkey_digests;
 };
-#endif
+#endif /* defined(TOR_X509_PRIVATE) */
 
 void tor_tls_pick_certificate_lifetime(time_t now,
                                        unsigned cert_lifetime,
@@ -47,7 +47,7 @@ tor_x509_cert_t *tor_x509_cert_replace_expiration(
                                                const tor_x509_cert_t *inp,
                                                time_t new_expiration_time,
                                                crypto_pk_t *signing_key);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 
@@ -72,4 +72,4 @@ int tor_tls_cert_is_valid(int severity,
                           time_t now,
                           int check_rsa_1024);
 
-#endif
+#endif /* !defined(TOR_X509_H) */
diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h
index bf2bec9689..f858baae98 100644
--- a/src/lib/tls/x509_internal.h
+++ b/src/lib/tls/x509_internal.h
@@ -50,4 +50,4 @@ int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert);
 #define tor_x509_cert_set_cached_der_encoding(cert) (0)
 #endif
 
-#endif
+#endif /* !defined(TOR_X509_INTERNAL_H) */
diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c
index fb4af54c52..e04afaf07b 100644
--- a/src/lib/tls/x509_nss.c
+++ b/src/lib/tls/x509_nss.c
@@ -120,13 +120,13 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                      der.data, der.len,
                                      (SECKEYPrivateKey *)signing_key,//const
                                      &cert->signature);
-#else
+#else /* !(0) */
   s = SEC_DerSignData(cert->arena,
                       &signed_der,
                       der.data, der.len,
                       (SECKEYPrivateKey *)signing_key,//const
                       SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION);
-#endif
+#endif /* 0 */
 
   if (s != SECSuccess)
     goto err;
@@ -145,7 +145,7 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                &result_cert->signatureWrap, issuer_pk, NULL);
     tor_assert(cert_ok == SECSuccess);
   }
-#endif
+#endif /* 1 */
 
  err:
   if (subject_spki)
@@ -455,4 +455,4 @@ tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
 
   return newcert ? tor_x509_cert_new(newcert) : NULL;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c
index a344279c22..7724288279 100644
--- a/src/lib/tls/x509_openssl.c
+++ b/src/lib/tls/x509_openssl.c
@@ -59,12 +59,12 @@ ENABLE_GCC_WARNING(redundant-decls)
 #define X509_get_notAfter(cert) \
     X509_getm_notAfter(cert)
 #endif
-#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
+#else /* !defined(OPENSSL_1_1_API) */
 #define X509_get_notBefore_const(cert) \
   ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
 #define X509_get_notAfter_const(cert) \
   ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 /** Return a newly allocated X509 name with commonName <b>cname</b>. */
 static X509_NAME *
diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h
index e35616cf50..92bb95c883 100644
--- a/src/lib/trace/debug.h
+++ b/src/lib/trace/debug.h
@@ -27,4 +27,4 @@
                         "\"" XSTR(subsystem) "\" hit. " \
                         "(line "XSTR(__LINE__) ")")
 
-#endif /* TOR_TRACE_LOG_DEBUG_H */
+#endif /* !defined(TOR_TRACE_LOG_DEBUG_H) */
diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h
index 1e1e7b9d16..9de86d63f2 100644
--- a/src/lib/trace/events.h
+++ b/src/lib/trace/events.h
@@ -34,12 +34,12 @@
 #include "lib/trace/debug.h"
 #endif
 
-#else /* TOR_EVENT_TRACING_ENABLED */
+#else /* !defined(TOR_EVENT_TRACING_ENABLED) */
 
 /* Reaching this point, we NOP every event declaration because event tracing
  * is not been enabled at compile time. */
 #define tor_trace(subsystem, name, args...)
 
-#endif /* TOR_EVENT_TRACING_ENABLED */
+#endif /* defined(TOR_EVENT_TRACING_ENABLED) */
 
-#endif /* TOR_TRACE_EVENTS_H */
+#endif /* !defined(TOR_TRACE_EVENTS_H) */
diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am
index 6f10c98744..98098c87f4 100644
--- a/src/lib/trace/include.am
+++ b/src/lib/trace/include.am
@@ -2,6 +2,7 @@
 noinst_LIBRARIES += \
 	src/lib/libtor-trace.a
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 TRACEHEADERS = \
 	src/lib/trace/trace.h \
 	src/lib/trace/events.h
@@ -11,7 +12,7 @@ TRACEHEADERS += \
 	src/lib/trace/debug.h
 endif
 
-# Library source files.
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_trace_a_SOURCES = \
 	src/lib/trace/trace.c
 
diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h
index 606d435568..5001b28a1d 100644
--- a/src/lib/trace/trace.h
+++ b/src/lib/trace/trace.h
@@ -11,4 +11,4 @@
 
 void tor_trace_init(void);
 
-#endif // TOR_TRACE_TRACE_H
+#endif /* !defined(TOR_TRACE_TRACE_H) */
diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include
new file mode 100644
index 0000000000..d159ceb41f
--- /dev/null
+++ b/src/lib/version/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+micro-revision.i
+lib/version/*.h
\ No newline at end of file
diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c
index 7d27549cad..900a1e12a0 100644
--- a/src/lib/log/git_revision.c
+++ b/src/lib/version/git_revision.c
@@ -4,7 +4,7 @@
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "lib/log/git_revision.h"
+#include "lib/version/git_revision.h"
 
 /** String describing which Tor Git repository version the source was
  * built from.  This string is generated by a bit of shell kludging in
diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h
index 79e3c6684b..79e3c6684b 100644
--- a/src/lib/log/git_revision.h
+++ b/src/lib/version/git_revision.h
diff --git a/src/lib/version/include.am b/src/lib/version/include.am
new file mode 100644
index 0000000000..0ae31be1b2
--- /dev/null
+++ b/src/lib/version/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-version.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-version-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_version_a_SOURCES =			\
+	src/lib/version/git_revision.c			\
+	src/lib/version/version.c
+
+src_lib_libtor_version_testing_a_SOURCES = \
+	$(src_lib_libtor_version_a_SOURCES)
+src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# Declare that these object files depend on micro-revision.i.  Without this
+# rule, we could try to build them before micro-revision.i was created.
+src/lib/version/git_revision.$(OBJEXT) \
+  src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \
+	micro-revision.i
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/version/git_revision.h			\
+	src/lib/version/torversion.h
diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h
new file mode 100644
index 0000000000..7b0fb66ec0
--- /dev/null
+++ b/src/lib/version/torversion.h
@@ -0,0 +1,12 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_VERSION_H
+#define TOR_VERSION_H
+
+const char *get_version(void);
+const char *get_short_version(void);
+
+#endif /* !defined(TOR_VERSION_H) */
diff --git a/src/lib/version/version.c b/src/lib/version/version.c
new file mode 100644
index 0000000000..434e6fb424
--- /dev/null
+++ b/src/lib/version/version.c
@@ -0,0 +1,50 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/version/torversion.h"
+#include "lib/version/git_revision.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/** A shorter version of this Tor process's version, for export in our router
+ *  descriptor.  (Does not include the git version, if any.) */
+static const char the_short_tor_version[] =
+  VERSION
+#ifdef TOR_BUILD_TAG
+  " ("TOR_BUILD_TAG")"
+#endif
+  "";
+
+#define MAX_VERSION_LEN 128
+
+/** The version of this Tor process, possibly including git version */
+static char the_tor_version[MAX_VERSION_LEN] = "";
+
+/** Return the current Tor version. */
+const char *
+get_version(void)
+{
+  if (the_tor_version[0] == 0) {
+    if (strlen(tor_git_revision)) {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s (git-%s)", the_short_tor_version, tor_git_revision);
+    } else {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s", the_short_tor_version);
+    }
+    the_tor_version[sizeof(the_tor_version)-1] = 0;
+  }
+
+  return the_tor_version;
+}
+
+/** Return the current Tor version, without any git tag. */
+const char *
+get_short_version(void)
+{
+  return the_short_tor_version;
+}
diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include
index dc010da063..ce7a26472b 100644
--- a/src/lib/wallclock/.may_include
+++ b/src/lib/wallclock/.may_include
@@ -3,4 +3,5 @@ lib/cc/*.h
 lib/err/*.h
 lib/wallclock/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c
index ee498702d5..77eeddaf56 100644
--- a/src/lib/wallclock/approx_time.c
+++ b/src/lib/wallclock/approx_time.c
@@ -9,7 +9,9 @@
  **/
 
 #include "orconfig.h"
+#include "lib/subsys/subsys.h"
 #include "lib/wallclock/approx_time.h"
+#include "lib/wallclock/wallclock_sys.h"
 
 /* =====
  * Cached time
@@ -41,3 +43,19 @@ update_approx_time(time_t now)
   cached_approx_time = now;
 }
 #endif /* !defined(TIME_IS_FAST) */
+
+static int
+subsys_wallclock_initialize(void)
+{
+  update_approx_time(time(NULL));
+  return 0;
+}
+
+const subsys_fns_t sys_wallclock = {
+  .name = "wallclock",
+  .supported = true,
+  /* Approximate time is a diagnostic feature, we want it to init right after
+   * low-level error handling. */
+  .level = -98,
+  .initialize = subsys_wallclock_initialize,
+};
diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h
index e6b53f2c27..e7da160122 100644
--- a/src/lib/wallclock/approx_time.h
+++ b/src/lib/wallclock/approx_time.h
@@ -22,4 +22,4 @@ time_t approx_time(void);
 void update_approx_time(time_t now);
 #endif /* defined(TIME_IS_FAST) */
 
-#endif
+#endif /* !defined(TOR_APPROX_TIME_H) */
diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am
index 1961639bd7..2b50d6ccbb 100644
--- a/src/lib/wallclock/include.am
+++ b/src/lib/wallclock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_wallclock_a_SOURCES =			\
 	src/lib/wallclock/approx_time.c			\
 	src/lib/wallclock/time_to_tm.c			\
@@ -15,8 +16,10 @@ src_lib_libtor_wallclock_testing_a_SOURCES = \
 src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/wallclock/approx_time.h			\
 	src/lib/wallclock/timeval.h			\
 	src/lib/wallclock/time_to_tm.h			\
-	src/lib/wallclock/tor_gettimeofday.h
+	src/lib/wallclock/tor_gettimeofday.h		\
+	src/lib/wallclock/wallclock_sys.h
diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h
index abe78c0efe..da27fcaba1 100644
--- a/src/lib/wallclock/time_to_tm.h
+++ b/src/lib/wallclock/time_to_tm.h
@@ -19,4 +19,4 @@ struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result,
 struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result,
                             char **err_out);
 
-#endif
+#endif /* !defined(TOR_WALLCLOCK_TIME_TO_TM_H) */
diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h
index 4967e939bf..e632d04a04 100644
--- a/src/lib/wallclock/timeval.h
+++ b/src/lib/wallclock/timeval.h
@@ -20,6 +20,27 @@
 #include <sys/time.h>
 #endif
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we use a slower definition of these macros without
+ * branches, to make coverage consistent. */
+#undef timeradd
+#undef timersub
+#define timeradd(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec;    \
+    (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;      \
+    (tvout)->tv_usec %= 1000000;                        \
+  } while (0)
+#define timersub(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec - 1;            \
+    (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec + 1000000;   \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;                  \
+    (tvout)->tv_usec %= 1000000;                                    \
+  } while (0)
+#endif /* defined(TOR_COVERAGE) */
+
 #ifndef timeradd
 /** Replacement for timeradd on platforms that do not have it: sets tvout to
  * the sum of tv1 and tv2. */
@@ -62,4 +83,4 @@
    ((tv1)->tv_sec op (tv2)->tv_sec))
 #endif /* !defined(timercmp) */
 
-#endif
+#endif /* !defined(TOR_TIMEVAL_H) */
diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h
index c7fff9747a..6fec2fc893 100644
--- a/src/lib/wallclock/tor_gettimeofday.h
+++ b/src/lib/wallclock/tor_gettimeofday.h
@@ -17,4 +17,4 @@ struct timeval;
 
 MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
 
-#endif
+#endif /* !defined(TOR_GETTIMEOFDAY_H) */
diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h
new file mode 100644
index 0000000000..a30912b8fb
--- /dev/null
+++ b/src/lib/wallclock/wallclock_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file wallclock_sys.h
+ * \brief Declare subsystem object for the wallclock module.
+ **/
+
+#ifndef TOR_WALLCLOCK_SYS_H
+#define TOR_WALLCLOCK_SYS_H
+
+extern const struct subsys_fns_t sys_wallclock;
+
+#endif /* !defined(TOR_WALLCLOCK_SYS_H) */
diff --git a/src/rust/build.rs b/src/rust/build.rs
index 123d5c0682..5626b35f75 100644
--- a/src/rust/build.rs
+++ b/src/rust/build.rs
@@ -149,8 +149,9 @@ pub fn main() {
             cfg.component("tor-sandbox-testing");
             cfg.component("tor-encoding-testing");
             cfg.component("tor-fs-testing");
-            cfg.component("tor-time-testing");
             cfg.component("tor-net-testing");
+            cfg.component("tor-buf-testing");
+            cfg.component("tor-time-testing");
             cfg.component("tor-thread-testing");
             cfg.component("tor-memarea-testing");
             cfg.component("tor-log-testing");
@@ -162,6 +163,7 @@ pub fn main() {
             cfg.component("tor-malloc");
             cfg.component("tor-wallclock");
             cfg.component("tor-err-testing");
+            cfg.component("tor-version-testing");
             cfg.component("tor-intmath-testing");
             cfg.component("tor-ctime-testing");
             cfg.component("curve25519_donna");
diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs
index 6ee63adb10..14170d0353 100644
--- a/src/rust/protover/ffi.rs
+++ b/src/rust/protover/ffi.rs
@@ -30,6 +30,8 @@ fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> {
         7 => Ok(Protocol::Desc),
         8 => Ok(Protocol::Microdesc),
         9 => Ok(Protocol::Cons),
+        10 => Ok(Protocol::Padding),
+        11 => Ok(Protocol::FlowCtrl),
         _ => Err(ProtoverError::UnknownProtocol),
     }
 }
diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs
index 06fdf56c69..0ca960bd69 100644
--- a/src/rust/protover/protover.rs
+++ b/src/rust/protover/protover.rs
@@ -46,6 +46,8 @@ pub enum Protocol {
     LinkAuth,
     Microdesc,
     Relay,
+    Padding,
+    FlowCtrl,
 }
 
 impl fmt::Display for Protocol {
@@ -73,6 +75,8 @@ impl FromStr for Protocol {
             "LinkAuth" => Ok(Protocol::LinkAuth),
             "Microdesc" => Ok(Protocol::Microdesc),
             "Relay" => Ok(Protocol::Relay),
+            "Padding" => Ok(Protocol::Padding),
+            "FlowCtrl" => Ok(Protocol::FlowCtrl),
             _ => Err(ProtoverError::UnknownProtocol),
         }
     }
@@ -163,7 +167,9 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
              Link=1-5 \
              LinkAuth=3 \
              Microdesc=1-2 \
-             Relay=1-2"
+             Relay=1-2 \
+             Padding=2 \
+             FlowCtrl=1"
         )
     } else {
         cstr!(
@@ -176,7 +182,9 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
              Link=1-5 \
              LinkAuth=1,3 \
              Microdesc=1-2 \
-             Relay=1-2"
+             Relay=1-2 \
+             Padding=2 \
+             FlowCtrl=1"
         )
     }
 }
diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs
index 98fccba5a9..bbaf97129c 100644
--- a/src/rust/tor_log/tor_log.rs
+++ b/src/rust/tor_log/tor_log.rs
@@ -99,14 +99,14 @@ pub mod log {
     /// Domain log types. These mirror definitions in src/lib/log/log.h
     /// C_RUST_COUPLED: src/lib/log/log.c, log severity types
     extern "C" {
-        static LD_NET_: u32;
-        static LD_GENERAL_: u32;
+        static LD_NET_: u64;
+        static LD_GENERAL_: u64;
     }
 
     /// Translate Rust defintions of log domain levels to C. This exposes a 1:1
     /// mapping between types.
     #[inline]
-    pub unsafe fn translate_domain(domain: LogDomain) -> u32 {
+    pub unsafe fn translate_domain(domain: LogDomain) -> u64 {
         match domain {
             LogDomain::Net => LD_NET_,
             LogDomain::General => LD_GENERAL_,
@@ -128,7 +128,7 @@ pub mod log {
     extern "C" {
         pub fn tor_log_string(
             severity: c_int,
-            domain: u32,
+            domain: u64,
             function: *const c_char,
             string: *const c_char,
         );
diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs
index 2a19458f46..ede42c6ea8 100644
--- a/src/rust/tor_util/strings.rs
+++ b/src/rust/tor_util/strings.rs
@@ -105,11 +105,7 @@ macro_rules! cstr {
     ($($bytes:expr),*) => (
         ::std::ffi::CStr::from_bytes_with_nul(
             concat!($($bytes),*, "\0").as_bytes()
-        ).unwrap_or(
-            unsafe{
-                ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0")
-            }
-        )
+        ).unwrap_or_default()
     )
 }
 
diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake
index cfbe281b94..ca6a84cf8a 100644
--- a/src/test/Makefile.nmake
+++ b/src/test/Makefile.nmake
@@ -1,4 +1,4 @@
-all: test.exe test-child.exe bench.exe
+all: test.exe bench.exe
 
 CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \
     /I ..\ext
@@ -19,6 +19,7 @@ TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \
 	test_cell_formats.obj test_relay.obj test_replay.obj \
 	test_channelpadding.obj \
 	test_circuitstats.obj \
+	test_circuitpadding.obj \
 	test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj
 
 tinytest.obj: ..\ext\tinytest.c
@@ -30,8 +31,5 @@ test.exe: $(TEST_OBJECTS)
 bench.exe: bench.obj
 	$(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@
 
-test-child.exe: test-child.obj
-	$(CC) $(CFLAGS) test-child.obj /Fe$@
-
 clean:
-	del *.obj *.lib test.exe bench.exe test-child.exe
+	del *.obj *.lib test.exe bench.exe
diff --git a/src/test/bench.c b/src/test/bench.c
index 06c616c3b0..cf732df593 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -14,16 +14,19 @@
 #include "core/crypto/onion_tap.h"
 #include "core/crypto/relay_crypto.h"
 
+#include "lib/intmath/weakrng.h"
+
 #ifdef ENABLE_OPENSSL
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #include <openssl/ec.h>
 #include <openssl/ecdh.h>
 #include <openssl/obj_mac.h>
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "core/or/circuitlist.h"
 #include "app/config/config.h"
+#include "app/main/subsysmgr.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "core/crypto/onion_ntor.h"
@@ -38,6 +41,9 @@
 #include "lib/crypt_ops/digestset.h"
 #include "lib/crypt_ops/crypto_init.h"
 
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/nodelist/microdesc.h"
+
 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
 static uint64_t nanostart;
 static inline uint64_t
@@ -332,6 +338,65 @@ bench_ed25519(void)
 }
 
 static void
+bench_rand_len(int len)
+{
+  const int N = 100000;
+  int i;
+  char *buf = tor_malloc(len);
+  uint64_t start,end;
+
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_rand(buf, len);
+  }
+  end = perftime();
+  printf("crypto_rand(%d): %f nsec.\n", len, NANOCOUNT(start,end,N));
+
+  crypto_fast_rng_t *fr = crypto_fast_rng_new();
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_fast_rng_getbytes(fr,(uint8_t*)buf,len);
+  }
+  end = perftime();
+  printf("crypto_fast_rng_getbytes(%d): %f nsec.\n", len,
+         NANOCOUNT(start,end,N));
+  crypto_fast_rng_free(fr);
+
+  if (len <= 32) {
+    start = perftime();
+    for (i = 0; i < N; ++i) {
+      crypto_strongest_rand((uint8_t*)buf, len);
+    }
+    end = perftime();
+    printf("crypto_strongest_rand(%d): %f nsec.\n", len,
+           NANOCOUNT(start,end,N));
+  }
+
+  if (len == 4) {
+    tor_weak_rng_t weak;
+    tor_init_weak_random(&weak, 1337);
+
+    start = perftime();
+    uint32_t t=0;
+    for (i = 0; i < N; ++i) {
+      t += tor_weak_random(&weak);
+    }
+    end = perftime();
+    printf("weak_rand(4): %f nsec.\n", NANOCOUNT(start,end,N));
+  }
+
+  tor_free(buf);
+}
+
+static void
+bench_rand(void)
+{
+  bench_rand_len(4);
+  bench_rand_len(16);
+  bench_rand_len(128);
+}
+
+static void
 bench_cell_aes(void)
 {
   uint64_t start, end;
@@ -636,7 +701,42 @@ bench_ecdh_p224(void)
 {
   bench_ecdh_impl(NID_secp224r1, "P-224");
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+
+static void
+bench_md_parse(void)
+{
+  uint64_t start, end;
+  const int N = 100000;
+  // selected arbitrarily
+  const char md_text[] =
+    "@last-listed 2018-12-14 18:14:14\n"
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n"
+    "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n"
+    "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n"
+    "family $00E9A86E7733240E60D8435A7BBD634A23894098 "
+    "$329BD7545DEEEBBDC8C4285F243916F248972102 "
+    "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D "
+    "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n"
+    "p accept 53,80,443,5222-5223,25565\n"
+    "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n";
+
+  reset_perftime();
+  start = perftime();
+  for (int i = 0; i < N; ++i) {
+    smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1,
+                                                  SAVED_IN_CACHE, NULL);
+    SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md));
+    smartlist_free(s);
+  }
+
+  end = perftime();
+  printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N));
+}
 
 typedef void (*bench_fn)(void);
 
@@ -656,6 +756,7 @@ static struct benchmark_t benchmarks[] = {
   ENT(onion_TAP),
   ENT(onion_ntor),
   ENT(ed25519),
+  ENT(rand),
 
   ENT(cell_aes),
   ENT(cell_ops),
@@ -665,6 +766,8 @@ static struct benchmark_t benchmarks[] = {
   ENT(ecdh_p256),
   ENT(ecdh_p224),
 #endif
+
+  ENT(md_parse),
   {NULL,NULL,0}
 };
 
@@ -690,9 +793,10 @@ main(int argc, const char **argv)
   char *errmsg;
   or_options_t *options;
 
-  tor_threads_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
+
   tor_compress_init();
-  init_logging(1);
 
   if (argc == 4 && !strcmp(argv[1], "diff")) {
     const int N = 200;
@@ -702,11 +806,13 @@ main(int argc, const char **argv)
       perror("X");
       return 1;
     }
+    size_t f1len = strlen(f1);
+    size_t f2len = strlen(f2);
     for (i = 0; i < N; ++i) {
-      char *diff = consensus_diff_generate(f1, f2);
+      char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
       tor_free(diff);
     }
-    char *diff = consensus_diff_generate(f1, f2);
+    char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
     printf("%s", diff);
     tor_free(f1);
     tor_free(f2);
@@ -737,7 +843,6 @@ main(int argc, const char **argv)
 
   init_protocol_warning_severity_level();
   options = options_new();
-  init_logging(1);
   options->command = CMD_RUN_UNITTESTS;
   options->DataDirectory = tor_strdup("");
   options->KeyDirectory = tor_strdup("");
diff --git a/src/test/conf_examples/badnick_1/error b/src/test/conf_examples/badnick_1/error
new file mode 100644
index 0000000000..3e92ddc832
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/error
@@ -0,0 +1 @@
+nicknames must be between 1 and 19 characters inclusive
diff --git a/src/test/conf_examples/badnick_1/torrc b/src/test/conf_examples/badnick_1/torrc
new file mode 100644
index 0000000000..41ee4894f1
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/torrc
@@ -0,0 +1,2 @@
+# This nickname is too long; we won't accept it.
+Nickname TooManyCharactersInThisNickname
diff --git a/src/test/conf_examples/badnick_2/error b/src/test/conf_examples/badnick_2/error
new file mode 100644
index 0000000000..ceac99f012
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/error
@@ -0,0 +1 @@
+must contain only the characters \[a-zA-Z0-9\]
diff --git a/src/test/conf_examples/badnick_2/torrc b/src/test/conf_examples/badnick_2/torrc
new file mode 100644
index 0000000000..07acc61698
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/torrc
@@ -0,0 +1,2 @@
+# this nickname has spaces in it and won't work.
+Nickname has a space
diff --git a/src/test/conf_examples/contactinfo_notutf8/error b/src/test/conf_examples/contactinfo_notutf8/error
new file mode 100644
index 0000000000..6d165152ce
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/error
@@ -0,0 +1 @@
+ContactInfo config option must be UTF-8
diff --git a/src/test/conf_examples/contactinfo_notutf8/torrc b/src/test/conf_examples/contactinfo_notutf8/torrc
new file mode 100644
index 0000000000..2ee4d093c5
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/torrc
@@ -0,0 +1 @@
+ContactInfo ÄëÄëÄë@example.com
diff --git a/src/test/conf_examples/example_1/expected b/src/test/conf_examples/example_1/expected
new file mode 100644
index 0000000000..9d6688a565
--- /dev/null
+++ b/src/test/conf_examples/example_1/expected
@@ -0,0 +1,2 @@
+ContactInfo tor_tellini@example.com
+SocksPort 80
diff --git a/src/test/conf_examples/example_1/torrc b/src/test/conf_examples/example_1/torrc
new file mode 100644
index 0000000000..bff7fa0aa2
--- /dev/null
+++ b/src/test/conf_examples/example_1/torrc
@@ -0,0 +1,5 @@
+
+# Here is a simple example torrc.
+  SocksPort  80
+
+ContactInfo "tor_tellini@example.com"
diff --git a/src/test/conf_examples/example_2/error b/src/test/conf_examples/example_2/error
new file mode 100644
index 0000000000..ce18b68db4
--- /dev/null
+++ b/src/test/conf_examples/example_2/error
@@ -0,0 +1 @@
+Unknown option 'JumpingJellyjars'
diff --git a/src/test/conf_examples/example_2/torrc b/src/test/conf_examples/example_2/torrc
new file mode 100644
index 0000000000..8ec8133b24
--- /dev/null
+++ b/src/test/conf_examples/example_2/torrc
@@ -0,0 +1 @@
+JumpingJellyjars 1
diff --git a/src/test/conf_examples/example_3/cmdline b/src/test/conf_examples/example_3/cmdline
new file mode 100644
index 0000000000..5b2fadcebb
--- /dev/null
+++ b/src/test/conf_examples/example_3/cmdline
@@ -0,0 +1 @@
+--socksport 99
diff --git a/src/test/conf_examples/example_3/expected b/src/test/conf_examples/example_3/expected
new file mode 100644
index 0000000000..867fb8bcc8
--- /dev/null
+++ b/src/test/conf_examples/example_3/expected
@@ -0,0 +1 @@
+SocksPort 99
diff --git a/src/test/conf_examples/example_3/torrc b/src/test/conf_examples/example_3/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/example_3/torrc
diff --git a/src/test/conf_examples/include_1/expected b/src/test/conf_examples/include_1/expected
new file mode 100644
index 0000000000..4bbf52ce9f
--- /dev/null
+++ b/src/test/conf_examples/include_1/expected
@@ -0,0 +1,3 @@
+ContactInfo includefile@example.com
+Nickname nested
+ORPort 8008
diff --git a/src/test/conf_examples/include_1/included.inc b/src/test/conf_examples/include_1/included.inc
new file mode 100644
index 0000000000..8d1834345d
--- /dev/null
+++ b/src/test/conf_examples/include_1/included.inc
@@ -0,0 +1,4 @@
+
+ContactInfo includefile@example.com
+
+%include "nested.inc"
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/nested.inc b/src/test/conf_examples/include_1/nested.inc
new file mode 100644
index 0000000000..789b044a2b
--- /dev/null
+++ b/src/test/conf_examples/include_1/nested.inc
@@ -0,0 +1,2 @@
+
+Nickname nested
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/torrc b/src/test/conf_examples/include_1/torrc
new file mode 100644
index 0000000000..2ed4074f6e
--- /dev/null
+++ b/src/test/conf_examples/include_1/torrc
@@ -0,0 +1,4 @@
+
+%include "included.inc"
+
+ORPort 8008
diff --git a/src/test/conf_examples/include_bug_31408/expected b/src/test/conf_examples/include_bug_31408/expected
new file mode 100644
index 0000000000..2e822f1a78
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/expected
@@ -0,0 +1,2 @@
+Nickname test31408
+ORPort 31408
diff --git a/src/test/conf_examples/include_bug_31408/included/01_nickname.inc b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
new file mode 100644
index 0000000000..508dd89a35
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
@@ -0,0 +1 @@
+Nickname test31408
diff --git a/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
new file mode 100644
index 0000000000..140e927f19
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
@@ -0,0 +1,3 @@
+# Bug 31048 is triggered when the last file in a config directory:
+#   * contains no configuration options,
+#   * but is non-empty: that is, it contains comments or whitespace.
diff --git a/src/test/conf_examples/include_bug_31408/torrc b/src/test/conf_examples/include_bug_31408/torrc
new file mode 100644
index 0000000000..a42685e93c
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/torrc
@@ -0,0 +1,2 @@
+%include "included"
+ORPort 31408
diff --git a/src/test/conf_examples/large_1/expected b/src/test/conf_examples/large_1/expected
new file mode 100644
index 0000000000..5866f5823e
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected
@@ -0,0 +1,159 @@
+AccountingMax 10737418240
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2147483647
+BandwidthRate 1610612736
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientAutoIPv6ORPort 1
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirPortFrontPage /dirport/frontpage
+DirPort 99
+DirReqStatistics 0
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 1260
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 300
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPortCookieAuthFile /foobar
+ExtORPort 99
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 691200
+HeartbeatPeriod 2700
+IPv6Exit 1
+KeepalivePeriod 540
+KeyDirectory /keyz
+KISTSchedRunInterval 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 60000
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 3600
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 2629728
+MaxMemInQueues 314572800
+MaxOnionQueueDelay 60000
+MaxUnparseableDescSizeToLog 1048576
+MiddleNodes grommit,truffle,parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 7200
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PerConnBWBurst 10485760
+PerConnBWRate 102400
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 600
+RephistTrackTime 600
+SafeLogging 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10
+SigningKeyLifetime 4838400
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 600
+SSLKeyLifetime 86400
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1000
+TrackHostExits www.example.com
+TrackHostExitsExpire 3600
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/large_1/torrc b/src/test/conf_examples/large_1/torrc
new file mode 100644
index 0000000000..e99acd9fb7
--- /dev/null
+++ b/src/test/conf_examples/large_1/torrc
@@ -0,0 +1,167 @@
+AccountingMax 10 GB
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2 GB
+BandwidthRate 1.5 GB
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitPadding 1
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientAutoIPv6ORPort 1
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirReqStatistics 0
+DirPort 99
+DirPortFrontPage /dirport/frontpage
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 21 minutes
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 5 minutes
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPort 99
+ExtORPortCookieAuthFile /foobar
+ExtraInfoStatistics 1
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchHidServDescriptors 1
+FetchServerDescriptors 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 8 days
+HeartbeatPeriod 45 minutes
+IPv6Exit 1
+KeepalivePeriod 9 minutes
+KeyDirectory /keyz
+KISTSchedRunInterval 1 msec
+LearnCircuitBuildTimeout 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 1 minute
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 1 hour
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 1 month
+MaxMemInQueues 300 MB
+MaxOnionQueueDelay 60 seconds
+MaxUnparseableDescSizeToLog 1 MB
+MiddleNodes grommit, truffle, parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 2 hours
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PaddingStatistics 1
+PerConnBWBurst 10 MB
+PerConnBWRate 100 kb
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 10 minutes
+RephistTrackTime 10 minutes
+SafeLogging 0
+SafeSocks 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10 seconds
+SigningKeyLifetime 8 weeks
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 10 minutes
+SSLKeyLifetime 1 day
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1 second
+TrackHostExits www.example.com
+TrackHostExitsExpire 1 hour
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseEntryGuards 1
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/obsolete_1/expected b/src/test/conf_examples/obsolete_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/expected
diff --git a/src/test/conf_examples/obsolete_1/torrc b/src/test/conf_examples/obsolete_1/torrc
new file mode 100644
index 0000000000..3cd9a6d777
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/torrc
@@ -0,0 +1,68 @@
+# These options are obsolete as of 0.4.2
+AllowDotExit
+AllowInvalidNodes
+AllowSingleHopCircuits
+AllowSingleHopExits
+AlternateHSAuthority
+AuthDirBadDir
+AuthDirBadDirCCs
+AuthDirRejectUnlisted
+AuthDirListBadDirs
+AuthDirMaxServersPerAuthAddr
+CircuitIdleTimeout
+ControlListenAddress
+DirListenAddress
+DisableIOCP
+DisableV2DirectoryInfo_
+DynamicDHGroups
+DNSListenAddress
+TestingEnableTbEmptyEvent
+ExcludeSingleHopRelays
+FallbackNetworkstatusFile
+FastFirstHopPK
+FetchV2Networkstatus
+Group
+HidServDirectoryV2
+CloseHSClientCircuitsImmediatelyOnTimeout
+CloseHSServiceRendCircuitsImmediatelyOnTimeout
+MaxOnionsPending
+NamingAuthoritativeDirectory
+NATDListenAddress
+PredictedPortsRelevanceTime
+WarnUnsafeSocks
+ORListenAddress
+PathBiasDisableRate
+PathBiasScaleFactor
+PathBiasMultFactor
+PathBiasUseCloseCounts
+PortForwarding
+PortForwardingHelper
+PreferTunneledDirConns
+RecommendedPackages
+RunTesting
+SchedulerLowWaterMark__
+SchedulerHighWaterMark__
+SchedulerMaxFlushCells__
+SocksListenAddress
+StrictEntryNodes
+StrictExitNodes
+Support022HiddenServices
+Tor2webMode
+Tor2webRendezvousPoints
+TLSECGroup
+TransListenAddress
+TunnelDirConns
+UseEntryGuardsAsDirGuards
+UseNTorHandshake
+UserspaceIOCPBuffers
+V1AuthoritativeDirectory
+V2AuthoritativeDirectory
+VoteOnHidServDirectoriesV2
+UseFilteringSSLBufferevents
+__UseFilteringSSLBufferevents
+TestingConsensusMaxDownloadTries
+ClientBootstrapConsensusMaxDownloadTries
+ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries
+TestingDescriptorMaxDownloadTries
+TestingMicrodescMaxDownloadTries
+TestingCertMaxDownloadTries
diff --git a/src/test/conf_examples/obsolete_2/expected b/src/test/conf_examples/obsolete_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/expected
diff --git a/src/test/conf_examples/obsolete_2/torrc b/src/test/conf_examples/obsolete_2/torrc
new file mode 100644
index 0000000000..4f78d47625
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/torrc
@@ -0,0 +1,2 @@
+# This option has been obsolete for some time
+AllowDotExit
diff --git a/src/test/conf_examples/ops_1/cmdline b/src/test/conf_examples/ops_1/cmdline
new file mode 100644
index 0000000000..2bb9bfa132
--- /dev/null
+++ b/src/test/conf_examples/ops_1/cmdline
@@ -0,0 +1 @@
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/expected b/src/test/conf_examples/ops_1/expected
new file mode 100644
index 0000000000..84be6a70e2
--- /dev/null
+++ b/src/test/conf_examples/ops_1/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/torrc b/src/test/conf_examples/ops_1/torrc
new file mode 100644
index 0000000000..daf8ae60fe
--- /dev/null
+++ b/src/test/conf_examples/ops_1/torrc
@@ -0,0 +1,3 @@
+# We'll replace this option on the command line.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_2/cmdline b/src/test/conf_examples/ops_2/cmdline
new file mode 100644
index 0000000000..fdd48a045c
--- /dev/null
+++ b/src/test/conf_examples/ops_2/cmdline
@@ -0,0 +1 @@
+/ORPort
diff --git a/src/test/conf_examples/ops_2/expected b/src/test/conf_examples/ops_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_2/expected
diff --git a/src/test/conf_examples/ops_2/torrc b/src/test/conf_examples/ops_2/torrc
new file mode 100644
index 0000000000..21fcc93f9a
--- /dev/null
+++ b/src/test/conf_examples/ops_2/torrc
@@ -0,0 +1,3 @@
+# We'll remove this option on the command line, and not replace it.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_3/cmdline b/src/test/conf_examples/ops_3/cmdline
new file mode 100644
index 0000000000..e4965d26f8
--- /dev/null
+++ b/src/test/conf_examples/ops_3/cmdline
@@ -0,0 +1 @@
++ORPort 1000
diff --git a/src/test/conf_examples/ops_3/expected b/src/test/conf_examples/ops_3/expected
new file mode 100644
index 0000000000..569d26b577
--- /dev/null
+++ b/src/test/conf_examples/ops_3/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9999
+ORPort 1000
diff --git a/src/test/conf_examples/ops_3/torrc b/src/test/conf_examples/ops_3/torrc
new file mode 100644
index 0000000000..14adf87d7f
--- /dev/null
+++ b/src/test/conf_examples/ops_3/torrc
@@ -0,0 +1,3 @@
+# We will extend this option on the command line
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_4/expected b/src/test/conf_examples/ops_4/expected
new file mode 100644
index 0000000000..bf52f6a330
--- /dev/null
+++ b/src/test/conf_examples/ops_4/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/torrc b/src/test/conf_examples/ops_4/torrc
new file mode 100644
index 0000000000..dcec2aa95d
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc
@@ -0,0 +1,3 @@
+# This value is unadorned, so replaces the one from defaults.torrc.
+
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/torrc.defaults b/src/test/conf_examples/ops_4/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_5/expected b/src/test/conf_examples/ops_5/expected
new file mode 100644
index 0000000000..288721da53
--- /dev/null
+++ b/src/test/conf_examples/ops_5/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9000
+ORPort 9099
diff --git a/src/test/conf_examples/ops_5/torrc b/src/test/conf_examples/ops_5/torrc
new file mode 100644
index 0000000000..3284fc1c55
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc
@@ -0,0 +1,3 @@
+# This value has a plus, and so extends the one from defaults.torrc.
+
++ORPort 9099
diff --git a/src/test/conf_examples/ops_5/torrc.defaults b/src/test/conf_examples/ops_5/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_6/expected b/src/test/conf_examples/ops_6/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_6/expected
diff --git a/src/test/conf_examples/ops_6/torrc b/src/test/conf_examples/ops_6/torrc
new file mode 100644
index 0000000000..4d51caaff7
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc
@@ -0,0 +1,3 @@
+# This value has a slash, and so clears the one from defaults.torrc.
+
+/ORPort
diff --git a/src/test/conf_examples/ops_6/torrc.defaults b/src/test/conf_examples/ops_6/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/relpath_rad/error b/src/test/conf_examples/relpath_rad/error
new file mode 100644
index 0000000000..e131744475
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/error
@@ -0,0 +1 @@
+RunAsDaemon is not compatible with relative paths.
diff --git a/src/test/conf_examples/relpath_rad/torrc b/src/test/conf_examples/relpath_rad/torrc
new file mode 100644
index 0000000000..fe02441c3f
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/torrc
@@ -0,0 +1,4 @@
+
+# Relative-path data directories are incompatible with RunAsDaemon
+DataDirectory ./datadir
+RunAsDaemon 1
diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh
index 68efc1abc5..f730d532a5 100755
--- a/src/test/fuzz/fixup_filenames.sh
+++ b/src/test/fuzz/fixup_filenames.sh
@@ -8,9 +8,9 @@ if [ ! -d "$1" ] ; then
 fi
 
 for fn in "$1"/* ; do
-    prev=`basename "$fn"`
-    post=`sha256sum "$fn" | sed -e 's/ .*//;'`
-    if [ "$prev" == "$post" ] ; then
+    prev=$(basename "$fn")
+    post=$(sha256sum "$fn" | sed -e 's/ .*//;')
+    if [ "$prev" = "$post" ] ; then
       echo "OK $prev"
     else
       echo "mv $prev $post"
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
index 5947a3f48c..656ef0bdb2 100644
--- a/src/test/fuzz/fuzz_consensus.c
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -61,13 +61,13 @@ int
 fuzz_main(const uint8_t *data, size_t sz)
 {
   networkstatus_t *ns;
-  char *str = tor_memdup_nulterm(data, sz);
   const char *eos = NULL;
   networkstatus_type_t tp = NS_TYPE_CONSENSUS;
   if (tor_memstr(data, MIN(sz, 1024), "tus vote"))
     tp = NS_TYPE_VOTE;
   const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote";
-  ns = networkstatus_parse_vote_from_string(str,
+  ns = networkstatus_parse_vote_from_string((const char *)data,
+                                            sz,
                                             &eos,
                                             tp);
   if (ns) {
@@ -76,6 +76,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   } else {
     log_debug(LD_GENERAL, "Parsing as %s failed", what);
   }
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c
index 1bc60e50ee..a31445666c 100644
--- a/src/test/fuzz/fuzz_diff.c
+++ b/src/test/fuzz/fuzz_diff.c
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -42,28 +44,34 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_generate(c1, c2);
+  const char *cp = memchr(c1, 0, c1_len);
+  if (cp)
+    c1_len = cp - c1;
+
+  cp = memchr(c2, 0, c2_len);
+  if (cp)
+    c2_len = cp - c2;
+
+  char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len);
 
   if (c3) {
-    char *c4 = consensus_diff_apply(c1, c3);
+    char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3));
     tor_assert(c4);
-    if (strcmp(c2, c4)) {
-      printf("%s\n", escaped(c1));
-      printf("%s\n", escaped(c2));
+    int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len);
+    if (! equal) {
+      //printf("%s\n", escaped(c1));
+      //printf("%s\n", escaped(c2));
       printf("%s\n", escaped(c3));
       printf("%s\n", escaped(c4));
     }
-    tor_assert(! strcmp(c2, c4));
+    tor_assert(equal);
     tor_free(c3);
     tor_free(c4);
   }
-  tor_free(c1);
-  tor_free(c2);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c
index 9bd3cb0bf8..d8a0f9e590 100644
--- a/src/test/fuzz/fuzz_diff_apply.c
+++ b/src/test/fuzz/fuzz_diff_apply.c
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_apply(c1, c2);
+  char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len);
 
-  tor_free(c1);
-  tor_free(c2);
   tor_free(c3);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c
index 2cbd655898..9d4a6dbb55 100644
--- a/src/test/fuzz/fuzz_hsdescv3.c
+++ b/src/test/fuzz/fuzz_hsdescv3.c
@@ -35,16 +35,21 @@ mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert,
 
 static size_t
 mock_decrypt_desc_layer(const hs_descriptor_t *desc,
-                        const uint8_t *encrypted_blob,
-                        size_t encrypted_blob_size,
                         const uint8_t *descriptor_cookie,
-                        int is_superencrypted_layer,
+                        bool is_superencrypted_layer,
                         char **decrypted_out)
 {
   (void)is_superencrypted_layer;
   (void)desc;
   (void)descriptor_cookie;
   const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
+
   if (encrypted_blob_size < overhead)
     return 0;
   *decrypted_out = tor_memdup_nulterm(
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
index 2798c47d23..44393b3a10 100644
--- a/src/test/fuzz/fuzz_http.c
+++ b/src/test/fuzz/fuzz_http.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/dircache/dircache.h"
diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c
index a60fc36804..2a597cae74 100644
--- a/src/test/fuzz/fuzz_http_connect.c
+++ b/src/test/fuzz/fuzz_http_connect.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh
index b4a17ed8cb..406ab498d9 100755
--- a/src/test/fuzz/fuzz_multi.sh
+++ b/src/test/fuzz/fuzz_multi.sh
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 MEMLIMIT_BYTES=21990500990976
 
 N_CPUS=1
@@ -6,9 +8,9 @@ if [ $# -ge 1 ]; then
     shift
 fi
 
-FILTER=echo
+FILTER="echo"
 
-for i in `seq -w "$N_CPUS"`; do
+for i in $(seq -w "$N_CPUS"); do
     if [ "$i" -eq 1 ]; then
         if [ "$N_CPUS" -eq 1 ]; then
             INSTANCE=""
diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c
index 06cb08391e..d6c416a0f9 100644
--- a/src/test/fuzz/fuzz_socks.c
+++ b/src/test/fuzz/fuzz_socks.c
@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/err/backtrace.h"
 #include "lib/log/log.h"
 #include "core/proto/proto_socks.h"
diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c
new file mode 100644
index 0000000000..459b4e21aa
--- /dev/null
+++ b/src/test/fuzz/fuzz_strops.c
@@ -0,0 +1,253 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fuzz_strops.c
+ * \brief Fuzzers for various string encoding/decoding operations
+ **/
+
+#include "orconfig.h"
+
+#include "lib/cc/torint.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/confline.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/muldiv.h"
+
+#include "test/fuzz/fuzzing.h"
+
+#include <stdio.h>
+#include <string.h>
+
+int
+fuzz_init(void)
+{
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+typedef struct chunk_t {
+  uint8_t *buf;
+  size_t len;
+} chunk_t;
+
+#define chunk_free(ch)                          \
+  FREE_AND_NULL(chunk_t, chunk_free_, (ch))
+
+static chunk_t *
+chunk_new(size_t len)
+{
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = tor_malloc(len);
+  ch->len = len;
+  return ch;
+}
+static void
+chunk_free_(chunk_t *ch)
+{
+  if (!ch)
+    return;
+  tor_free(ch->buf);
+  tor_free(ch);
+}
+static bool
+chunk_eq(const chunk_t *a, const chunk_t *b)
+{
+  return a->len == b->len && fast_memeq(a->buf, b->buf, a->len);
+}
+
+static chunk_t *
+b16_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2));
+  int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b16_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len * 2 + 1);
+  base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  return ch;
+}
+
+static chunk_t *
+b32_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX
+  int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b32_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(base32_encoded_size(inp->len));
+  base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+b64_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter.
+  int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b64_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len));
+  base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+c_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  char *r = NULL;
+  (void) unescape_string(s, &r, &ch->len);
+  tor_free(s);
+  ch->buf = (uint8_t*) r;
+  if (!ch->buf) {
+    tor_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+c_enc(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = (uint8_t*)esc_for_log(s);
+  tor_free(s);
+  ch->len = strlen((char*)ch->buf);
+  return ch;
+}
+
+static int kv_flags = 0;
+static config_line_t *
+kv_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  config_line_t *res = kvline_parse(s, kv_flags);
+  tor_free(s);
+  return res;
+}
+static chunk_t *
+kv_enc(const config_line_t *inp)
+{
+  char *s = kvline_encode(inp, kv_flags);
+  if (!s)
+    return NULL;
+  chunk_t *res = tor_malloc(sizeof(chunk_t));
+  res->buf = (uint8_t*)s;
+  res->len = strlen(s);
+  return res;
+}
+
+/* Given an encoder function, a decoder function, and a function to free
+ * the decoded object, check whether any string that successfully decoded
+ * will then survive an encode-decode-encode round-trip unchanged.
+ */
+#define ENCODE_ROUNDTRIP(E,D,FREE)              \
+  STMT_BEGIN {                                  \
+    bool err = false;                           \
+    a = D(&inp);                                \
+    if (!a)                                     \
+      return 0;                                 \
+    b = E(a);                                   \
+    tor_assert(b);                              \
+    c = D(b);                                   \
+    tor_assert(c);                              \
+    d = E(c);                                   \
+    tor_assert(d);                              \
+    if (!chunk_eq(b,d)) {                       \
+      printf("Unequal chunks: %s\n",            \
+             hex_str((char*)b->buf, b->len));   \
+      printf("             vs %s\n",            \
+             hex_str((char*)d->buf, d->len));   \
+      err = true;                               \
+    }                                           \
+    FREE(a);                                    \
+    chunk_free(b);                              \
+    FREE(c);                                    \
+    chunk_free(d);                              \
+    tor_assert(!err);                           \
+  } STMT_END
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  if (!data_size)
+    return 0;
+
+  chunk_t inp = { (uint8_t*)stdin_buf, data_size };
+  chunk_t *b=NULL,*d=NULL;
+  void *a=NULL,*c=NULL;
+
+  switch (stdin_buf[0]) {
+    case 0:
+      ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_);
+      break;
+    case 1:
+      ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_);
+      break;
+    case 2:
+      ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_);
+      break;
+    case 3:
+      ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_);
+      break;
+    case 5:
+      kv_flags = KV_QUOTED|KV_OMIT_KEYS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 6:
+      kv_flags = 0;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 7:
+      kv_flags = KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 8:
+      kv_flags = KV_QUOTED;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 9:
+      kv_flags = KV_QUOTED|KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    }
+
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
index 967397d1af..7b61b8df2d 100644
--- a/src/test/fuzz/fuzz_vrs.c
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -3,6 +3,7 @@
 #define NS_PARSE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #include "core/or/or.h"
+#include "feature/dirauth/dirvote.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/dirparse/unparseable.h"
 #include "lib/memarea/memarea.h"
@@ -35,9 +36,12 @@ fuzz_init(void)
   dummy_vote = tor_malloc_zero(sizeof(*dummy_vote));
   dummy_vote->known_flags = smartlist_new();
   smartlist_split_string(dummy_vote->known_flags,
-                         "Authority BadExit Exit Fast Guard HSDir "
-                         "NoEdConsensus Running Stable V2Dir Valid",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          " ", 0, 0);
+  smartlist_split_string(dummy_vote->known_flags,
+                         DIRVOTE_OPTIONAL_FLAGS,
+                         " ", 0, 0);
+  smartlist_sort_strings(dummy_vote->known_flags);
   return 0;
 }
 
@@ -53,24 +57,24 @@ fuzz_cleanup(void)
 int
 fuzz_main(const uint8_t *data, size_t sz)
 {
-  char *str = tor_memdup_nulterm(data, sz);
   const char *s;
   routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL;
   vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs));
   smartlist_t *tokens = smartlist_new();
+  const char *eos = (const char *)data + sz;
 
-  s = str;
-  rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_md = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_MICRODESC);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                               dummy_vote, vrs, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
@@ -82,6 +86,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   vote_routerstatus_free(vrs);
   memarea_clear(area);
   smartlist_free(tokens);
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h
index 150ac4aa7d..2d278825ec 100644
--- a/src/test/fuzz/fuzzing.h
+++ b/src/test/fuzz/fuzzing.h
@@ -9,5 +9,5 @@ int fuzz_main(const uint8_t *data, size_t sz);
 
 void disable_signature_checking(void);
 
-#endif /* FUZZING_H */
+#endif /* !defined(FUZZING_H) */
 
diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c
index 8ea4898522..604aba7a7f 100644
--- a/src/test/fuzz/fuzzing_common.c
+++ b/src/test/fuzz/fuzzing_common.c
@@ -1,14 +1,17 @@
 /* Copyright (c) 2016-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define CRYPTO_ED25519_PRIVATE
+#define CONFIG_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
+#include "app/main/subsysmgr.h"
 #include "lib/err/backtrace.h"
 #include "app/config/config.h"
 #include "test/fuzz/fuzzing.h"
 #include "lib/compress/compress.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 static or_options_t *mock_options = NULL;
 static const or_options_t *
@@ -94,12 +97,10 @@ disable_signature_checking(void)
 static void
 global_init(void)
 {
-  tor_threads_init();
-  tor_compress_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
 
-  /* Initialise logging first */
-  init_logging(1);
-  configure_backtrace_handler(get_version());
+  tor_compress_init();
 
   if (crypto_global_init(0, NULL, NULL) < 0)
     abort();
@@ -111,7 +112,7 @@ global_init(void)
   }
 
   /* set up the options. */
-  mock_options = tor_malloc_zero(sizeof(or_options_t));
+  mock_options = options_new();
   MOCK(get_options, mock_get_options);
 
   /* Make BUG() and nonfatal asserts crash */
@@ -137,7 +138,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
   return fuzz_main(Data, Size);
 }
 
-#else /* Not LLVM_FUZZ, so AFL. */
+#else /* !defined(LLVM_FUZZ) */
 
 int
 main(int argc, char **argv)
@@ -166,7 +167,7 @@ main(int argc, char **argv)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
 
@@ -189,9 +190,9 @@ main(int argc, char **argv)
   if (fuzz_cleanup() < 0)
     abort();
 
-  tor_free(mock_options);
+  or_options_free(mock_options);
   UNMOCK(get_options);
   return 0;
 }
 
-#endif
+#endif /* defined(LLVM_FUZZ) */
diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am
index 27eeced8c5..d0711f05d6 100644
--- a/src/test/fuzz/include.am
+++ b/src/test/fuzz/include.am
@@ -153,6 +153,16 @@ src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_strops_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_strops.c
+src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_fuzz_vrs_SOURCES = \
 	src/test/fuzz/fuzzing_common.c \
 	src/test/fuzz/fuzz_vrs.c
@@ -176,6 +186,7 @@ FUZZERS = \
 	src/test/fuzz/fuzz-iptsv2 \
 	src/test/fuzz/fuzz-microdesc \
 	src/test/fuzz/fuzz-socks \
+	src/test/fuzz/fuzz-strops \
 	src/test/fuzz/fuzz-vrs
 endif
 
@@ -291,6 +302,15 @@ src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_strops_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_lf_fuzz_vrs_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
@@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \
 	src/test/fuzz/lf-fuzz-iptsv2 \
 	src/test/fuzz/lf-fuzz-microdesc \
 	src/test/fuzz/lf-fuzz-socks \
+	src/test/fuzz/lf-fuzz-strops \
 	src/test/fuzz/lf-fuzz-vrs
 
 else
@@ -406,6 +427,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
@@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \
 	src/test/fuzz/liboss-fuzz-iptsv2.a \
 	src/test/fuzz/liboss-fuzz-microdesc.a \
 	src/test/fuzz/liboss-fuzz-socks.a \
+	src/test/fuzz/liboss-fuzz-strops.a \
 	src/test/fuzz/liboss-fuzz-vrs.a
 
 else
diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh
index 87d3dda13c..ce43812bb8 100755
--- a/src/test/fuzz/minimize.sh
+++ b/src/test/fuzz/minimize.sh
@@ -7,7 +7,7 @@ if [ ! -d "$1" ] ; then
     exit 1
 fi
 
-which=`basename "$1"`
+which=$(basename "$1")
 
 mkdir "$1.out"
 afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}"
diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh
index f7b3adffb1..b883352402 100755
--- a/src/test/fuzz_static_testcases.sh
+++ b/src/test/fuzz_static_testcases.sh
@@ -14,7 +14,7 @@ fi
 
 
 for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do
-    f=`basename $fuzzer`
+    f=$(basename "$fuzzer")
     case="${f#fuzz-}"
     if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then
         echo "Running tests for ${case}"
diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c
index f2ae8398df..0a21fe576b 100644
--- a/src/test/hs_test_helpers.c
+++ b/src/test/hs_test_helpers.c
@@ -21,26 +21,35 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
   /* For a usable intro point we need at least two link specifiers: One legacy
    * keyid and one ipv4 */
   {
-    hs_desc_link_specifier_t *ls_legacy = tor_malloc_zero(sizeof(*ls_legacy));
-    hs_desc_link_specifier_t *ls_v4 = tor_malloc_zero(sizeof(*ls_v4));
-    ls_legacy->type = LS_LEGACY_ID;
-    memcpy(ls_legacy->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8",
-           DIGEST_LEN);
-    ls_v4->u.ap.port = 9001;
-    int family = tor_addr_parse(&ls_v4->u.ap.addr, addr);
+    tor_addr_t a;
+    tor_addr_make_unspec(&a);
+    link_specifier_t *ls_legacy = link_specifier_new();
+    link_specifier_t *ls_ip = link_specifier_new();
+    link_specifier_set_ls_type(ls_legacy, LS_LEGACY_ID);
+    memset(link_specifier_getarray_un_legacy_id(ls_legacy), 'C',
+           link_specifier_getlen_un_legacy_id(ls_legacy));
+    int family = tor_addr_parse(&a, addr);
     switch (family) {
     case AF_INET:
-          ls_v4->type = LS_IPV4;
+          link_specifier_set_ls_type(ls_ip, LS_IPV4);
+          link_specifier_set_un_ipv4_addr(ls_ip, tor_addr_to_ipv4h(&a));
+          link_specifier_set_un_ipv4_port(ls_ip, 9001);
           break;
         case AF_INET6:
-          ls_v4->type = LS_IPV6;
+          link_specifier_set_ls_type(ls_ip, LS_IPV6);
+          memcpy(link_specifier_getarray_un_ipv6_addr(ls_ip),
+                 tor_addr_to_in6_addr8(&a),
+                 link_specifier_getlen_un_ipv6_addr(ls_ip));
+          link_specifier_set_un_ipv6_port(ls_ip, 9001);
           break;
         default:
-          /* Stop the test, not suppose to have an error. */
-          tt_int_op(family, OP_EQ, AF_INET);
+          /* Stop the test, not supposed to have an error.
+           * Compare with -1 to show the actual family.
+           */
+          tt_int_op(family, OP_EQ, -1);
     }
     smartlist_add(ip->link_specifiers, ls_legacy);
-    smartlist_add(ip->link_specifiers, ls_v4);
+    smartlist_add(ip->link_specifiers, ls_ip);
   }
 
   ret = ed25519_keypair_generate(&auth_kp, 0);
@@ -202,7 +211,6 @@ void
 hs_helper_desc_equal(const hs_descriptor_t *desc1,
                      const hs_descriptor_t *desc2)
 {
-  char *addr1 = NULL, *addr2 = NULL;
   /* Plaintext data section. */
   tt_int_op(desc1->plaintext_data.version, OP_EQ,
             desc2->plaintext_data.version);
@@ -291,35 +299,57 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
       tt_int_op(smartlist_len(ip1->link_specifiers), ==,
                 smartlist_len(ip2->link_specifiers));
       for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) {
-        hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
-                                 *ls2 = smartlist_get(ip2->link_specifiers, j);
-        tt_int_op(ls1->type, ==, ls2->type);
-        switch (ls1->type) {
+        link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
+                         *ls2 = smartlist_get(ip2->link_specifiers, j);
+        tt_int_op(link_specifier_get_ls_type(ls1), ==,
+                  link_specifier_get_ls_type(ls2));
+        switch (link_specifier_get_ls_type(ls1)) {
           case LS_IPV4:
+            {
+              uint32_t addr1 = link_specifier_get_un_ipv4_addr(ls1);
+              uint32_t addr2 = link_specifier_get_un_ipv4_addr(ls2);
+              tt_int_op(addr1, OP_EQ, addr2);
+              uint16_t port1 = link_specifier_get_un_ipv4_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv4_port(ls2);
+              tt_int_op(port1, ==, port2);
+            }
+            break;
           case LS_IPV6:
             {
-              addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr);
-              addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr);
-              tt_str_op(addr1, OP_EQ, addr2);
-              tor_free(addr1);
-              tor_free(addr2);
-              tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port);
+              const uint8_t *addr1 =
+                link_specifier_getconstarray_un_ipv6_addr(ls1);
+              const uint8_t *addr2 =
+                link_specifier_getconstarray_un_ipv6_addr(ls2);
+              tt_int_op(link_specifier_getlen_un_ipv6_addr(ls1), OP_EQ,
+                        link_specifier_getlen_un_ipv6_addr(ls2));
+              tt_mem_op(addr1, OP_EQ, addr2,
+                        link_specifier_getlen_un_ipv6_addr(ls1));
+              uint16_t port1 = link_specifier_get_un_ipv6_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv6_port(ls2);
+              tt_int_op(port1, ==, port2);
             }
             break;
           case LS_LEGACY_ID:
-            tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id,
-                      sizeof(ls1->u.legacy_id));
+            {
+              const uint8_t *id1 =
+                link_specifier_getconstarray_un_legacy_id(ls1);
+              const uint8_t *id2 =
+                link_specifier_getconstarray_un_legacy_id(ls2);
+              tt_int_op(link_specifier_getlen_un_legacy_id(ls1), OP_EQ,
+                        link_specifier_getlen_un_legacy_id(ls2));
+              tt_mem_op(id1, OP_EQ, id2,
+                        link_specifier_getlen_un_legacy_id(ls1));
+            }
             break;
           default:
             /* Unknown type, caught it and print its value. */
-            tt_int_op(ls1->type, OP_EQ, -1);
+            tt_int_op(link_specifier_get_ls_type(ls1), OP_EQ, -1);
         }
       }
     }
   }
 
  done:
-  tor_free(addr1);
-  tor_free(addr2);
+  ;
 }
 
diff --git a/src/test/include.am b/src/test/include.am
index ecb7689579..d8e25dea9f 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -23,7 +23,9 @@ TESTSCRIPTS = \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh \
-	src/test/test_switch_id.sh
+	src/test/test_switch_id.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh
 
 if USE_RUST
 TESTSCRIPTS += \
@@ -31,9 +33,20 @@ TESTSCRIPTS += \
 endif
 
 if USEPYTHON
-TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += \
+	src/test/test_ntor.sh \
+	src/test/test_hs_ntor.sh \
+	src/test/test_bt.sh \
+	scripts/maint/practracker/test_practracker.sh
+
+if COVERAGE_ENABLED
+# ...
+else
+# Only do this when coverage is not on, since it invokes lots of code
+# in a kind of unpredictable way.
 TESTSCRIPTS += src/test/test_rebind.sh
 endif
+endif
 
 TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue \
@@ -46,10 +59,8 @@ TESTS += src/test/test src/test/test-slow src/test/test-memwipe \
 TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \
 	single-onion-v23
 # only run if we can ping6 ::1 (localhost)
-# IPv6-only v3 single onion services don't work yet, so we don't test the
-# single-onion-v23-ipv6-md flavor
 TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \
-	single-onion-ipv6-md
+	single-onion-v23-ipv6-md
 # only run if we can find a stable (or simply another) version of tor
 TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2
 
@@ -65,10 +76,11 @@ noinst_PROGRAMS+= \
 	src/test/test \
 	src/test/test-slow \
 	src/test/test-memwipe \
-	src/test/test-child \
+	src/test/test-process \
 	src/test/test_workqueue \
 	src/test/test-switch-id \
-	src/test/test-timers
+	src/test/test-timers \
+	src/test/test-rng
 endif
 
 src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
@@ -84,22 +96,28 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
 src_test_test_SOURCES =
 
 if UNITTESTS_ENABLED
+
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_test_test_SOURCES += \
 	src/test/log_test_helpers.c \
 	src/test/hs_test_helpers.c \
 	src/test/rend_test_helpers.c \
+	src/test/resolve_test_helpers.c \
+	src/test/rng_test_helpers.c \
 	src/test/test.c \
 	src/test/test_accounting.c \
 	src/test/test_addr.c \
 	src/test/test_address.c \
 	src/test/test_address_set.c \
 	src/test/test_bridges.c \
+	src/test/test_btrack.c \
 	src/test/test_buffers.c \
 	src/test/test_bwmgt.c \
 	src/test/test_cell_formats.c \
 	src/test/test_cell_queue.c \
 	src/test/test_channel.c \
 	src/test/test_channelpadding.c \
+	src/test/test_circuitpadding.c \
 	src/test/test_channeltls.c \
 	src/test/test_checkdir.c \
 	src/test/test_circuitlist.c \
@@ -109,6 +127,8 @@ src_test_test_SOURCES += \
 	src/test/test_circuitstats.c \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
+	src/test/test_confmgr.c \
+	src/test/test_confparse.c \
 	src/test/test_connection.c \
 	src/test/test_conscache.c \
 	src/test/test_consdiff.c \
@@ -118,10 +138,12 @@ src_test_test_SOURCES += \
 	src/test/test_controller_events.c \
 	src/test/test_crypto.c \
 	src/test/test_crypto_ope.c \
+	src/test/test_crypto_rng.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
 	src/test/test_dir_common.c \
 	src/test/test_dir_handle_get.c \
+	src/test/test_dispatch.c \
 	src/test/test_dos.c \
 	src/test/test_entryconn.c \
 	src/test/test_entrynodes.c \
@@ -140,12 +162,15 @@ src_test_test_SOURCES += \
 	src/test/test_handles.c \
 	src/test/test_hs_cache.c \
 	src/test/test_hs_descriptor.c \
+	src/test/test_hs_dos.c \
 	src/test/test_introduce.c \
 	src/test/test_keypin.c \
 	src/test/test_link_handshake.c \
 	src/test/test_logging.c \
 	src/test/test_mainloop.c \
 	src/test/test_microdesc.c \
+	src/test/test_namemap.c \
+	src/test/test_netinfo.c \
 	src/test/test_nodelist.c \
 	src/test/test_oom.c \
 	src/test/test_oos.c \
@@ -153,11 +178,16 @@ src_test_test_SOURCES += \
 	src/test/test_pem.c \
 	src/test/test_periodic_event.c \
 	src/test/test_policy.c \
+	src/test/test_process.c \
+	src/test/test_process_descs.c \
+	src/test/test_prob_distr.c \
 	src/test/test_procmon.c \
 	src/test/test_proto_http.c \
 	src/test/test_proto_misc.c \
 	src/test/test_protover.c \
 	src/test/test_pt.c \
+	src/test/test_pubsub_build.c \
+	src/test/test_pubsub_msg.c \
 	src/test/test_relay.c \
 	src/test/test_relaycell.c \
 	src/test/test_relaycrypt.c \
@@ -168,19 +198,23 @@ src_test_test_SOURCES += \
 	src/test/test_routerlist.c \
 	src/test/test_routerset.c \
 	src/test/test_scheduler.c \
+	src/test/test_sendme.c \
 	src/test/test_shared_random.c \
 	src/test/test_socks.c \
 	src/test/test_status.c \
 	src/test/test_storagedir.c \
 	src/test/test_threads.c \
+	src/test/test_token_bucket.c \
 	src/test/test_tortls.c \
 	src/test/test_util.c \
 	src/test/test_util_format.c \
 	src/test/test_util_process.c \
+	src/test/test_voting_flags.c \
 	src/test/test_voting_schedule.c \
 	src/test/test_x509.c \
 	src/test/test_helpers.c \
 	src/test/test_dns.c \
+	src/test/test_parsecommon.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -198,9 +232,13 @@ endif
 src_test_test_slow_SOURCES =
 if UNITTESTS_ENABLED
 src_test_test_slow_SOURCES += \
+	src/test/rng_test_helpers.c \
 	src/test/test_slow.c \
 	src/test/test_crypto_slow.c \
-	src/test/test_util_slow.c \
+	src/test/test_process_slow.c \
+	src/test/test_prob_distr.c \
+	src/test/ptr_helpers.c \
+	src/test/test_ptr_slow.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -251,6 +289,12 @@ src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS)
 src_test_test_slow_LDADD = $(src_test_test_LDADD)
 src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS)
 
+src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS)
+src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS)
+src_test_test_rng_SOURCES = src/test/test_rng.c
+src_test_test_rng_LDFLAGS =  $(src_test_test_LDFLAGS)
+src_test_test_rng_LDADD = $(src_test_test_LDADD)
+
 src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS)
 # Don't use bugtrap cflags here: memwipe tests require memory violations.
 src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS)
@@ -292,12 +336,16 @@ src_test_test_timers_LDADD = \
 	@TOR_LZMA_LIBS@
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS+= \
 	src/test/fakechans.h \
 	src/test/hs_test_helpers.h \
 	src/test/log_test_helpers.h \
 	src/test/rend_test_helpers.h \
+	src/test/resolve_test_helpers.h \
+	src/test/rng_test_helpers.h \
 	src/test/test.h \
+	src/test/ptr_helpers.h \
 	src/test/test_helpers.h \
 	src/test/test_dir_common.h \
 	src/test/test_connection.h \
@@ -369,7 +417,9 @@ EXTRA_DIST += \
 	src/test/test_workqueue_efd2.sh \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
-	src/test/test_workqueue_socketpair.sh
+	src/test/test_workqueue_socketpair.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh
 
 test-rust:
 	$(TESTS_ENVIRONMENT) "$(abs_top_srcdir)/src/test/test_rust.sh"
diff --git a/src/test/ope_ref.py b/src/test/ope_ref.py
index f9bd97c546..b2f7012563 100644
--- a/src/test/ope_ref.py
+++ b/src/test/ope_ref.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # Copyright 2018-2019, The Tor Project, Inc. See LICENSE for licensing info.
 
 # Reference implementation for our rudimentary OPE code, used to
diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c
new file mode 100644
index 0000000000..425733dc1b
--- /dev/null
+++ b/src/test/prob_distr_mpfr_ref.c
@@ -0,0 +1,64 @@
+/* Copyright 2012-2019, The Tor Project, Inc
+ * See LICENSE for licensing information */
+
+/** prob_distr_mpfr_ref.c
+ *
+ * Example reference file for GNU MPFR vectors tested in test_prob_distr.c .
+ * Code by Riastradh.
+ */
+
+#include <complex.h>
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+
+/* Must come after <stdio.h> so we get mpfr_printf.  */
+#include <mpfr.h>
+
+/*  gcc -o mpfr prob_distr_mpfr_ref.c -lmpfr -lm */
+
+/* Computes logit(p) for p = .49999 */
+int
+main(void)
+{
+  mpfr_t p, q, r;
+  mpfr_init(p);
+  mpfr_set_prec(p, 200);
+  mpfr_init(q);
+  mpfr_set_prec(q, 200);
+  mpfr_init(r);
+  mpfr_set_prec(r, 200);
+  mpfr_set_d(p, .49999, MPFR_RNDN);
+  mpfr_set_d(q, 1, MPFR_RNDN);
+  /* r := q - p = 1 - p */
+  mpfr_sub(r, q, p, MPFR_RNDN);
+  /* q := p/r = p/(1 - p) */
+  mpfr_div(q, p, r, MPFR_RNDN);
+  /* r := log(q) = log(p/(1 - p)) */
+  mpfr_log(r, q, MPFR_RNDN);
+  mpfr_printf("mpfr 200-bit\t%.128Rg\n", r);
+
+  /*
+   * Print a double approximation to logit three different ways.  All
+   * three agree bit for bit on the libms I tried, with the nextafter
+   * adjustment (which is well within the 10 eps relative error bound
+   * advertised).  Apparently I must have used the Goldberg expression
+   * for what I wrote down in the test case.
+   */
+  printf("mpfr 53-bit\t%.17g\n", nextafter(mpfr_get_d(r, MPFR_RNDN), 0), 0);
+  volatile double p0 = .49999;
+  printf("log1p\t\t%.17g\n", nextafter(-log1p((1 - 2*p0)/p0), 0));
+  volatile double x = (1 - 2*p0)/p0;
+  volatile double xp1 = x + 1;
+  printf("Goldberg\t%.17g\n", -x*log(xp1)/(xp1 - 1));
+
+  /*
+   * Print a bad approximation, using the naive expression, to see a
+   * lot of wrong digits, far beyond the 10 eps relative error attained
+   * by -log1p((1 - 2*p)/p).
+   */
+  printf("naive\t\t%.17g\n", log(p0/(1 - p0)));
+
+  fflush(stdout);
+  return ferror(stdout);
+}
diff --git a/src/test/ptr_helpers.c b/src/test/ptr_helpers.c
new file mode 100644
index 0000000000..a55ab437fa
--- /dev/null
+++ b/src/test/ptr_helpers.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/ptr_helpers.h"
+
+/**
+ * Cast <b> (inptr_t value) to a void pointer.
+ */
+void *
+cast_intptr_to_voidstar(intptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to inptr_t value.
+ */
+intptr_t
+cast_voidstar_to_intptr(void *x)
+{
+  intptr_t r = (intptr_t)x;
+
+  return r;
+}
+
+/**
+ * Cast x (uinptr_t value) to void pointer.
+ */
+void *
+cast_uintptr_to_voidstar(uintptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to uinptr_t value.
+ */
+uintptr_t
+cast_voidstar_to_uintptr(void *x)
+{
+  uintptr_t r = (uintptr_t)x;
+
+  return r;
+}
diff --git a/src/test/ptr_helpers.h b/src/test/ptr_helpers.h
new file mode 100644
index 0000000000..7349bddd51
--- /dev/null
+++ b/src/test/ptr_helpers.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PTR_HELPERS_H
+#define TOR_PTR_HELPERS_H
+
+#include <stdint.h>
+
+void *
+cast_intptr_to_voidstar(intptr_t x);
+
+intptr_t
+cast_voidstar_to_intptr(void *x);
+
+void *
+cast_uintptr_to_voidstar(uintptr_t x);
+
+uintptr_t
+cast_voidstar_to_uintptr(void *x);
+
+#endif /* !defined(TOR_PTR_HELPERS_H) */
diff --git a/src/test/resolve_test_helpers.c b/src/test/resolve_test_helpers.c
new file mode 100644
index 0000000000..73ea730149
--- /dev/null
+++ b/src/test/resolve_test_helpers.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.c
+ * @brief Helper functions for mocking libc's blocking hostname lookup
+ *   facilities.
+ **/
+
+#define RESOLVE_PRIVATE
+#include "orconfig.h"
+#include "test/resolve_test_helpers.h"
+#include "lib/net/address.h"
+#include "lib/net/resolve.h"
+#include "test/test.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * Mock replacement for our getaddrinfo/gethostbyname wrapper.
+ **/
+static int
+replacement_host_lookup(const char *name, uint16_t family, tor_addr_t *addr)
+{
+  static const struct lookup_table_ent {
+    const char *name;
+    const char *ipv4;
+    const char *ipv6;
+  } entries[] = {
+    { "localhost", "127.0.0.1", "::1" },
+    { "torproject.org", "198.51.100.6", "2001:DB8::700" },
+    { NULL, NULL, NULL },
+  };
+
+  int r = -1;
+
+  for (unsigned i = 0; entries[i].name != NULL; ++i) {
+    if (!strcasecmp(name, entries[i].name)) {
+      if (family == AF_INET6) {
+        int s = tor_addr_parse(addr, entries[i].ipv6);
+        tt_int_op(s, OP_EQ, AF_INET6);
+      } else {
+        int s = tor_addr_parse(addr, entries[i].ipv4);
+        tt_int_op(s, OP_EQ, AF_INET);
+      }
+      r = 0;
+      break;
+    }
+  }
+
+  log_debug(LD_GENERAL, "resolve(%s,%d) => %s",
+            name, family, r == 0 ? fmt_addr(addr) : "-1");
+
+  return r;
+ done:
+  return -1;
+}
+
+/**
+ * Set up a mock replacement for our wrapper on libc's resolver code.
+ *
+ * According to our replacement, only "localhost" and "torproject.org"
+ * are real addresses; everything else doesn't exist.
+ *
+ * Use this function to avoid using the DNS resolver during unit tests;
+ * call unmock_hostname_resolver() when you're done.
+ **/
+void
+mock_hostname_resolver(void)
+{
+  MOCK(tor_addr_lookup_host_impl, replacement_host_lookup);
+}
+
+/**
+ * Unmock our wrappers for libc's blocking hostname resolver code.
+ **/
+void
+unmock_hostname_resolver(void)
+{
+  UNMOCK(tor_addr_lookup_host_impl);
+}
diff --git a/src/test/resolve_test_helpers.h b/src/test/resolve_test_helpers.h
new file mode 100644
index 0000000000..e7d2e29373
--- /dev/null
+++ b/src/test/resolve_test_helpers.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.h
+ * @brief Header for test/resolve_test_helpers.c
+ **/
+
+#ifndef TOR_TEST_RESOLVE_TEST_HELPERS_H
+#define TOR_TEST_RESOLVE_TEST_HELPERS_H
+
+void mock_hostname_resolver(void);
+void unmock_hostname_resolver(void);
+
+#endif /* !defined(TOR_TEST_RESOLVE_TEST_HELPERS_H) */
diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c
new file mode 100644
index 0000000000..7024fb5793
--- /dev/null
+++ b/src/test/rng_test_helpers.c
@@ -0,0 +1,259 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rng_test_helpers.c
+ * \brief Helpers for overriding PRNGs during unit tests.
+ *
+ * We define two PRNG overrides: a "reproducible PRNG" where the seed is
+ * chosen randomly but the stream can be replayed later on in case a bug is
+ * found, and a "deterministic PRNG" where the seed is fixed in the unit
+ * tests.
+ *
+ * Obviously, this code is testing-only.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "ext/tinytest.h"
+
+#include "test/rng_test_helpers.h"
+
+#ifndef TOR_UNIT_TESTS
+#error "No. Never link this code into Tor proper."
+#endif
+
+/**
+ * True iff the RNG is currently replaced.  Prevents double-replacement.
+ **/
+static bool rng_is_replaced = false;
+
+/**
+ * Mutex to protect deterministic prng.
+ *
+ * Note that if you actually _use_ the prng from two threads at the same time,
+ * the results will probably be nondeterministic anyway.
+ */
+static tor_mutex_t *rng_mutex = NULL;
+
+/**
+ * Cached old value for the thread prng.
+ **/
+static crypto_fast_rng_t *stored_fast_rng = NULL;
+
+/** replacement for crypto_strongest_rand that delegates to crypto_rand. */
+static void
+mock_crypto_strongest_rand(uint8_t *out, size_t len)
+{
+  crypto_rand((char *)out, len);
+}
+
+/* This is the seed of the deterministic randomness. */
+static uint8_t rng_seed[16];
+static crypto_xof_t *rng_xof = NULL;
+
+/**
+ * Print the seed for our PRNG to stdout.  We use this when we're failed
+ * test that had a reproducible RNG set.
+ **/
+void
+testing_dump_reproducible_rng_seed(void)
+{
+  printf("\n"
+         "Seed: %s\n",
+         hex_str((const char*)rng_seed, sizeof(rng_seed)));
+}
+
+/** Produce deterministic randomness for the stochastic tests using the global
+ * rng_xof output.
+ *
+ * This function produces deterministic data over multiple calls iff it's
+ * called in the same call order with the same 'n' parameter.
+ * If not, outputs will deviate. */
+static void
+crypto_rand_deterministic(char *out, size_t n)
+{
+  tor_assert(rng_xof);
+  tor_mutex_acquire(rng_mutex);
+  crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n);
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Implementation helper: override our crypto_rand() PRNG with a given seed of
+ * length <b>seed_len</b>.  Overlong seeds are truncated; short ones are
+ * padded.
+ **/
+static void
+enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len)
+{
+  tor_assert(!rng_is_replaced);
+  tor_assert(crypto_rand == crypto_rand__real);
+
+  memset(rng_seed, 0, sizeof(rng_seed));
+  memcpy(rng_seed, seed, MIN(seed_len, sizeof(rng_seed)));
+
+  rng_mutex = tor_mutex_new();
+
+  crypto_xof_free(rng_xof);
+  rng_xof = crypto_xof_new();
+  crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed));
+  MOCK(crypto_rand, crypto_rand_deterministic);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+
+  uint8_t fast_rng_seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(fast_rng_seed, 0xff, sizeof(fast_rng_seed));
+  memcpy(fast_rng_seed, rng_seed, MIN(sizeof(rng_seed),
+                                      sizeof(fast_rng_seed)));
+  crypto_fast_rng_t *fast_rng = crypto_fast_rng_new_from_seed(fast_rng_seed);
+  crypto_fast_rng_disable_reseed(fast_rng);
+  stored_fast_rng = crypto_replace_thread_fast_rng(fast_rng);
+
+  rng_is_replaced = true;
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a randomly chosen seed.  In the event of an
+ * error, you can log the seed later on with
+ * testing_dump_reproducible_rng_seed.
+ **/
+void
+testing_enable_reproducible_rng(void)
+{
+  const char *provided_seed = getenv("TOR_TEST_RNG_SEED");
+  if (provided_seed) {
+    size_t hexlen = strlen(provided_seed);
+    size_t seedlen = hexlen / 2;
+    uint8_t *seed = tor_malloc(hexlen / 2);
+    if (base16_decode((char*)seed, seedlen, provided_seed, hexlen) < 0) {
+      puts("Cannot decode value in TOR_TEST_RNG_SEED");
+      exit(1);
+    }
+    enable_deterministic_rng_impl(seed, seedlen);
+    tor_free(seed);
+  } else {
+    uint8_t seed[16];
+    crypto_rand((char*)seed, sizeof(seed));
+    enable_deterministic_rng_impl(seed, sizeof(seed));
+  }
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a fixed seed.  This variant is mainly useful
+ * for cases when we don't want coverage to change between runs.
+ *
+ * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of
+ * this "rng".  If you need a specific output, use
+ * testing_enable_prefilled_rng() instead.
+ **/
+void
+testing_enable_deterministic_rng(void)
+{
+  static const uint8_t quotation[] =
+    "What will it be? A tree? A weed? "
+    "Each one is started from a seed."; // -- Mary Ann Hoberman
+  enable_deterministic_rng_impl(quotation, sizeof(quotation));
+}
+
+static uint8_t *prefilled_rng_buffer = NULL;
+static size_t prefilled_rng_buflen;
+static size_t prefilled_rng_idx;
+
+/**
+ * crypto_rand() replacement that returns canned data.
+ **/
+static void
+crypto_rand_prefilled(char *out, size_t n)
+{
+  tor_mutex_acquire(rng_mutex);
+  while (n) {
+    size_t n_to_copy = MIN(prefilled_rng_buflen - prefilled_rng_idx, n);
+    memcpy(out, prefilled_rng_buffer + prefilled_rng_idx, n_to_copy);
+    out += n_to_copy;
+    n -= n_to_copy;
+    prefilled_rng_idx += n_to_copy;
+
+    if (prefilled_rng_idx == prefilled_rng_buflen) {
+      prefilled_rng_idx = 0;
+    }
+  }
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant
+ * that yields output from a buffer.  If it reaches the end of the buffer, it
+ * starts over.
+ *
+ * Note: the get_thread_fast_rng() prng is not replaced by this; we'll need
+ * more code to support that.
+ **/
+void
+testing_enable_prefilled_rng(const void *buffer, size_t buflen)
+{
+  tor_assert(buflen > 0);
+  tor_assert(!rng_mutex);
+  rng_mutex = tor_mutex_new();
+
+  tor_mutex_acquire(rng_mutex);
+
+  prefilled_rng_buffer = tor_memdup(buffer, buflen);
+  prefilled_rng_buflen = buflen;
+  prefilled_rng_idx = 0;
+
+  tor_mutex_release(rng_mutex);
+
+  MOCK(crypto_rand, crypto_rand_prefilled);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+}
+
+/**
+ * Reset the position in the prefilled RNG buffer to the start.
+ */
+void
+testing_prefilled_rng_reset(void)
+{
+  tor_mutex_acquire(rng_mutex);
+  prefilled_rng_idx = 0;
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Undo the overrides for our PRNG.  To be used at the end of testing.
+ *
+ * Note that this function should be safe to call even if the rng has not
+ * yet been replaced.
+ **/
+void
+testing_disable_rng_override(void)
+{
+  crypto_xof_free(rng_xof);
+  tor_free(prefilled_rng_buffer);
+  UNMOCK(crypto_rand);
+  UNMOCK(crypto_strongest_rand_);
+  tor_mutex_free(rng_mutex);
+
+  crypto_fast_rng_t *rng = crypto_replace_thread_fast_rng(stored_fast_rng);
+  crypto_fast_rng_free(rng);
+
+  rng_is_replaced = false;
+}
+
+/**
+ * As testing_disable_rng_override(), but dump the seed if the current
+ * test has failed.
+ */
+void
+testing_disable_reproducible_rng(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    testing_dump_reproducible_rng_seed();
+  }
+  testing_disable_rng_override();
+}
diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h
new file mode 100644
index 0000000000..d7925148ae
--- /dev/null
+++ b/src/test/rng_test_helpers.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_RNG_TEST_HELPERS_H
+#define TOR_RNG_TEST_HELPERS_H
+
+#include "core/or/or.h"
+
+void testing_enable_deterministic_rng(void);
+void testing_enable_reproducible_rng(void);
+void testing_enable_prefilled_rng(const void *buffer, size_t buflen);
+
+void testing_prefilled_rng_reset(void);
+
+void testing_disable_rng_override(void);
+
+void testing_disable_reproducible_rng(void);
+#define testing_disable_deterministic_rng() \
+  testing_disable_rng_override()
+#define testing_disable_prefilled_rng() \
+  testing_disable_rng_override()
+
+void testing_dump_reproducible_rng_seed(void);
+
+#endif /* !defined(TOR_RNG_TEST_HELPERS_H) */
diff --git a/src/test/test-child.c b/src/test/test-child.c
deleted file mode 100644
index 11a1695cad..0000000000
--- a/src/test/test-child.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#include <stdio.h>
-#ifdef _WIN32
-#define WINDOWS_LEAN_AND_MEAN
-#include <windows.h>
-#else
-#include <unistd.h>
-#endif /* defined(_WIN32) */
-#include <string.h>
-
-#ifdef _WIN32
-#define SLEEP(sec) Sleep((sec)*1000)
-#else
-#define SLEEP(sec) sleep(sec)
-#endif
-
-/** Trivial test program which prints out its command line arguments so we can
- * check if tor_spawn_background() works */
-int
-main(int argc, char **argv)
-{
-  int i;
-  int delay = 1;
-  int fast = 0;
-
-  if (argc > 1)  {
-    if (!strcmp(argv[1], "--hang")) {
-      delay = 60;
-    } else if (!strcmp(argv[1], "--fast")) {
-      fast = 1;
-      delay = 0;
-    }
-  }
-
-  fprintf(stdout, "OUT\n");
-  fprintf(stderr, "ERR\n");
-  for (i = 1; i < argc; i++)
-    fprintf(stdout, "%s\n", argv[i]);
-  if (!fast)
-    fprintf(stdout, "SLEEPING\n");
-  /* We need to flush stdout so that test_util_spawn_background_partial_read()
-     succeed. Otherwise ReadFile() will get the entire output in one */
-  // XXX: Can we make stdio flush on newline?
-  fflush(stdout);
-  if (!fast)
-    SLEEP(1);
-  fprintf(stdout, "DONE\n");
-  fflush(stdout);
-  if (fast)
-    return 0;
-
-  while (--delay) {
-    SLEEP(1);
-  }
-
-  return 0;
-}
-
diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c
index 43754ed1c2..3f952e484f 100644
--- a/src/test/test-memwipe.c
+++ b/src/test/test-memwipe.c
@@ -49,7 +49,7 @@ const char *s = NULL;
  * us do bad things, such as access freed buffers, without crashing. */
 extern const char *malloc_options;
 const char *malloc_options = "sufjj";
-#endif
+#endif /* defined(OpenBSD) */
 
 static unsigned
 fill_a_buffer_memset(void)
diff --git a/src/test/test-network.sh b/src/test/test-network.sh
index b7a9f1b3c0..5ef995f1a4 100755
--- a/src/test/test-network.sh
+++ b/src/test/test-network.sh
@@ -5,7 +5,7 @@
 # If we already know CHUTNEY_PATH, don't bother with argument parsing
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     # we can't produce any output, because we might be --quiet
     # this preserves arguments with spaces correctly
     exec "$TEST_NETWORK" "$@"
@@ -16,34 +16,16 @@ fi
 # Do we output anything at all?
 ECHO="${ECHO:-echo}"
 # Output is prefixed with the name of the script
-myname=$(basename $0)
-
-# Save the arguments before we destroy them
-# This might not preserve arguments with spaces in them
-ORIGINAL_ARGS="$@"
+myname=$(basename "$0")
 
 # We need to find CHUTNEY_PATH, so that we can call the version of this script
 # in chutney/tools with the same arguments. We also need to respect --quiet.
-until [ -z "$1" ]
-do
-  case "$1" in
-    --chutney-path)
-      CHUTNEY_PATH="$2"
-      shift
-    ;;
-    --tor-path)
-      TOR_DIR="$2"
-      shift
-    ;;
-    --quiet)
-      ECHO=true
-    ;;
-    *)
-      # maybe chutney's test-network.sh can handle it
-    ;;
-  esac
-  shift
-done
+CHUTNEY_PATH=$(echo "$@" | awk -F '--chutney-path ' '{sub(" .*","",$2); print $2}')
+TOR_DIR=$(echo "$@" | awk -F '--tor-dir ' '{sub(" .*","",$2); print $2}')
+
+if echo "$@" | grep -e "--quiet" > /dev/null; then
+  ECHO=true
+fi
 
 # optional: $TOR_DIR is the tor build directory
 # it's used to find the location of tor binaries
@@ -52,12 +34,12 @@ done
 #  - if $PWD looks like a tor build directory, set it to $PWD, or
 #  - unset $TOR_DIR, and let chutney fall back to finding tor binaries in $PATH
 if [ ! -d "$TOR_DIR" ]; then
-    if [ -d "$BUILDDIR/src/core/or" -a -d "$BUILDDIR/src/tools" ]; then
+    if [ -d "$BUILDDIR/src/core/or" ] && [ -d "$BUILDDIR/src/tools" ]; then
         # Choose the build directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$BUILDDIR"
         TOR_DIR="$BUILDDIR"
-    elif [ -d "$PWD/src/core/or" -a -d "$PWD/src/tools" ]; then
+    elif [ -d "$PWD/src/core/or" ] && [ -d "$PWD/src/tools" ]; then
         # Guess the tor directory is the current directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$PWD"
@@ -73,12 +55,12 @@ fi
 #  - if $PWD looks like a chutney directory, set it to $PWD, or
 #  - set it based on $TOR_DIR, expecting chutney to be next to tor, or
 #  - fail and tell the user how to clone the chutney repository
-if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then
+if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then
     if [ -x "$PWD/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$PWD"
         CHUTNEY_PATH="$PWD"
-    elif [ -d "$TOR_DIR" -a -d "$TOR_DIR/../chutney" -a \
-           -x "$TOR_DIR/../chutney/chutney" ]; then
+    elif [ -d "$TOR_DIR" ] && [ -d "$TOR_DIR/../chutney" ] && \
+          [ -x "$TOR_DIR/../chutney/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney"
         CHUTNEY_PATH="$TOR_DIR/../chutney"
     else
@@ -94,12 +76,12 @@ fi
 
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     $ECHO "$myname: Calling newer chutney script $TEST_NETWORK"
     # this may fail if some arguments have spaces in them
     # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces
     # will be handled correctly
-    exec "$TEST_NETWORK" $ORIGINAL_ARGS
+    exec "$TEST_NETWORK" "$@"
 else
     $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH."
     $ECHO "$myname: Please update your chutney using 'git pull'."
diff --git a/src/test/test-process.c b/src/test/test-process.c
new file mode 100644
index 0000000000..eb28ad90e9
--- /dev/null
+++ b/src/test/test-process.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include <stdio.h>
+#ifdef _WIN32
+#define WINDOWS_LEAN_AND_MEAN
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif /* defined(_WIN32) */
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define SLEEP(sec) Sleep((sec)*1000)
+#else
+#define SLEEP(sec) sleep(sec)
+#endif
+
+/* Trivial test program to test process_t. */
+int
+main(int argc, char **argv)
+{
+  /* Does our process get the right arguments? */
+  for (int i = 0; i < argc; ++i) {
+    fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]);
+    fflush(stdout);
+  }
+
+  /* Make sure our process got our environment variable. */
+  fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n",
+          getenv("TOR_TEST_ENV"));
+  fflush(stdout);
+
+  /* Test line handling on stdout and stderr. */
+  fprintf(stdout, "Output on stdout\nThis is a new line\n");
+  fflush(stdout);
+
+  fprintf(stderr, "Output on stderr\nThis is a new line\n");
+  fflush(stderr);
+
+  fprintf(stdout, "Partial line on stdout ...");
+  fflush(stdout);
+
+  fprintf(stderr, "Partial line on stderr ...");
+  fflush(stderr);
+
+  SLEEP(2);
+
+  fprintf(stdout, "end of partial line on stdout\n");
+  fflush(stdout);
+  fprintf(stderr, "end of partial line on stderr\n");
+  fflush(stderr);
+
+  /* Echo input from stdin. */
+  char buffer[1024];
+
+  int count = 0;
+
+  while (fgets(buffer, sizeof(buffer), stdin)) {
+    /* Strip the newline. */
+    size_t size = strlen(buffer);
+
+    if (size >= 1 && buffer[size - 1] == '\n') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    if (size >= 1 && buffer[size - 1] == '\r') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    fprintf(stdout, "Read line from stdin: '%s'\n", buffer);
+    fflush(stdout);
+
+    if (++count == 3)
+      break;
+  }
+
+  fprintf(stdout, "We are done for here, thank you!\n");
+
+  return 0;
+}
diff --git a/src/test/test.c b/src/test/test.c
index 58b468775c..6dbec26fa8 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -12,6 +12,7 @@
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "app/config/or_state_st.h"
+#include "test/rng_test_helpers.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -37,7 +38,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "lib/compress/compress.h"
@@ -283,7 +284,7 @@ test_fast_handshake(void *arg)
   /* First, test an entire handshake. */
   memset(client_handshake, 0, sizeof(client_handshake));
   tt_int_op(0, OP_EQ, fast_onionskin_create(&state, client_handshake));
-  tt_assert(! tor_mem_is_zero((char*)client_handshake,
+  tt_assert(! fast_mem_is_zero((char*)client_handshake,
                               sizeof(client_handshake)));
 
   tt_int_op(0, OP_EQ,
@@ -354,18 +355,6 @@ test_onion_queues(void *arg)
   tor_free(onionskin);
 }
 
-static crypto_cipher_t *crypto_rand_aes_cipher = NULL;
-
-// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR
-// cipher in <b>crypto_rand_aes_cipher</b>.
-static void
-crypto_rand_deterministic_aes(char *out, size_t n)
-{
-  tor_assert(crypto_rand_aes_cipher);
-  memset(out, 0, n);
-  crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n);
-}
-
 static void
 test_circuit_timeout(void *arg)
 {
@@ -397,8 +386,7 @@ test_circuit_timeout(void *arg)
 
   // Use a deterministic RNG here, or else we'll get nondeterministic
   // coverage in some of the circuitstats functions.
-  MOCK(crypto_rand, crypto_rand_deterministic_aes);
-  crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover");
+  testing_enable_deterministic_rng();
 
   circuitbuild_running_unit_tests();
 #define timeout0 (build_time_t)(30*1000.0)
@@ -534,8 +522,8 @@ test_circuit_timeout(void *arg)
   circuit_build_times_free_timeouts(&final);
   or_state_free(state);
   teardown_periodic_events();
-  UNMOCK(crypto_rand);
-  crypto_cipher_free(crypto_rand_aes_cipher);
+
+  testing_disable_deterministic_rng();
 }
 
 /** Test encoding and parsing of rendezvous service descriptors. */
@@ -845,18 +833,23 @@ struct testgroup_t testgroups[] = {
   { "channeltls/", channeltls_tests },
   { "checkdir/", checkdir_tests },
   { "circuitbuild/", circuitbuild_tests },
+  { "circuitpadding/", circuitpadding_tests },
   { "circuitlist/", circuitlist_tests },
   { "circuitmux/", circuitmux_tests },
-  { "circuituse/", circuituse_tests },
   { "circuitstats/", circuitstats_tests },
+  { "circuituse/", circuituse_tests },
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
+  { "config/mgr/", confmgr_tests },
+  { "config/parse/", confparse_tests },
   { "connection/", connection_tests },
   { "conscache/", conscache_tests },
   { "consdiff/", consdiff_tests },
   { "consdiffmgr/", consdiffmgr_tests },
   { "container/", container_tests },
+  { "container/namemap/", namemap_tests },
   { "control/", controller_tests },
+  { "control/btrack/", btrack_tests },
   { "control/event/", controller_event_tests },
   { "crypto/", crypto_tests },
   { "crypto/ope/", crypto_ope_tests },
@@ -864,42 +857,54 @@ struct testgroup_t testgroups[] = {
   { "crypto/openssl/", crypto_openssl_tests },
 #endif
   { "crypto/pem/", pem_tests },
+  { "crypto/rng/", crypto_rng_tests },
   { "dir/", dir_tests },
-  { "dir_handle_get/", dir_handle_get_tests },
+  { "dir/auth/process_descs/", process_descs_tests },
   { "dir/md/", microdesc_tests },
-  { "dir/voting-schedule/", voting_schedule_tests },
+  { "dir/voting/flags/", voting_flags_tests },
+  { "dir/voting/schedule/", voting_schedule_tests },
+  { "dir_handle_get/", dir_handle_get_tests },
+  { "dispatch/", dispatch_tests, },
+  { "dns/", dns_tests },
   { "dos/", dos_tests },
   { "entryconn/", entryconn_tests },
   { "entrynodes/", entrynodes_tests },
-  { "guardfraction/", guardfraction_tests },
   { "extorport/", extorport_tests },
   { "geoip/", geoip_tests },
-  { "legacy_hs/", hs_tests },
+  { "guardfraction/", guardfraction_tests },
   { "hs_cache/", hs_cache },
   { "hs_cell/", hs_cell_tests },
+  { "hs_client/", hs_client_tests },
   { "hs_common/", hs_common_tests },
   { "hs_config/", hs_config_tests },
   { "hs_control/", hs_control_tests },
   { "hs_descriptor/", hs_descriptor },
+  { "hs_dos/", hs_dos_tests },
+  { "hs_intropoint/", hs_intropoint_tests },
   { "hs_ntor/", hs_ntor_tests },
   { "hs_service/", hs_service_tests },
-  { "hs_client/", hs_client_tests },
-  { "hs_intropoint/", hs_intropoint_tests },
   { "introduce/", introduce_tests },
   { "keypin/", keypin_tests },
+  { "legacy_hs/", hs_tests },
   { "link-handshake/", link_handshake_tests },
   { "mainloop/", mainloop_tests },
+  { "netinfo/", netinfo_tests },
   { "nodelist/", nodelist_tests },
   { "oom/", oom_tests },
   { "oos/", oos_tests },
   { "options/", options_tests },
+  { "parsecommon/", parsecommon_tests },
   { "periodic-event/" , periodic_event_tests },
   { "policy/" , policy_tests },
+  { "prob_distr/", prob_distr_tests },
   { "procmon/", procmon_tests },
+  { "process/", process_tests },
   { "proto/http/", proto_http_tests },
   { "proto/misc/", proto_misc_tests },
   { "protover/", protover_tests },
   { "pt/", pt_tests },
+  { "pubsub/build/", pubsub_build_tests },
+  { "pubsub/msg/", pubsub_msg_tests },
   { "relay/" , relay_tests },
   { "relaycell/", relaycell_tests },
   { "relaycrypt/", relaycrypt_tests },
@@ -910,10 +915,12 @@ struct testgroup_t testgroups[] = {
   { "routerlist/", routerlist_tests },
   { "routerset/" , routerset_tests },
   { "scheduler/", scheduler_tests },
-  { "socks/", socks_tests },
+  { "sendme/", sendme_tests },
   { "shared-random/", sr_tests },
+  { "socks/", socks_tests },
   { "status/" , status_tests },
   { "storagedir/", storagedir_tests },
+  { "token_bucket/", token_bucket_tests },
   { "tortls/", tortls_tests },
 #ifndef ENABLE_NSS
   { "tortls/openssl/", tortls_openssl_tests },
@@ -921,10 +928,9 @@ struct testgroup_t testgroups[] = {
   { "tortls/x509/", x509_tests },
   { "util/", util_tests },
   { "util/format/", util_format_tests },
+  { "util/handle/", handle_tests },
   { "util/logging/", logging_tests },
   { "util/process/", util_process_tests },
   { "util/thread/", thread_tests },
-  { "util/handle/", handle_tests },
-  { "dns/", dns_tests },
   END_OF_GROUPS
 };
diff --git a/src/test/test.h b/src/test/test.h
index aacc9dba87..76c4c0ec75 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -177,71 +177,89 @@ extern const struct testcase_setup_t ed25519_test_setup;
 
 extern struct testcase_t accounting_tests[];
 extern struct testcase_t addr_tests[];
-extern struct testcase_t address_tests[];
 extern struct testcase_t address_set_tests[];
+extern struct testcase_t address_tests[];
 extern struct testcase_t bridges_tests[];
-extern struct testcase_t bwmgt_tests[];
+extern struct testcase_t btrack_tests[];
 extern struct testcase_t buffer_tests[];
+extern struct testcase_t bwmgt_tests[];
 extern struct testcase_t cell_format_tests[];
 extern struct testcase_t cell_queue_tests[];
 extern struct testcase_t channel_tests[];
 extern struct testcase_t channelpadding_tests[];
+extern struct testcase_t circuitpadding_tests[];
 extern struct testcase_t channeltls_tests[];
 extern struct testcase_t checkdir_tests[];
 extern struct testcase_t circuitbuild_tests[];
 extern struct testcase_t circuitlist_tests[];
 extern struct testcase_t circuitmux_tests[];
-extern struct testcase_t circuituse_tests[];
 extern struct testcase_t circuitstats_tests[];
+extern struct testcase_t circuituse_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
+extern struct testcase_t confmgr_tests[];
+extern struct testcase_t confparse_tests[];
 extern struct testcase_t connection_tests[];
 extern struct testcase_t conscache_tests[];
 extern struct testcase_t consdiff_tests[];
 extern struct testcase_t consdiffmgr_tests[];
 extern struct testcase_t container_tests[];
-extern struct testcase_t controller_tests[];
 extern struct testcase_t controller_event_tests[];
-extern struct testcase_t crypto_tests[];
+extern struct testcase_t controller_tests[];
 extern struct testcase_t crypto_ope_tests[];
 extern struct testcase_t crypto_openssl_tests[];
-extern struct testcase_t dir_tests[];
+extern struct testcase_t crypto_rng_tests[];
+extern struct testcase_t crypto_tests[];
 extern struct testcase_t dir_handle_get_tests[];
+extern struct testcase_t dir_tests[];
+extern struct testcase_t dispatch_tests[];
+extern struct testcase_t dns_tests[];
 extern struct testcase_t dos_tests[];
 extern struct testcase_t entryconn_tests[];
 extern struct testcase_t entrynodes_tests[];
-extern struct testcase_t guardfraction_tests[];
 extern struct testcase_t extorport_tests[];
 extern struct testcase_t geoip_tests[];
-extern struct testcase_t hs_tests[];
+extern struct testcase_t guardfraction_tests[];
+extern struct testcase_t handle_tests[];
 extern struct testcase_t hs_cache[];
 extern struct testcase_t hs_cell_tests[];
+extern struct testcase_t hs_client_tests[];
 extern struct testcase_t hs_common_tests[];
 extern struct testcase_t hs_config_tests[];
 extern struct testcase_t hs_control_tests[];
 extern struct testcase_t hs_descriptor[];
+extern struct testcase_t hs_dos_tests[];
+extern struct testcase_t hs_intropoint_tests[];
 extern struct testcase_t hs_ntor_tests[];
 extern struct testcase_t hs_service_tests[];
-extern struct testcase_t hs_client_tests[];
-extern struct testcase_t hs_intropoint_tests[];
+extern struct testcase_t hs_tests[];
 extern struct testcase_t introduce_tests[];
 extern struct testcase_t keypin_tests[];
 extern struct testcase_t link_handshake_tests[];
 extern struct testcase_t logging_tests[];
 extern struct testcase_t mainloop_tests[];
 extern struct testcase_t microdesc_tests[];
+extern struct testcase_t namemap_tests[];
+extern struct testcase_t netinfo_tests[];
 extern struct testcase_t nodelist_tests[];
 extern struct testcase_t oom_tests[];
 extern struct testcase_t oos_tests[];
 extern struct testcase_t options_tests[];
+extern struct testcase_t parsecommon_tests[];
 extern struct testcase_t pem_tests[];
 extern struct testcase_t periodic_event_tests[];
 extern struct testcase_t policy_tests[];
+extern struct testcase_t prob_distr_tests[];
+extern struct testcase_t slow_stochastic_prob_distr_tests[];
 extern struct testcase_t procmon_tests[];
+extern struct testcase_t process_tests[];
+extern struct testcase_t process_descs_tests[];
 extern struct testcase_t proto_http_tests[];
 extern struct testcase_t proto_misc_tests[];
 extern struct testcase_t protover_tests[];
 extern struct testcase_t pt_tests[];
+extern struct testcase_t pubsub_build_tests[];
+extern struct testcase_t pubsub_msg_tests[];
 extern struct testcase_t relay_tests[];
 extern struct testcase_t relaycell_tests[];
 extern struct testcase_t relaycrypt_tests[];
@@ -252,23 +270,25 @@ extern struct testcase_t routerkeys_tests[];
 extern struct testcase_t routerlist_tests[];
 extern struct testcase_t routerset_tests[];
 extern struct testcase_t scheduler_tests[];
-extern struct testcase_t storagedir_tests[];
+extern struct testcase_t sendme_tests[];
 extern struct testcase_t socks_tests[];
+extern struct testcase_t sr_tests[];
 extern struct testcase_t status_tests[];
+extern struct testcase_t storagedir_tests[];
 extern struct testcase_t thread_tests[];
-extern struct testcase_t tortls_tests[];
+extern struct testcase_t token_bucket_tests[];
 extern struct testcase_t tortls_openssl_tests[];
-extern struct testcase_t util_tests[];
+extern struct testcase_t tortls_tests[];
 extern struct testcase_t util_format_tests[];
 extern struct testcase_t util_process_tests[];
+extern struct testcase_t util_tests[];
+extern struct testcase_t voting_flags_tests[];
 extern struct testcase_t voting_schedule_tests[];
-extern struct testcase_t dns_tests[];
-extern struct testcase_t handle_tests[];
-extern struct testcase_t sr_tests[];
 extern struct testcase_t x509_tests[];
 
 extern struct testcase_t slow_crypto_tests[];
-extern struct testcase_t slow_util_tests[];
+extern struct testcase_t slow_process_tests[];
+extern struct testcase_t slow_ptr_tests[];
 
 extern struct testgroup_t testgroups[];
 
diff --git a/src/test/test_addr.c b/src/test/test_addr.c
index 8868edce25..c89c6e78d4 100644
--- a/src/test/test_addr.c
+++ b/src/test/test_addr.c
@@ -11,6 +11,8 @@
 #include "feature/client/addressmap.h"
 #include "test/log_test_helpers.h"
 #include "lib/net/resolve.h"
+#include "test/rng_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #ifdef HAVE_SYS_UN_H
 #include <sys/un.h>
@@ -239,7 +241,7 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(0,OP_EQ, tor_addr_lookup("9000::5", AF_UNSPEC, &t1));
   tt_int_op(AF_INET6,OP_EQ, tor_addr_family(&t1));
   tt_int_op(0x90,OP_EQ, tor_addr_to_in6_addr8(&t1)[0]);
-  tt_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
+  tt_assert(fast_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
   tt_int_op(0x05,OP_EQ, tor_addr_to_in6_addr8(&t1)[15]);
 
   /* === Test pton: valid af_inet6 */
@@ -696,7 +698,7 @@ test_addr_ip6_helpers(void *arg)
                               &t1,&mask,&port1,&port2);
   tt_int_op(r,OP_EQ,AF_INET6);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET6);
-  tt_assert(tor_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
+  tt_assert(fast_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
   tt_int_op(mask,OP_EQ,0);
   tt_int_op(port1,OP_EQ,1);
   tt_int_op(port2,OP_EQ,65535);
@@ -723,104 +725,559 @@ test_addr_ip6_helpers(void *arg)
   ;
 }
 
-/** Test tor_addr_port_parse(). */
+/* Test that addr_str successfully parses, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_PARSE_FMT(addr_str, expect_family, fmt_decorated, \
+                            expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that addr_str fails to parse, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_PARSE_XFAIL(addr_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str and default_port successfully parse, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, expect_family, \
+                                 fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that addr_port_str and default_port fail to parse, and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_PARSE_XFAIL(addr_port_str, default_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as an IPv4 address using
+ * tor_lookup_hostname(), and:
+ *  - the fmt_addr32() of the result is expect_str.
+ */
+#define TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, expect_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(addr_str, &addr32h); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_str_op(fmt_addr32(addr32h), OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse using tor_lookup_hostname(), with a
+ * permanent failure, and:
+ *  - the returned address is 0.
+ */
+#define TEST_ADDR_V4_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(bad_str, &addr32h); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_str as an IPv4 address using tor_lookup_hostname()
+ * does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is 0.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_V4_LOOKUP(host_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(host_str, &addr32h); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a require_family IP address using
+ * tor_addr_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_LOOKUP_FMT(addr_str, require_family, expect_family, \
+                             fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(addr_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse as a require_family IP address using
+ * tor_addr_lookup(), with a permanent failure, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_LOOKUP_XFAIL(bad_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(bad_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that looking up host_string as a require_family IP address using
+ * tor_addr_lookup(),  does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is null.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_LOOKUP(host_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(host_str, require_family, &addr); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, expect_family, \
+                                  fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(addr_port_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that bad_str fails to parse as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(bad_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_port_str as an IP address using
+ * tor_addr_port_lookup(),  does something sensible:
+ *  - the result is -1 or 0.
+ *  - if the result is a failure, the returned address is null, and the
+ *    returned port is zero,
+ *  - if the result is a success, the returned port is expect_success_port,
+ *    and the returned family is AF_INET or AF_INET6.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(host_port_str, &addr, &port); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 0); \
+    if (r == -1) { \
+      tt_assert(tor_addr_is_null(&addr)); \
+      tt_int_op(port, OP_EQ, 0); \
+    } else { \
+      tt_assert(tor_addr_family(&addr) == AF_INET || \
+                tor_addr_family(&addr) == AF_INET6); \
+      tt_int_op(port, OP_EQ, expect_success_port); \
+    } \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical IPv4 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup() with AF_INET,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_port_lookup(), because there is no port.
+ */
+#define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \
+                             addr_str, 111); \
+    TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET, 0, addr_str, 0); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET, AF_INET, 0, addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical fmt_decorated
+ * IPv6 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \
+                             addr_str, 222); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, addr_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_str successfully parses, and the fmt_decorated canonical
+ * IPv6 string is expect_str.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port.
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \
+                             expect_str, 333); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, expect_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical IPv4 address
+ * string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port.
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical undecorated
+ * IPv6 address string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port, and because it only
+ *    supports IPv4,
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET6, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+  STMT_END
+
+/* Test that bad_str fails to parse due to a bad address or port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup().
+ */
+#define TEST_ADDR_PARSE_XFAIL_MALFORMED(bad_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_XFAIL(bad_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str, 666); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(bad_str); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET6); \
+    TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str); \
+  STMT_END
+
+/* Test that host_str is treated as a hostname, and not an address.
+ * Check for success or failure using the network-dependent functions:
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup(), expecting a zero port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port.
+ */
+#define TEST_HOSTNAME(host_str) \
+  STMT_BEGIN \
+    TEST_HOST_V4_LOOKUP(host_str); \
+    TEST_HOST_LOOKUP(host_str, AF_UNSPEC); \
+    TEST_HOST_LOOKUP(host_str, AF_INET); \
+    TEST_HOST_LOOKUP(host_str, AF_INET6); \
+    TEST_HOST_PORT_LOOKUP(host_str, 0); \
+    TEST_ADDR_PARSE_XFAIL(host_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \
+  STMT_END
+
+/* Test that host_port_str is treated as a hostname and port, and not a
+ * hostname or an address.
+ * Check for success or failure using the network-dependent function:
+ *  - tor_addr_port_lookup(), expecting expect_success_port if the lookup is
+ *    successful.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(), because it doesn't support ports,
+ *  - tor_addr_lookup(), regardless of the address family, because it doesn't
+ *    support ports.
+ */
+#define TEST_HOSTNAME_PORT(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port); \
+    TEST_ADDR_PARSE_XFAIL(host_port_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET6); \
+  STMT_END
+
+static void
+test_addr_parse_canonical(void *arg)
+{
+  int r;
+  tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
+  char buf[TOR_ADDR_BUF_LEN];
+
+  (void)arg;
+
+  /* Correct calls. */
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.1");
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2");
+
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::33:44]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[2::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1);
+
+  /* Allow IPv6 without square brackets, when there is no port, but only if
+   * there is a default port */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::1", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("2::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0);
+ done:
+  ;
+}
+
+/** Test tor_addr_parse() and tor_addr_port_parse(). */
 static void
 test_addr_parse(void *arg)
 {
+
   int r;
   tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
   char buf[TOR_ADDR_BUF_LEN];
-  uint16_t port = 0;
 
-  /* Correct call. */
   (void)arg;
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.1:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "192.0.2.1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "::1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  /* Domain name. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
 
-  /* Only IP. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  mock_hostname_resolver();
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,200);
+  /* IPv6-mapped IPv4 addresses. Tor doesn't really use these. */
+  TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0,
+                     "11:22:33:44:55:66:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_ADDR_V6_PARSE("11:22::33:44:1.2.3.4", 0,
+                     "11:22::33:44:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, 400);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,400);
+  /* Ports. */
+  TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234);
+  TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234);
 
-  /* Bad port. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  /* Host names. */
+  TEST_HOSTNAME("localhost");
+  TEST_HOSTNAME_PORT("localhost:1234", 1234);
+  TEST_HOSTNAME_PORT("localhost:0", 0);
 
-  /* Only domain name */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("torproject.org");
+  TEST_HOSTNAME_PORT("torproject.org:56", 56);
 
-  /* Bad IP address */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("probably-not-a-valid-dns.name-tld");
+  TEST_HOSTNAME_PORT("probably-not-a-valid-dns.name-tld:789", 789);
+
+  /* Malformed addresses. */
+  /* Empty string. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("");
+
+  /* Square brackets around IPv4 address. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.1]");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345");
 
-  /* Make sure that the default port has lower priority than the real
-     one */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:1337",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1337);
+  /* Only left square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[11:22::33:44");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1369",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1369);
+  /* Only right square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44]");
+
+  /* Leading colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED(":11:22::33:44");
+
+  /* Trailing colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:");
+
+  /* Bad port. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:77777");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:88888");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:99999");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:-2");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:-3");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:-4");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:1 bad");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:bad-port-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port-1");
+
+  /* Bad hostname */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid:22222");
+
+  /* Ambiguous cases */
+  /* Too many hex words in IPv4-mapped IPv6 address.
+   * But some OS host lookup routines accept it as a hostname, or
+   * as an IP address?? (I assume they discard unused characters). */
+  TEST_HOSTNAME("11:22:33:44:55:66:77:88:1.2.3.4");
+
+  /* IPv6 address with port and no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address:port ? */
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+  /* Is it a port, or are there too many hex words?
+   * We reject it either way, but some OS host lookup routines accept it as an
+   * IPv6 address:port */
+  TEST_HOSTNAME_PORT("11:22:33:44:55:66:77:88:99", 99);
+  /* But we accept it if it has square brackets. */
+  TEST_ADDR_V6_PORT_PARSE("[11:22:33:44:55:66:77:88]:99",
+                           "11:22:33:44:55:66:77:88",99);
+
+  /* Bad IPv4 address
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv4 address[:port], with a zero last octet */
+  TEST_HOSTNAME("192.0.1");
+  TEST_HOSTNAME_PORT("192.0.2:1234", 1234);
+
+  /* More bad IPv6 addresses and ports: no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address[:port] */
+  TEST_HOSTNAME_PORT("::1:12345", 12345);
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+
+  /* And this is an ambiguous case, which is interpreted as an IPv6 address. */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::88:99", 0);
+  /* Use square brackets to resolve the ambiguity */
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::88:99]", 1);
+  TEST_ADDR_V6_PORT_PARSE("[11:22::88]:99",
+                           "11:22::88",99);
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -891,27 +1348,6 @@ test_virtaddrmap(void *data)
   ;
 }
 
-static const char *canned_data = NULL;
-static size_t canned_data_len = 0;
-
-/* Mock replacement for crypto_rand() that returns canned data from
- * canned_data above. */
-static void
-crypto_canned(char *ptr, size_t n)
-{
-  if (canned_data_len) {
-    size_t to_copy = MIN(n, canned_data_len);
-    memcpy(ptr, canned_data, to_copy);
-    canned_data += to_copy;
-    canned_data_len -= to_copy;
-    n -= to_copy;
-    ptr += to_copy;
-  }
-  if (n) {
-    crypto_rand_unmocked(ptr, n);
-  }
-}
-
 static void
 test_virtaddrmap_persist(void *data)
 {
@@ -919,6 +1355,8 @@ test_virtaddrmap_persist(void *data)
   const char *a, *b, *c;
   tor_addr_t addr;
   char *ones = NULL;
+  const char *canned_data;
+  size_t canned_data_len;
 
   addressmap_init();
 
@@ -937,7 +1375,7 @@ test_virtaddrmap_persist(void *data)
                 "1234567890" // the second call returns this.
                 "abcdefghij"; // the third call returns this.
   canned_data_len = 30;
-  MOCK(crypto_rand, crypto_canned);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
 
   a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
                                           tor_strdup("quuxit.baz"));
@@ -947,9 +1385,9 @@ test_virtaddrmap_persist(void *data)
   tt_assert(b);
   tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual");
   tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual");
+  testing_disable_prefilled_rng();
 
   // Now try something to get us an ipv4 address
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16",
                                                 AF_INET, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -966,22 +1404,23 @@ test_virtaddrmap_persist(void *data)
 
   // Try some canned entropy and verify all the we discard duplicates,
   // addresses that end with 0, and addresses that end with 255.
-  MOCK(crypto_rand, crypto_canned);
   canned_data = "\x01\x02\x03\x04" // okay
                 "\x01\x02\x03\x04" // duplicate
                 "\x03\x04\x00\x00" // bad ending 1
                 "\x05\x05\x00\xff" // bad ending 2
                 "\x05\x06\x07\xf0"; // okay
   canned_data_len = 20;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumble.onion"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumpus.onion"));
   tt_str_op(a, OP_EQ, "192.168.3.4");
   tt_str_op(b, OP_EQ, "192.168.7.240");
+  testing_disable_prefilled_rng();
 
   // Now try IPv6!
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20",
                                                 AF_INET6, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -997,7 +1436,7 @@ test_virtaddrmap_persist(void *data)
   tt_assert(!strcmpstart(b, "[1010:f"));
 
   // Try IPv6 with canned entropy, to make sure we detect duplicates.
-  MOCK(crypto_rand, crypto_canned);
+
   canned_data = "acanthopterygian" // okay
                 "cinematographist" // okay
                 "acanthopterygian" // duplicate
@@ -1006,6 +1445,8 @@ test_virtaddrmap_persist(void *data)
                 "cinematographist" // duplicate
                 "coadministration"; // okay
   canned_data_len = 16 * 7;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
                                           tor_strdup("wuffle.baz"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -1018,9 +1459,11 @@ test_virtaddrmap_persist(void *data)
 
   // Try address exhaustion: make sure we can actually fail if we
   // get too many already-existing addresses.
+  testing_disable_prefilled_rng();
   canned_data_len = 128*1024;
   canned_data = ones = tor_malloc(canned_data_len);
   memset(ones, 1, canned_data_len);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
   // There is some chance this one will fail if a previous random
   // allocation gave out the address already.
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -1037,7 +1480,7 @@ test_virtaddrmap_persist(void *data)
   expect_single_log_msg_containing("Ran out of virtual addresses!");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(ones);
   addressmap_free_all();
   teardown_capture_of_logs();
@@ -1213,6 +1656,7 @@ struct testcase_t addr_tests[] = {
   ADDR_LEGACY(basic),
   ADDR_LEGACY(ip6_helpers),
   ADDR_LEGACY(parse),
+  ADDR_LEGACY(parse_canonical),
   { "virtaddr", test_virtaddrmap, 0, NULL, NULL },
   { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL },
   { "localname", test_addr_localname, 0, NULL, NULL },
diff --git a/src/test/test_address.c b/src/test/test_address.c
index c33c30aee5..32fb4aa232 100644
--- a/src/test/test_address.c
+++ b/src/test/test_address.c
@@ -24,6 +24,10 @@
 #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/node_st.h"
 #include "feature/nodelist/nodelist.h"
 #include "lib/net/address.h"
 #include "test/test.h"
@@ -1170,8 +1174,167 @@ test_address_tor_addr_in_same_network_family(void *ignored)
   return;
 }
 
+static node_t *
+helper_create_mock_node(char id_char)
+{
+  node_t *node = tor_malloc_zero(sizeof(node_t));
+  routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t));
+  tor_addr_make_null(&ri->ipv6_addr, AF_INET6);
+  node->ri = ri;
+  memset(node->identity, id_char, sizeof(node->identity));
+  return node;
+}
+
+static void
+helper_free_mock_node(node_t *node)
+{
+  if (!node)
+    return;
+  tor_free(node->ri);
+  tor_free(node);
+}
+
+#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, ipv4_addr); \
+    node->ri->addr = tor_addr_to_ipv4h(&addr); \
+    node->ri->or_port = ipv4_port; \
+  }
+
+#define NODE_CLEAR_IPV4(node) { \
+    node->ri->addr = 0; \
+    node->ri->or_port = 0; \
+  }
+
+#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \
+    tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \
+    node->ri->ipv6_orport = ipv6_port; \
+  }
+
+static void
+test_address_tor_node_in_same_network_family(void *ignored)
+{
+  (void)ignored;
+  node_t *node_a = helper_create_mock_node('a');
+  node_t *node_b = helper_create_mock_node('b');
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "8.8.4.4", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "1.1.1.1", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_a);
+  NODE_SET_IPV6(node_a, "2001:470:20::2", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_b);
+  NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1);
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+ done:
+  helper_free_mock_node(node_a);
+  helper_free_mock_node(node_b);
+}
+
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+/* Test dirserv_router_has_valid_address() on a stub routerinfo, with only its
+ * address fields set. Use IPv4 ipv4_addr_str and IPv6 ipv6_addr_str.
+ * Fail if it does not return rv. */
+#define TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, ipv6_addr_str, rv) \
+  STMT_BEGIN \
+    ri = tor_malloc_zero(sizeof(routerinfo_t)); \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, (ipv4_addr_str));   \
+    ri->addr = tor_addr_to_ipv4h(&addr); \
+    tor_addr_parse(&ri->ipv6_addr, (ipv6_addr_str)); \
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, (rv)); \
+    tor_free(ri); \
+  STMT_END
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a null
+ * IPv6 address. */
+#define CHECK_RI_ADDR(ipv4_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, "::", rv)
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a non-internal
+ * IPv4 address, so that the IPv6 check is reached. */
+#define CHECK_RI_ADDR6(ipv6_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER("1.0.0.1", ipv6_addr_str, rv)
+
+static void
+test_address_dirserv_router_addr_private(void *opt_dir_allow_private)
+{
+  /* A stub routerinfo structure, with only its address fields set. */
+  routerinfo_t *ri = NULL;
+  /* The expected return value for private addresses.
+   * Modified if DirAllowPrivateAddresses is 1. */
+  int private_rv = -1;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(get_options, mock_get_options);
+
+  if (opt_dir_allow_private) {
+    mock_options.DirAllowPrivateAddresses = 1;
+    private_rv = 0;
+  }
+
+  CHECK_RI_ADDR("1.0.0.1", 0);
+  CHECK_RI_ADDR("10.0.0.1", private_rv);
+
+  CHECK_RI_ADDR6("2600::1", 0);
+  CHECK_RI_ADDR6("fe80::1", private_rv);
+
+  /* Null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  CHECK_RI_ADDR("0.0.0.0", private_rv);
+  TEST_ROUTER_VALID_ADDRESS_HELPER("0.0.0.0", "::", private_rv);
+
+  /* IPv6 null succeeds, because IPv4 is not null */
+  CHECK_RI_ADDR6("::", 0);
+
+  /* Byte-zeroed null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, private_rv);
+    tor_free(ri);
+  }
+
+  /* IPv6 null succeeds, because IPv4 is not internal */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    ri->addr = 16777217; /* 1.0.0.1 */
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, 0);
+    tor_free(ri);
+  }
+
+ done:
+  tor_free(ri);
+  UNMOCK(get_options);
+}
+
 #define ADDRESS_TEST(name, flags) \
   { #name, test_address_ ## name, flags, NULL, NULL }
+#define ADDRESS_TEST_STR_ARG(name, flags, str_arg) \
+  { #name "/" str_arg, test_address_ ## name, flags, &passthrough_setup, \
+    (void *)(str_arg) }
 
 struct testcase_t address_tests[] = {
   ADDRESS_TEST(udp_socket_trick_whitebox, TT_FORK),
@@ -1202,5 +1365,8 @@ struct testcase_t address_tests[] = {
   ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0),
   ADDRESS_TEST(tor_addr_eq_ipv4h, 0),
   ADDRESS_TEST(tor_addr_in_same_network_family, 0),
+  ADDRESS_TEST(tor_node_in_same_network_family, 0),
+  ADDRESS_TEST(dirserv_router_addr_private, 0),
+  ADDRESS_TEST_STR_ARG(dirserv_router_addr_private, 0, "allow_private"),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c
index fb8408b3c3..6e299d779e 100644
--- a/src/test/test_address_set.c
+++ b/src/test/test_address_set.c
@@ -4,6 +4,7 @@
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/address_set.h"
+#include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -31,6 +32,12 @@ mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
   return dummy_ns;
 }
 
+static void
+mock_dirlist_add_trusted_dir_addresses(void)
+{
+  return;
+}
+
 /* Number of address a single node_t can have. Default to the production
  * value. This is to control the size of the bloom filter. */
 static int addr_per_node = 2;
@@ -98,6 +105,8 @@ test_nodelist(void *arg)
        mock_networkstatus_get_latest_consensus_by_flavor);
   MOCK(get_estimated_address_per_node,
        mock_get_estimated_address_per_node);
+  MOCK(dirlist_add_trusted_dir_addresses,
+       mock_dirlist_add_trusted_dir_addresses);
 
   dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
   dummy_ns->flavor = FLAV_MICRODESC;
@@ -113,7 +122,10 @@ test_nodelist(void *arg)
    * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
   addr_per_node = 1024;
 
-  /* No node no nothing. The lookups should be empty. */
+  /* No node no nothing. The lookups should be empty. We've mocked the
+   * dirlist_add_trusted_dir_addresses in order for _no_ authorities to be
+   * added to the filter else it makes this test to trigger many false
+   * positive. */
   nodelist_set_consensus(dummy_ns);
 
   /* The address set should be empty. */
@@ -167,6 +179,7 @@ test_nodelist(void *arg)
   UNMOCK(networkstatus_get_latest_consensus);
   UNMOCK(networkstatus_get_latest_consensus_by_flavor);
   UNMOCK(get_estimated_address_per_node);
+  UNMOCK(dirlist_add_trusted_dir_addresses);
 }
 
 struct testcase_t address_set_tests[] = {
diff --git a/src/test/test_bt.sh b/src/test/test_bt.sh
index df8bcb8eda..312905a4e2 100755
--- a/src/test/test_bt.sh
+++ b/src/test/test_bt.sh
@@ -3,8 +3,6 @@
 
 exitcode=0
 
-ulimit -c 0
-
 export ASAN_OPTIONS="handle_segv=0:allow_user_segv_handler=1"
 "${builddir:-.}/src/test/test-bt-cl" backtraces || exit $?
 "${builddir:-.}/src/test/test-bt-cl" assert 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?"
diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c
index 0c15a02ee4..b29c2c6cbc 100644
--- a/src/test/test_bt_cl.c
+++ b/src/test/test_bt_cl.c
@@ -4,6 +4,9 @@
 #include "orconfig.h"
 #include <stdio.h>
 #include <stdlib.h>
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
 
 /* To prevent 'assert' from going away. */
 #undef TOR_COVERAGE
@@ -43,7 +46,7 @@ crash(int x)
     *(volatile int *)0 = 0;
 #endif /* defined(__clang_analyzer__) || defined(__COVERITY__) */
   } else if (crashtype == 1) {
-    tor_assert(1 == 0);
+    tor_assertf(1 == 0, "%d != %d", 1, 0);
   } else if (crashtype == -1) {
     ;
   }
@@ -88,6 +91,11 @@ main(int argc, char **argv)
     return 1;
   }
 
+#ifdef HAVE_SYS_RESOURCE_H
+  struct rlimit rlim = { .rlim_cur = 0, .rlim_max = 0 };
+  setrlimit(RLIMIT_CORE, &rlim);
+#endif
+
 #if !(defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
    defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION))
     puts("Backtrace reporting is not supported on this platform");
diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c
new file mode 100644
index 0000000000..80da7829ae
--- /dev/null
+++ b/src/test/test_btrack.c
@@ -0,0 +1,129 @@
+/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+
+static void
+send_state(const orconn_state_msg_t *msg_in)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_state_publish(msg);
+}
+
+static void
+send_status(const orconn_status_msg_t *msg_in)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_status_publish(msg);
+}
+
+static void
+send_chan(const ocirc_chan_msg_t *msg_in)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_btrack_launch(void *arg)
+{
+  orconn_state_msg_t conn;
+  ocirc_chan_msg_t circ;
+  memset(&conn, 0, sizeof(conn));
+  memset(&circ, 0, sizeof(circ));
+
+  (void)arg;
+  conn.gid = 1;
+  conn.chan = 1;
+  conn.proxy_type = PROXY_NONE;
+  conn.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 1;
+  circ.onehop = true;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1");
+  expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1");
+  teardown_capture_of_logs();
+
+  conn.gid = 2;
+  conn.chan = 2;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 2;
+  circ.onehop = false;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0");
+  expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+static void
+test_btrack_delete(void *arg)
+{
+  orconn_state_msg_t state;
+  orconn_status_msg_t status;
+  memset(&state, 0, sizeof(state));
+  memset(&status, 0, sizeof(status));
+
+  (void)arg;
+  state.gid = 1;
+  state.chan = 1;
+  state.proxy_type = PROXY_NONE;
+  state.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&state);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0");
+  teardown_capture_of_logs();
+
+  status.gid = 1;
+  status.status = OR_CONN_EVENT_CLOSED;
+  status.reason = 0;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_status(&status);
+  expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+struct testcase_t btrack_tests[] = {
+  { "launch", test_btrack_launch, TT_FORK, &helper_pubsub_setup, NULL },
+  { "delete", test_btrack_delete, TT_FORK, &helper_pubsub_setup, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c
index 3e7364a5c8..3084c19d74 100644
--- a/src/test/test_buffers.c
+++ b/src/test/test_buffers.c
@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/tls/tortls.h"
 #include "lib/compress/compress.h"
diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c
index 5a013aa268..e6f028ed74 100644
--- a/src/test/test_bwmgt.c
+++ b/src/test/test_bwmgt.c
@@ -6,18 +6,67 @@
  * \brief tests for bandwidth management / token bucket functions
  */
 
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
 #define TOKEN_BUCKET_PRIVATE
 
 #include "core/or/or.h"
-#include "test/test.h"
 
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/evloop/token_bucket.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+
+#include "app/config/or_options_st.h"
+#include "core/or/connection_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 // an imaginary time, in timestamp units. Chosen so it will roll over.
 static const uint32_t START_TS = UINT32_MAX-10;
 static const int32_t KB = 1024;
 static const uint32_t GB = (UINT64_C(1) << 30);
 
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+static networkstatus_t *dummy_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return dummy_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  tor_assert(f == FLAV_MICRODESC);
+  return dummy_ns;
+}
+
+/* Number of address a single node_t can have. Default to the production
+ * value. This is to control the size of the bloom filter. */
+static int addr_per_node = 2;
+static int
+mock_get_estimated_address_per_node(void)
+{
+  return addr_per_node;
+}
+
 static void
 test_bwmgt_token_buf_init(void *arg)
 {
@@ -220,8 +269,162 @@ test_bwmgt_token_buf_helpers(void *arg)
   ;
 }
 
+static void
+test_bwmgt_dir_conn_global_write_low(void *arg)
+{
+  bool ret;
+  int addr_family;
+  connection_t *conn = NULL;
+  routerstatus_t *rs = NULL; microdesc_t *md = NULL; routerinfo_t *ri = NULL;
+  tor_addr_t relay_addr;
+
+  (void) arg;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+  MOCK(get_estimated_address_per_node,
+       mock_get_estimated_address_per_node);
+
+  /*
+   * The following is rather complex but that is what it takes to add a dummy
+   * consensus with a valid routerlist which will populate our node address
+   * set that we need to lookup to test the known relay code path.
+   *
+   * We MUST do that before we MOCK(get_options) else it is another world of
+   * complexity.
+   */
+
+  /* This will be the address of our relay. */
+  tor_addr_parse(&relay_addr, "1.2.3.4");
+
+  /* We'll now add a relay into our routerlist and see if we let it. */
+  dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
+  dummy_ns->flavor = FLAV_MICRODESC;
+  dummy_ns->routerstatus_list = smartlist_new();
+
+  md = tor_malloc_zero(sizeof(*md));
+  ri = tor_malloc_zero(sizeof(*ri));
+  rs = tor_malloc_zero(sizeof(*rs));
+  crypto_rand(rs->identity_digest, sizeof(rs->identity_digest));
+  crypto_rand(md->digest, sizeof(md->digest));
+  memcpy(rs->descriptor_digest, md->digest, DIGEST256_LEN);
+
+  /* Set IP address. */
+  rs->addr = tor_addr_to_ipv4h(&relay_addr);
+  ri->addr = rs->addr;
+  /* Add the rs to the consensus becoming a node_t. */
+  smartlist_add(dummy_ns->routerstatus_list, rs);
+
+  /* Add all configured authorities (hardcoded) before we set the consensus so
+   * the address set exists. */
+  ret = consider_adding_dir_servers(&mock_options, &mock_options);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* This will make the nodelist bloom filter very large
+   * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
+  addr_per_node = 1024;
+
+  nodelist_set_consensus(dummy_ns);
+
+  /* Ok, now time to control which options we use. */
+  MOCK(get_options, mock_get_options);
+
+  /* Set ourselves as an authoritative dir. */
+  mock_options.AuthoritativeDir = 1;
+  mock_options.V3AuthoritativeDir = 1;
+  mock_options.UseDefaultFallbackDirs = 0;
+
+  /* This will set our global bucket to 1 byte and thus we will hit the
+   * banwdith limit in our test. */
+  mock_options.BandwidthRate = 1;
+  mock_options.BandwidthBurst = 1;
+
+  /* Else an IPv4 address screams. */
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Initialize the global buckets. */
+  connection_bucket_init();
+
+  /* The address "127.0.0.1" is set with this helper. */
+  conn = test_conn_get_connection(DIR_CONN_STATE_MIN_, CONN_TYPE_DIR,
+                                  DIR_PURPOSE_MIN_);
+  tt_assert(conn);
+
+  /* First try a non authority non relay IP thus a client but we are not
+   * configured to reject requests under load so we should get a false value
+   * that our limit is _not_ low. */
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Now, we will reject requests under load so try again a non authority non
+   * relay IP thus a client. We should get a warning that our limit is too
+   * low. */
+  mock_options.AuthDirRejectRequestsUnderLoad = 1;
+
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Now, lets try with a connection address from moria1. It should always
+   * pass even though our limit is too low. */
+  addr_family = tor_addr_parse(&conn->addr, "128.31.0.39");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* IPv6 testing of gabelmoo. */
+  addr_family = tor_addr_parse(&conn->addr, "[2001:638:a000:4140::ffff:189]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a known relay address. It should pass. Possible due to
+   * our consensus setting above. */
+  memcpy(&conn->addr, &relay_addr, sizeof(tor_addr_t));
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a random IP that is not an authority nor a relay. */
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Finally, just make sure it still denies an IP if we are _not_ a v3
+   * directory authority. */
+  mock_options.V3AuthoritativeDir = 0;
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Random IPv6 should not be allowed. */
+  addr_family = tor_addr_parse(&conn->addr, "[CAFE::ACAB]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  connection_free_minimal(conn);
+  routerstatus_free(rs); routerinfo_free(ri); microdesc_free(md);
+  smartlist_clear(dummy_ns->routerstatus_list);
+  networkstatus_vote_free(dummy_ns);
+
+  UNMOCK(get_estimated_address_per_node);
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+  UNMOCK(get_options);
+}
+
 #define BWMGT(name)                                          \
-  { #name, test_bwmgt_ ## name , 0, NULL, NULL }
+  { #name, test_bwmgt_ ## name , TT_FORK, NULL, NULL }
 
 struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_init),
@@ -229,5 +432,7 @@ struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_dec),
   BWMGT(token_buf_refill),
   BWMGT(token_buf_helpers),
+
+  BWMGT(dir_conn_global_write_low),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_channel.c b/src/test/test_channel.c
index e55b9b0750..6a6bc9d810 100644
--- a/src/test/test_channel.c
+++ b/src/test/test_channel.c
@@ -598,7 +598,6 @@ test_channel_outbound_cell(void *arg)
   circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan);
   tt_int_op(channel_num_circuits(chan), OP_EQ, 1);
   /* Test the cmux state. */
-  tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux);
   tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
             OP_EQ, 1);
 
@@ -1541,6 +1540,10 @@ test_channel_listener(void *arg)
   channel_listener_dump_statistics(chan, LOG_INFO);
 
  done:
+  if (chan) {
+    channel_listener_unregister(chan);
+    tor_free(chan);
+  }
   channel_free_all();
 }
 
@@ -1567,4 +1570,3 @@ struct testcase_t channel_tests[] = {
     NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c
index 5da2d81377..885246628e 100644
--- a/src/test/test_channelpadding.c
+++ b/src/test/test_channelpadding.c
@@ -21,7 +21,7 @@
 #include "test/log_test_helpers.h"
 #include "lib/tls/tortls.h"
 #include "lib/evloop/timers.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "core/or/cell_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -289,8 +289,6 @@ test_channelpadding_timers(void *arg)
   channel_t *chans[CHANNELS_TO_TEST];
   (void)arg;
 
-  tor_libevent_postfork();
-
   if (!connection_array)
     connection_array = smartlist_new();
 
@@ -393,7 +391,6 @@ test_channelpadding_killonehop(void *arg)
   channelpadding_decision_t decision;
   int64_t new_time;
   (void)arg;
-  tor_libevent_postfork();
 
   routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
   monotime_init();
@@ -502,8 +499,6 @@ test_channelpadding_consensus(void *arg)
   int64_t new_time;
   (void)arg;
 
-  tor_libevent_postfork();
-
   /*
    * Params tested:
    *   nf_pad_before_usage
@@ -898,8 +893,6 @@ test_channelpadding_decide_to_pad_channel(void *arg)
     connection_array = smartlist_new();
   (void)arg;
 
-  tor_libevent_postfork();
-
   monotime_init();
   monotime_enable_test_mocking();
   monotime_set_mock_time_nsec(1);
diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c
index 10513e451d..054d3910e4 100644
--- a/src/test/test_channeltls.c
+++ b/src/test/test_channeltls.c
@@ -8,7 +8,7 @@
 #define TOR_CHANNEL_INTERNAL_
 #include "core/or/or.h"
 #include "lib/net/address.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/mainloop/connection.h"
diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c
index 538f20781f..7291e04d6a 100644
--- a/src/test/test_circuitbuild.c
+++ b/src/test/test_circuitbuild.c
@@ -27,11 +27,11 @@ static smartlist_t dummy_nodes;
 static extend_info_t dummy_ei;
 
 static int
-mock_count_acceptable_nodes(smartlist_t *nodes)
+mock_count_acceptable_nodes(const smartlist_t *nodes, int direct)
 {
   (void)nodes;
 
-  return DEFAULT_ROUTE_LEN + 1;
+  return direct ? 1 : DEFAULT_ROUTE_LEN + 1;
 }
 
 /* Test route lengths when the caller of new_route_len() doesn't
@@ -167,6 +167,7 @@ test_upgrade_from_guard_wait(void *arg)
   tt_assert(!list);
 
  done:
+  smartlist_free(list);
   circuit_free(circ);
   entry_guard_free_(guard);
 }
@@ -177,6 +178,6 @@ struct testcase_t circuitbuild_tests[] = {
   { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL },
   { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL },
   { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK,
-    NULL, NULL },
+    &helper_pubsub_setup, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c
new file mode 100644
index 0000000000..934ddb0208
--- /dev/null
+++ b/src/test/test_circuitpadding.c
@@ -0,0 +1,3205 @@
+#define TOR_CHANNEL_INTERNAL_
+#define TOR_TIMERS_PRIVATE
+#define CIRCUITPADDING_PRIVATE
+#define CIRCUITPADDING_MACHINES_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define CRYPT_PATH_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "lib/testsupport/testsupport.h"
+#include "core/or/connection_or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/crypt_path.h"
+#include <event.h>
+#include "lib/evloop/compat_libevent.h"
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "core/or/relay.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/mainloop/netstatus.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/or/protover.h"
+#include "feature/nodelist/nodelist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "test/rng_test_helpers.h"
+
+/* Start our monotime mocking at 1 second past whatever monotime_init()
+ * thought the actual wall clock time was, for platforms with bad resolution
+ * and weird timevalues during monotime_init() before mocking. */
+#define MONOTIME_MOCK_START   (monotime_absolute_nsec()+\
+                               TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC)
+
+extern smartlist_t *connection_array;
+void circuit_expire_old_circuits_clientside(void);
+
+circid_t get_unique_circ_id_by_chan(channel_t *chan);
+void helper_create_basic_machine(void);
+static void helper_create_conditional_machines(void);
+
+static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan);
+channel_t *new_fake_channel(void);
+void test_circuitpadding_negotiation(void *arg);
+void test_circuitpadding_wronghop(void *arg);
+void test_circuitpadding_conditions(void *arg);
+
+void test_circuitpadding_serialize(void *arg);
+void test_circuitpadding_rtt(void *arg);
+void test_circuitpadding_tokens(void *arg);
+void test_circuitpadding_state_length(void *arg);
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding);
+void free_fake_orcirc(circuit_t *circ);
+void free_fake_origin_circuit(origin_circuit_t *circ);
+
+static int deliver_negotiated = 1;
+static int64_t curr_mocked_time;
+
+static node_t padding_node;
+static node_t non_padding_node;
+
+static channel_t dummy_channel;
+static circpad_machine_spec_t circ_client_machine;
+
+static void
+timers_advance_and_run(int64_t msec_update)
+{
+  curr_mocked_time += msec_update*TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+  timers_run_pending();
+}
+
+static void
+nodes_init(void)
+{
+  padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  padding_node.rs->pv.supports_hs_setup_padding = 1;
+
+  non_padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  non_padding_node.rs->pv.supports_hs_setup_padding = 0;
+}
+
+static void
+nodes_free(void)
+{
+  tor_free(padding_node.rs);
+
+  tor_free(non_padding_node.rs);
+}
+
+static const node_t *
+node_get_by_id_mock(const char *identity_digest)
+{
+  if (identity_digest[0] == 1) {
+    return &padding_node;
+  } else if (identity_digest[0] == 0) {
+    return &non_padding_node;
+  }
+
+  return NULL;
+}
+
+static const node_t *
+circuit_get_nth_node_mock(origin_circuit_t *circ, int hop)
+{
+  (void) circ;
+  (void) hop;
+
+  return &padding_node;
+}
+
+static or_circuit_t *
+new_fake_orcirc(channel_t *nchan, channel_t *pchan)
+{
+  or_circuit_t *orcirc = NULL;
+  circuit_t *circ = NULL;
+  crypt_path_t tmp_cpath;
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+
+  orcirc = tor_malloc_zero(sizeof(*orcirc));
+  circ = &(orcirc->base_);
+  circ->magic = OR_CIRCUIT_MAGIC;
+
+  //circ->n_chan = nchan;
+  circ->n_circ_id = get_unique_circ_id_by_chan(nchan);
+  cell_queue_init(&(circ->n_chan_cells));
+  circ->n_hop = NULL;
+  circ->streams_blocked_on_n_chan = 0;
+  circ->streams_blocked_on_p_chan = 0;
+  circ->n_delete_pending = 0;
+  circ->p_delete_pending = 0;
+  circ->received_destroy = 0;
+  circ->state = CIRCUIT_STATE_OPEN;
+  circ->purpose = CIRCUIT_PURPOSE_OR;
+  circ->package_window = CIRCWINDOW_START_MAX;
+  circ->deliver_window = CIRCWINDOW_START_MAX;
+  circ->n_chan_create_cell = NULL;
+
+  //orcirc->p_chan = pchan;
+  orcirc->p_circ_id = get_unique_circ_id_by_chan(pchan);
+  cell_queue_init(&(orcirc->p_chan_cells));
+
+  circuit_set_p_circid_chan(orcirc, orcirc->p_circ_id, pchan);
+  circuit_set_n_circid_chan(circ, circ->n_circ_id, nchan);
+
+  memset(&tmp_cpath, 0, sizeof(tmp_cpath));
+  if (cpath_init_circuit_crypto(&tmp_cpath, whatevs_key,
+                                sizeof(whatevs_key), 0, 0)<0) {
+    log_warn(LD_BUG,"Circuit initialization failed");
+    return NULL;
+  }
+  orcirc->crypto = tmp_cpath.pvt_crypto;
+
+  return orcirc;
+}
+
+void
+free_fake_orcirc(circuit_t *circ)
+{
+  or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
+
+  relay_crypto_clear(&orcirc->crypto);
+
+  circpad_circuit_free_all_machineinfos(circ);
+  tor_free(circ);
+}
+
+void
+free_fake_origin_circuit(origin_circuit_t *circ)
+{
+  circpad_circuit_free_all_machineinfos(TO_CIRCUIT(circ));
+  circuit_clear_cpath(circ);
+  tor_free(circ);
+}
+
+void dummy_nop_timer(void);
+
+//static int dont_stop_libevent = 0;
+
+static circuit_t *client_side;
+static circuit_t *relay_side;
+
+static int n_client_cells = 0;
+static int n_relay_cells = 0;
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction)
+{
+  (void)cmux;
+  (void)circ;
+  (void)direction;
+
+  return;
+}
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno)
+{
+  (void)cell; (void)on_stream; (void)filename; (void)lineno;
+
+  if (circ == client_side) {
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      // Deliver to relay
+      circpad_handle_padding_negotiate(relay_side, cell);
+    } else {
+
+      int is_target_hop = circpad_padding_is_from_expected_hop(circ,
+                                                             layer_hint);
+      tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_OUT);
+      tt_int_op(is_target_hop, OP_EQ, 1);
+
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(client_side);
+
+      // Receive padding cell at middle
+      circpad_deliver_recognized_relay_cell_events(relay_side,
+              cell->payload[0], NULL);
+    }
+    n_client_cells++;
+  } else if (circ == relay_side) {
+    tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_IN);
+
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATED) {
+      // XXX: blah need right layer_hint..
+      if (deliver_negotiated)
+        circpad_handle_padding_negotiated(client_side, cell,
+                                          TO_ORIGIN_CIRCUIT(client_side)
+                                             ->cpath->next);
+    } else if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      circpad_handle_padding_negotiate(client_side, cell);
+    } else {
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(relay_side);
+
+      // Receive padding cell at client
+      circpad_deliver_recognized_relay_cell_events(client_side,
+              cell->payload[0],
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+    }
+
+    n_relay_cells++;
+  }
+
+ done:
+  timers_advance_and_run(1);
+  return 0;
+}
+
+// Test reading and writing padding to strings (or options_t + consensus)
+void
+test_circuitpadding_serialize(void *arg)
+{
+  (void)arg;
+}
+
+static signed_error_t
+circpad_send_command_to_hop_mock(origin_circuit_t *circ, uint8_t hopnum,
+                                 uint8_t relay_command, const uint8_t *payload,
+                                 ssize_t payload_len)
+{
+  (void) circ;
+  (void) hopnum;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  return 0;
+}
+
+void
+test_circuitpadding_rtt(void *arg)
+{
+  /* Test Plan:
+   *
+   * 1. Test RTT measurement server side
+   *    a. test usage of measured RTT
+   * 2. Test termination of RTT measurement
+   *    a. test non-update of RTT
+   * 3. Test client side circuit and non-application of RTT..
+   */
+  circpad_delay_t rtt_estimate;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+  helper_create_basic_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0);
+
+  /* Test 1: Test measuring RTT */
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+
+  timers_advance_and_run(20);
+
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 2: Termination of RTT measurement (from the previous test) */
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec;
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  timers_advance_and_run(4);
+  circpad_cell_event_nonpadding_sent(relay_side);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ,
+            rtt_estimate);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 3: Make sure client side machine properly ignores RTT */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_NE, client_side->padding_info[0]->rtt_estimate_usec);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_EQ,
+            circpad_machine_current_state(
+                client_side->padding_info[0])->histogram_edges[0]);
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+
+  return;
+}
+
+void
+helper_create_basic_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.name = "basic";
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_START].use_rtt_estimate = 1;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 2500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 5000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[4] = 20000;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+
+  return;
+}
+
+#define BIG_HISTOGRAM_LEN 10
+
+/** Setup a machine with a big histogram */
+static void
+helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy)
+{
+  const int tokens_per_bin = 2;
+
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circpad_state_t *burst_state =
+    &circ_client_machine.states[CIRCPAD_STATE_BURST];
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_RECV] =CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_SENT] =CIRCPAD_STATE_CANCEL;
+
+  burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  burst_state->histogram_len = BIG_HISTOGRAM_LEN;
+
+  int n_tokens = 0;
+  int i;
+  for (i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    burst_state->histogram[i] = tokens_per_bin;
+    n_tokens += tokens_per_bin;
+  }
+
+  burst_state->histogram_edges[0] = 0;
+  burst_state->histogram_edges[1] = 1;
+  burst_state->histogram_edges[2] = 7;
+  burst_state->histogram_edges[3] = 15;
+  burst_state->histogram_edges[4] = 31;
+  burst_state->histogram_edges[5] = 62;
+  burst_state->histogram_edges[6] = 125;
+  burst_state->histogram_edges[7] = 250;
+  burst_state->histogram_edges[8] = 500;
+  burst_state->histogram_edges[9] = 1000;
+
+  burst_state->histogram_total_tokens = n_tokens;
+  burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM;
+  burst_state->length_dist.param1 = n_tokens;
+  burst_state->length_dist.param2 = n_tokens;
+  burst_state->max_length = n_tokens;
+  burst_state->length_includes_nonpadding = 1;
+  burst_state->use_rtt_estimate = 0;
+  burst_state->token_removal = removal_strategy;
+}
+
+static circpad_decision_t
+circpad_machine_schedule_padding_mock(circpad_machine_runtime_t *mi)
+{
+  (void)mi;
+  return 0;
+}
+
+static uint64_t
+mock_monotime_absolute_usec(void)
+{
+  return 100;
+}
+
+/** Test higher token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_higher(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_HIGHER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Test right boundaries of each histogram bin: */
+  const circpad_delay_t bin_right_bounds[] =
+    {0,    6,  14,  30,  61,  124, 249, 499, 999, CIRCPAD_DELAY_INFINITE-1};
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_right_bounds[i], OP_EQ,
+               histogram_get_bin_upper_bound(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 5, 6, 7, 8};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all lower bins are not touched */
+  for (i=0; i < 4 ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  mi->padding_scheduled_at_usec = current_time;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test lower token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_lower(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_LOWER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 2, 1, 0};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all higher bins are untouched */
+  for (i = 5; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy by bin  */
+static void
+test_circuitpadding_closest_token_removal(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 6, 1, 7, 0, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_closest_token_removal_usec(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* XXX we want to test remove_token_exact and
+     circpad_machine_remove_closest_token() with usec */
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 1, 0, 6, 7, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_token_removal_exact(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_EXACT);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /**********************************************************************/
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Ensure that we will clear out bin #4 with this usec */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_cell_event_nonpadding_sent(client_side);
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 1);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[4], OP_EQ, 0);
+
+  /* Ensure that we will not remove any other tokens even tho we try to, since
+   * this is what the exact strategy dictates */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  circpad_cell_event_nonpadding_sent(client_side);
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    if (i != 4) {
+      tt_int_op(mi->histogram[i], OP_EQ, 2);
+    }
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+#undef BIG_HISTOGRAM_LEN
+
+void
+test_circuitpadding_tokens(void *arg)
+{
+  const circpad_state_t *state;
+  circpad_machine_runtime_t *mi;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  testing_enable_reproducible_rng();
+
+  /** Test plan:
+   *
+   * 1. Test symmetry between bin_to_usec and usec_to_bin
+   *    a. Test conversion
+   *    b. Test edge transitions (lower, upper)
+   * 2. Test remove higher on an empty bin
+   *    a. Normal bin
+   *    b. Infinity bin
+   *    c. Bin 0
+   *    d. No higher
+   * 3. Test remove lower
+   *    a. Normal bin
+   *    b. Bin 0
+   *    c. No lower
+   * 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  mi = client_side->padding_info[0];
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_sent(client_side);
+  /* We have to save the infinity bin because one inf delay
+   * could have been chosen when we transition to burst */
+  circpad_hist_token_t inf_bin = mi->histogram[4];
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  state = circpad_machine_current_state(client_side->padding_info[0]);
+
+  // Test 0: convert bin->usec->bin
+  // Bin 0+1 have different semantics
+  for (int bin = 0; bin < 2; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+  }
+  for (int bin = 2; bin < state->histogram_len-1; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    /* Verify we round down */
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec+3);
+    tt_int_op(bin, OP_EQ, bin2);
+
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec-1);
+    tt_int_op(bin, OP_EQ, bin2+1);
+  }
+
+  // Test 1: converting usec->bin->usec->bin
+  // Bin 0+1 have different semantics.
+  for (circpad_delay_t i = 0; i <= state->histogram_edges[0]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_LE, usec);
+  }
+  for (circpad_delay_t i = state->histogram_edges[0]+1;
+       i <= state->histogram_edges[0] +
+         state->histogram_edges[state->histogram_len-2]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_GE, usec);
+  }
+
+  /* 2.a. Normal higher bin */
+  {
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+  }
+
+  /* 2.b. Higher Infinity bin */
+  {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+
+    /* Test past the infinity bin */
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 5)+1000000);
+
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+  }
+
+  /* 2.c. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+    mi->histogram[0] = 1;
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* Drain the infinity bin and cause a refill */
+  while (inf_bin != 0) {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_cell_event_nonpadding_received(client_side);
+    inf_bin--;
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+
+  // We should have refilled here.
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 3.a. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 1);
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* 3.b. Test remove lower normal bin */
+  {
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    /* 3.c. No lower */
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+  }
+
+  /* 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 5. Test remove closest usec
+   *    a. Closest 0
+   *    b. Closest lower (below midpoint)
+   *    c. Closest higher (above midpoint)
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  // XXX: Need more coverage of the actual usec branches
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_wronghop(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Padding sent from hop 1 and 3 to client
+   * 2. Send negotiated from hop 1 and 3 to client
+   * 3. Garbled negotiated cell
+   * 4. Padding negotiate sent to client
+   * 5. Send negotiate stop command for unknown machine
+   * 6. Send negotiated to relay
+   * 7. Garbled padding negotiate cell
+   */
+  (void)arg;
+  uint32_t read_bw = 0, overhead_bw = 0;
+  cell_t cell;
+  signed_error_t ret;
+  origin_circuit_t *orig_client;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  /* Mock this function so that our cell counting tests don't get confused by
+   * padding that gets sent by scheduled timers. */
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  orig_client = TO_ORIGIN_CIRCUIT(client_side);
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build three hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+
+  /* 1. Test padding from first and third hop */
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_LT,
+            orig_client->n_overhead_read_circ_bw);
+
+  /* 2. Test padding negotiated not handled from hops 1,3 */
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 3. Garbled negotiated cell */
+  memset(&cell, 255, sizeof(cell));
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 4. Test that negotiate is dropped at origin */
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+  relay_send_command_from_edge(0, relay_side,
+                               RELAY_COMMAND_PADDING_NEGOTIATE,
+                               (void*)cell.payload,
+                               (size_t)3, NULL);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 5. Test that asking to stop the wrong machine does nothing */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            255, 2, CIRCPAD_COMMAND_STOP);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* 6. Sending negotiated command to relay does nothing */
+  ret = circpad_handle_padding_negotiated(relay_side, &cell, NULL);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 7. Test garbled negotated cell (bad command 255) */
+  memset(&cell, 0, sizeof(cell));
+  ret = circpad_handle_padding_negotiate(relay_side, &cell);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Spoof padding negotiated on circuit with no padding */
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_OK);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_ERR);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_negotiation(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Test circuit where padding is supported by middle
+   *    a. Make sure padding negotiation is sent
+   *    b. Test padding negotiation delivery and parsing
+   * 2. Test circuit where padding is unsupported by middle
+   *    a. Make sure padding negotiation is not sent
+   * 3. Test failure to negotiate a machine due to desync.
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Force negotiate padding. */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            CIRCPAD_MACHINE_CIRC_SETUP,
+                            2, CIRCPAD_COMMAND_START);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 3. Test failure to negotiate a machine due to desync */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  SMARTLIST_FOREACH(relay_padding_machines,
+          circpad_machine_spec_t *,
+          m, tor_free(m->states); tor_free(m));
+  smartlist_free(relay_padding_machines);
+  relay_padding_machines = smartlist_new();
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify echo was sent */
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+}
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding)
+{
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+  char digest[DIGEST_LEN];
+  tor_addr_t addr;
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_sent(client);
+
+  // Receive extend cell at middle
+  circpad_cell_event_nonpadding_received(mid_relay);
+
+  // Advance time a tiny bit so we can calculate an RTT
+  curr_mocked_time += 10 * TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+
+  // Receive extended cell at middle
+  circpad_cell_event_nonpadding_sent(mid_relay);
+
+  // Receive extended cell at first hop
+  circpad_cell_event_nonpadding_received(client);
+
+  // Add a hop to cpath
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+  cpath_extend_linked_list(&TO_ORIGIN_CIRCUIT(client)->cpath, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_OPEN;
+
+  // add an extend info to indicate if this node supports padding or not.
+  // (set the first byte of the digest for our mocked node_get_by_id)
+  digest[0] = padding;
+
+  hop->extend_info = extend_info_new(
+          padding ? "padding" : "non-padding",
+          digest, NULL, NULL, NULL,
+          &addr, padding);
+
+  cpath_init_circuit_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  // Signal that the hop was added
+  circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client));
+}
+
+static circpad_machine_spec_t *
+helper_create_length_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  /* No token removal.. end via state_length only */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Let's have this one end after 12 packets */
+  ret->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param1 = 12;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param2 = 13;
+  ret->states[CIRCPAD_STATE_BURST].max_length = 12;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 4;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 6;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 0;
+
+  return ret;
+}
+
+static circpad_machine_spec_t *
+helper_create_conditional_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Use EXACT removal strategy, otherwise setup_tokens() does not work */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 3;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 0;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  return ret;
+}
+
+static void
+helper_create_conditional_machines(void)
+{
+  circpad_machine_spec_t *add = helper_create_conditional_machine();
+
+  if (!origin_padding_machines)
+    origin_padding_machines = smartlist_new();
+  if (!relay_padding_machines)
+    relay_padding_machines = smartlist_new();
+
+  add->machine_num = 2;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 0;
+  add->conditions.min_hops = 2;
+  add->conditions.state_mask = CIRCPAD_CIRC_BUILDING|
+           CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 1;
+  add->conditions.min_hops = 3;
+  add->conditions.state_mask = CIRCPAD_CIRC_OPENED|
+           CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 2;
+  circpad_register_padding_machine(add, relay_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  circpad_register_padding_machine(add, relay_padding_machines);
+}
+
+void
+test_circuitpadding_state_length(void *arg)
+{
+  /**
+   * Test plan:
+   *  * Explicitly test that with no token removal enabled, we hit
+   *    the state length limit due to either padding, or non-padding.
+   *  * Repeat test with an arbitrary token removal strategy, and
+   *    verify that if we run out of tokens due to padding before we
+   *    hit the state length, we still go to state end (all our
+   *    token removal tests only test nonpadding token removal).
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  circpad_machine_spec_t *client_machine =
+      helper_create_length_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  client_side->padding_machine[0] = client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  circpad_machine_runtime_t *mi = client_side->padding_info[0];
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_EQ, NULL);
+
+  /* Verify that non-padding does not change our state length */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* verify that sending padding changes our state length */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+  circpad_send_padding_cell_for_callback(mi);
+
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+
+  /* Now, count nonpadding as part of the state length */
+  client_machine->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* Verify that non-padding does change our state length now */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_cell_event_nonpadding_sent(client_side);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Now, just test token removal when we send padding */
+  client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_NE, NULL);
+  tt_int_op(mi->chosen_bin, OP_EQ, 2);
+
+  /* verify that sending padding changes our state length and
+   * our histogram now */
+  for (uint32_t i = mi->histogram[2]-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_int_op(mi->chosen_bin, OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, i);
+  }
+
+  tt_i64_op(mi->state_length, OP_EQ, 7);
+  tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+  circpad_send_padding_cell_for_callback(mi);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+ done:
+  tor_free(client_machine->states);
+  tor_free(client_machine);
+
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(relay_side);
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(node_get_by_id);
+
+  return;
+}
+
+void
+test_circuitpadding_conditions(void *arg)
+{
+  /**
+   * Test plan:
+   *  0. Make a few origin and client machines with diff conditions
+   *     * vanguards, purposes, has_opened circs, no relay early
+   *     * Client side should_negotiate_end
+   *     * Length limits
+   *  1. Test STATE_END transitions
+   *  2. Test new machine after end with same conditions
+   *  3. Test new machine due to changed conditions
+   *     * Esp: built event, no relay early, no streams
+   * XXX: Diff test:
+   *  1. Test STATE_END with pending timers
+   *  2. Test marking a circuit before padding callback fires
+   *  3. Test freeing a circuit before padding callback fires
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  helper_create_conditional_machines();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Send another event.. verify machine gets re-added properly
+   * (test race with shutdown) */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = 0;
+  circpad_machine_event_circ_has_no_streams(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Now make the circuit opened and send built event */
+  TO_ORIGIN_CIRCUIT(client_side)->has_opened = 1;
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->remaining_relay_early_cells = 0;
+  circpad_machine_event_circ_has_no_relay_early(
+          TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  get_options_mutable()->HSLayer2Nodes = (void*)1;
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = (void*)1;
+  circpad_machine_event_circ_has_streams(TO_ORIGIN_CIRCUIT(client_side));
+
+  /* Verify different machine is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  /* Hold off on negotiated */
+  deliver_negotiated = 0;
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client
+   * but not the response for the padding machine */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify machine is gone from the relay (but negotiated not back yet */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Add another hop and verify it's back */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+ done:
+  /* XXX: Free everything */
+  testing_disable_reproducible_rng();
+  return;
+}
+
+/** Disabled unstable test until #29298 is implemented (see #29122) */
+#if 0
+void
+test_circuitpadding_circuitsetup_machine(void *arg)
+{
+  int64_t actual_mocked_monotime_start;
+  /**
+   * Test case plan:
+   *
+   * 1. Simulate a normal circuit setup pattern
+   *    a. Application traffic
+   *
+   * FIXME: This should focus more on exercising the machine
+   * features rather than actual traffic patterns. For example,
+   * test cancellation and bins empty/refill
+   */
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  dummy_channel.cmux = circuitmux_alloc();
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Test case #1: Build a 3 hop circuit, then wait and let pad */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(n_client_cells, OP_EQ, 1);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_BURST);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 6);
+
+  tt_int_op(client_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_END);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_GAP);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+
+  /* Verify we can't schedule padding in END state */
+  circpad_decision_t ret =
+      circpad_machine_schedule_padding(client_side->padding_info[0]);
+  tt_int_op(ret, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Simulate application traffic */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 7);
+
+  // Test timer cancellation
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 8);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Test timer cancel due to state rules */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  circpad_cell_event_padding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Simulate application traffic to cancel timer */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* No cells sent, except negotiate end from relay */
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 9);
+
+  /* Test mark for close and free */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE);
+  free_fake_orcirc(relay_side);
+  timers_advance_and_run(5000);
+
+  /* No cells sent */
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+
+  return;
+}
+#endif /* 0 */
+
+/** Helper function: Initializes a padding machine where every state uses the
+ *  uniform probability distribution.  */
+static void
+helper_circpad_circ_distribution_machine_setup(int min, int max)
+{
+  circpad_machine_states_init(&circ_client_machine, 7);
+
+  circpad_state_t *zero_st = &circ_client_machine.states[0];
+  zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1;
+  zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM;
+  /* param2 is upper bound, param1 is lower */
+  zero_st->iat_dist.param1 = min;
+  zero_st->iat_dist.param2 = max;
+  zero_st->dist_added_shift_usec = min;
+  zero_st->dist_max_sample_usec = max;
+
+  circpad_state_t *first_st = &circ_client_machine.states[1];
+  first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2;
+  first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC;
+  /* param1 is Mu, param2 is sigma. */
+  first_st->iat_dist.param1 = 9;
+  first_st->iat_dist.param2 = 3;
+  first_st->dist_added_shift_usec = min;
+  first_st->dist_max_sample_usec = max;
+
+  circpad_state_t *second_st = &circ_client_machine.states[2];
+  second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3;
+  second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC;
+  /* param1 is Alpha, param2 is 1.0/Beta */
+  second_st->iat_dist.param1 = 1;
+  second_st->iat_dist.param2 = 0.5;
+  second_st->dist_added_shift_usec = min;
+  second_st->dist_max_sample_usec = max;
+
+  circpad_state_t *third_st = &circ_client_machine.states[3];
+  third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4;
+  third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC;
+  /* param1 is 'p' (success probability) */
+  third_st->iat_dist.param1 = 0.2;
+  third_st->dist_added_shift_usec = min;
+  third_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fourth_st = &circ_client_machine.states[4];
+  fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5;
+  fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL;
+  /* param1 is k, param2 is Lambda */
+  fourth_st->iat_dist.param1 = 1.5;
+  fourth_st->iat_dist.param2 = 1;
+  fourth_st->dist_added_shift_usec = min;
+  fourth_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fifth_st = &circ_client_machine.states[5];
+  fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6;
+  fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO;
+  /* param1 is sigma, param2 is xi */
+  fifth_st->iat_dist.param1 = 1;
+  fifth_st->iat_dist.param2 = 5;
+  fifth_st->dist_added_shift_usec = min;
+  fifth_st->dist_max_sample_usec = max;
+}
+
+/** Simple test that the padding delays sampled from a uniform distribution
+ *  actually faill within the uniform distribution range. */
+static void
+test_circuitpadding_sample_distribution(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  int n_samples;
+  int n_states;
+
+  (void) arg;
+
+  /* mock this function so that we dont actually schedule any padding */
+  MOCK(circpad_machine_schedule_padding,
+       circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Initialize a machine with multiple probability distributions */
+  circpad_machines_init();
+  helper_circpad_circ_distribution_machine_setup(0, 10);
+
+  /* Initialize machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* For every state, sample a bunch of values from the distribution and ensure
+   * they fall within range. */
+  for (n_states = 0 ; n_states < 6; n_states++) {
+    /* Make sure we in the right state */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, n_states);
+
+    for (n_samples = 0; n_samples < 100; n_samples++) {
+      circpad_delay_t delay = circpad_machine_sample_delay(mi);
+      tt_int_op(delay, OP_GE, 0);
+      tt_int_op(delay, OP_LE, 10);
+    }
+
+    /* send a non-padding cell to move to the next machine state */
+    circpad_cell_event_nonpadding_received(client_side);
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  UNMOCK(circpad_machine_schedule_padding);
+  testing_disable_reproducible_rng();
+}
+
+static circpad_decision_t
+circpad_machine_spec_transition_mock(circpad_machine_runtime_t *mi,
+                                circpad_event_t event)
+{
+  (void) mi;
+  (void) event;
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/* Test per-machine padding rate limits */
+static void
+test_circuitpadding_machine_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  /* First we are going to test the per-machine rate limits */
+  circ_client_machine.max_padding_percent = 50;
+  circ_client_machine.allowed_padding_count = 100;
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Cover wrap */
+  for (;i<UINT16_MAX;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/2+1);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, mi);
+  for (i=0;i<UINT16_MAX;i++) {
+    circpad_cell_event_nonpadding_sent(client_side);
+  }
+
+  tt_int_op(mi->nonpadding_sent, OP_EQ, UINT16_MAX/2);
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/4+1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  testing_disable_reproducible_rng();
+}
+
+/* Test global padding rate limits */
+static void
+test_circuitpadding_global_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+  int64_t actual_mocked_monotime_start;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  testing_enable_reproducible_rng();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  dummy_channel.cmux = circuitmux_alloc();
+
+  /* Setup machine and circuits */
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  helper_create_basic_machine();
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(relay_side, 0);
+  mi = relay_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Now test the global limits by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_global_allowed_cells=100 circpad_global_max_padding_pct=50",
+                         NULL, 0, 0);
+  /* Register global limits with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Now send 92 non-padding cells to get near the
+   * circpad_global_max_padding_pct=50 limit; in particular with 96 non-padding
+   * cells, the padding traffic is still 51% of total traffic so limit should
+   * trigger */
+  for (i=0;i<92;i++) {
+    circpad_cell_event_nonpadding_sent(relay_side);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Send another non-padding cell to bring the padding traffic to 50% of total
+   * traffic and get past the limit */
+  circpad_cell_event_nonpadding_sent(relay_side);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  testing_disable_reproducible_rng();
+}
+
+/* Test reduced and disabled padding */
+static void
+test_circuitpadding_reduce_disable(void *arg)
+{
+  (void) arg;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel);
+  client_side = (circuit_t *)origin_circuit_new();
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  circpad_machines_init();
+  helper_create_conditional_machines();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Now test the reduced padding machine by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_reduced=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  /* Test that machines get torn down when padding is disabled */
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_disabled=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = NULL;
+  circpad_new_consensus_params(&vote1);
+
+  get_options_mutable()->ReducedCircuitPadding = 1;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  get_options_mutable()->CircuitPadding = 0;
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  testing_disable_reproducible_rng();
+}
+
+/** Just a basic machine whose whole purpose is to reach the END state */
+static void
+helper_create_ender_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_END;
+
+  circ_client_machine.conditions.state_mask = CIRCPAD_STATE_ALL;
+  circ_client_machine.conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+}
+
+static time_t mocked_timeofday;
+/** Set timeval to a mock date and time. This is necessary
+ * to make tor_gettimeofday() mockable. */
+static void
+mock_tor_gettimeofday(struct timeval *timeval)
+{
+  timeval->tv_sec = mocked_timeofday;
+  timeval->tv_usec = 0;
+}
+
+/** Test manual managing of circuit lifetimes by the circuitpadding
+ *  subsystem. In particular this test goes through all the cases of the
+ *  circpad_marked_circuit_for_padding() function, via
+ *  circuit_mark_for_close() as well as
+ *  circuit_expire_old_circuits_clientside(). */
+static void
+test_circuitpadding_manage_circuit_lifetime(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+  MOCK(tor_gettimeofday, mock_tor_gettimeofday);
+  mocked_timeofday = 23;
+
+  helper_create_ender_machine();
+
+  /* Enable manual circuit lifetime manage for this test */
+  circ_client_machine.manage_circ_lifetime = 1;
+
+  /* Test setup */
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Check that the circuit is not marked for close */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+
+  /* Mark this circuit for close due to a remote reason */
+  circuit_mark_for_close(client_side,
+                         END_CIRC_REASON_FLAG_REMOTE|END_CIRC_REASON_NONE);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close due to a protocol issue */
+  circuit_mark_for_close(client_side, END_CIRC_REASON_TORPROTOCOL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark a measurement circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  client_side->marked_for_close = 0;
+
+  /* Mark a general circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+
+  /* Check that this circuit is still not marked for close since we are
+   * managing the lifetime manually, but the circuit was tagged as such by the
+   * circpadding subsystem */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* We just tested case (1) from the comments of
+   * circpad_circuit_should_be_marked_for_close() */
+
+  /* Transition the machine to the END state but did not delete its machine */
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* We just tested case (3) from the comments of
+   * circpad_circuit_should_be_marked_for_close().
+   * Now let's go for case (2). */
+
+  /* Reset the close mark */
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close */
+  circuit_mark_for_close(client_side, 0);
+
+  /* See that the circ got closed since we are already in END state */
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* We just tested case (2). Now let's see that case (4) is unreachable as
+     that comment claims */
+
+  /* First, reset all close marks and tags */
+  client_side->marked_for_close = 0;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  /* Now re-create the ender machine so that we can transition to END again */
+  /* Free up some stuff first */
+  circpad_circuit_free_all_machineinfos(client_side);
+  tor_free(circ_client_machine.states);
+  helper_create_ender_machine();
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* Check we are in START. */
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Test that we don't expire this circuit yet */
+  client_side->timestamp_dirty = 0;
+  client_side->state = CIRCUIT_STATE_OPEN;
+  tor_gettimeofday(&client_side->timestamp_began);
+  TO_ORIGIN_CIRCUIT(client_side)->circuit_idle_timeout = 23;
+  mocked_timeofday += 24;
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->timestamp_dirty, OP_NE, 0);
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Runaway circpad test: if the machine does not transition to end,
+   * test that after CIRCPAD_DELAY_MAX_SECS, we get marked anyway */
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  client_side->padding_info[0]->last_cell_time_sec =
+      approx_time()-(CIRCPAD_DELAY_MAX_SECS+10);
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* Test back to normal: if we had activity, we won't close */
+  client_side->padding_info[0]->last_cell_time_sec = approx_time();
+  client_side->marked_for_close = 0;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Transition to END, but before we're past the dirty timer */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Verify that the circuit was not closed. */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Now that we are in END state, we can be closed by expiry, but via
+   * the timestamp_dirty path, not the idle path. So first test not dirty
+   * enough. */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  tor_free(circ_client_machine.states);
+  monotime_disable_test_mocking();
+  UNMOCK(tor_gettimeofday);
+}
+
+/** Helper for the test_circuitpadding_hs_machines test:
+ *
+ *  - Create a client and relay circuit.
+ *  - Setup right circuit purpose and attach a machine to the client circuit.
+ *  - Verify that state transitions work as intended and state length gets
+ *    enforced.
+ *
+ *  This function is able to do this test both for intro and rend circuits
+ *  depending on the value of <b>test_intro_circs</b>.
+ */
+static void
+helper_test_hs_machines(bool test_intro_circs)
+{
+  /* Setup the circuits */
+  origin_circuit_t *origin_client_side = origin_circuit_new();
+  client_side = TO_CIRCUIT(origin_client_side);
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+
+  /* extend the client circ to two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* machines only apply on opened circuits */
+  origin_client_side->has_opened = 1;
+
+  /************************************/
+
+  /* Attaching the client machine now won't work here because of a wrong
+   * purpose */
+  tt_assert(!client_side->padding_machine[0]);
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_assert(!client_side->padding_machine[0]);
+
+  /* Change the purpose, see the machine getting attached */
+  client_side->purpose = test_intro_circs ?
+    CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT : CIRCUIT_PURPOSE_C_REND_JOINED;
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify that the right machine is attached */
+  tt_str_op(client_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "client_ip_circ" : "client_rp_circ");
+  tt_str_op(relay_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "relay_ip_circ": "relay_rp_circ");
+
+  /***********************************/
+
+  /* Intro machines are at START state, but rend machines have already skipped
+   * to OBFUSCATE_CIRC_SETUP because of the sent PADDING_NEGOTIATE. */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /*Send non-padding to move the machines from START to OBFUSCATE_CIRC_SETUP */
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /* Check that the state lengths have been sampled and are within range */
+  circpad_machine_runtime_t *client_machine_runtime =
+    client_side->padding_info[0];
+  circpad_machine_runtime_t *relay_machine_runtime =
+    relay_side->padding_info[0];
+
+  if (test_intro_circs) {
+    /* on the client side, we don't send any padding so
+     * state length is not set */
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, -1);
+    /* relay side has state limits. check them */
+    tt_i64_op(relay_machine_runtime->state_length, OP_GE,
+              INTRO_MACHINE_MINIMUM_PADDING);
+    tt_i64_op(relay_machine_runtime->state_length, OP_LT,
+              INTRO_MACHINE_MAXIMUM_PADDING);
+  } else {
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, 1);
+    tt_i64_op(relay_machine_runtime->state_length, OP_EQ, 1);
+  }
+
+  if (test_intro_circs) {
+    int i;
+    /* Send state_length worth of padding from the relay and see that the
+     * client state goes to END */
+    for (i = (int) relay_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(relay_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted (the padding info should now be null on both sides) */
+    tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+    tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  } else {
+    int i;
+    /* Send state_length worth of padding and see that the state goes to END */
+    for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(client_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted. */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_END);
+  }
+
+ done:
+  free_fake_orcirc(relay_side);
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+}
+
+/** Test that the HS circuit padding machines work as intended. */
+static void
+test_circuitpadding_hs_machines(void *arg)
+{
+  (void)arg;
+
+  /* Test logic:
+   *
+   * 1) Register the HS machines, which aim to hide the presense of
+   *    onion service traffic on the client-side
+   *
+   * 2) Call helper_test_hs_machines() to perform tests for the intro circuit
+   *    machines and for the rend circuit machines.
+  */
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock);
+  MOCK(circuit_get_nth_node, circuit_get_nth_node_mock);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  curr_mocked_time = 1*TOR_NSEC_PER_USEC;
+
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  /************************************/
+
+  /* Register the HS machines */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  /***********************************/
+
+  /* Do the tests for the intro circuit machines */
+  helper_test_hs_machines(true);
+  /* Do the tests for the rend circuit machines */
+  helper_test_hs_machines(false);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+
+  SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  smartlist_free(origin_padding_machines);
+  smartlist_free(relay_padding_machines);
+
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuit_get_nth_node);
+  UNMOCK(circpad_machine_schedule_padding);
+}
+
+/** Test that we effectively ignore non-padding cells in padding circuits. */
+static void
+test_circuitpadding_ignore_non_padding_cells(void *arg)
+{
+  int retval;
+  relay_header_t rh;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_CIRCUIT_PADDING;
+
+  rh.command = RELAY_COMMAND_BEGIN;
+
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = handle_relay_cell_command(NULL, client_side, NULL, NULL, &rh, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  expect_log_msg_containing("Ignored cell");
+
+ done:
+  ;
+}
+
+#define TEST_CIRCUITPADDING(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t circuitpadding_tests[] = {
+  TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_state_length, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK),
+  /** Disabled unstable test until #29298 is implemented (see #29122) */
+  //  TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_reduce_disable, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_manage_circuit_lifetime, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_hs_machines, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_ignore_non_padding_cells, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c
index 1cbcb14f2b..9bfaabeb2f 100644
--- a/src/test/test_circuitstats.c
+++ b/src/test/test_circuitstats.c
@@ -28,7 +28,7 @@ origin_circuit_t *subtest_fourhop_circuit(struct timeval, int);
 origin_circuit_t *add_opened_threehop(void);
 origin_circuit_t *build_unopened_fourhop(struct timeval);
 
-int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 
 static int marked_for_close;
 /* Mock function because we are not trying to test the close circuit that does
@@ -57,9 +57,9 @@ add_opened_threehop(void)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = DEFAULT_ROUTE_LEN;
 
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
 
   or_circ->has_opened = 1;
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -82,10 +82,10 @@ build_unopened_fourhop(struct timeval circ_start_time)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = 4;
 
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
 
   tor_free(fakehop);
 
@@ -197,7 +197,7 @@ test_circuitstats_hoplen(void *arg)
 }
 
 #define TEST_CIRCUITSTATS(name, flags) \
-    { #name, test_##name, (flags), NULL, NULL }
+    { #name, test_##name, (flags), &helper_pubsub_setup, NULL }
 
 struct testcase_t circuitstats_tests[] = {
   TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK),
diff --git a/src/test/test_cmdline.sh b/src/test/test_cmdline.sh
new file mode 100755
index 0000000000..cf758c3851
--- /dev/null
+++ b/src/test/test_cmdline.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+umask 077
+set -e
+
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+die() { echo "$1" >&2 ; exit 5; }
+
+echo "A"
+
+DATA_DIR=$(mktemp -d -t tor_cmdline_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# 1. Test list-torrc-options.
+OUT="${DATA_DIR}/output"
+
+echo "B"
+"${TOR_BINARY}" --list-torrc-options > "$OUT"
+
+echo "C"
+
+# regular options are given.
+grep -i "SocksPort" "$OUT" >/dev/null || die "Did not find SocksPort"
+
+
+echo "D"
+
+# unlisted options are given, since they do not have the NOSET flag.
+grep -i "__SocksPort" "$OUT" > /dev/null || die "Did not find __SocksPort"
+
+echo "E"
+
+# unsettable options are not given.
+if grep -i "DisableIOCP" "$OUT"  /dev/null; then
+    die "Found DisableIOCP"
+fi
+if grep -i "HiddenServiceOptions" "$OUT" /dev/null ; then
+    die "Found HiddenServiceOptions"
+fi
+echo "OK"
diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c
index 2f8646e897..ecd97e3474 100644
--- a/src/test/test_compat_libevent.c
+++ b/src/test/test_compat_libevent.c
@@ -151,8 +151,6 @@ test_compat_libevent_postloop_events(void *arg)
   mainloop_event_t *a = NULL, *b = NULL;
   periodic_timer_t *timed = NULL;
 
-  tor_libevent_postfork();
-
   /* If postloop events don't work, then these events will activate one
    * another ad infinitum and, and the periodic event will never occur. */
   b = mainloop_event_postloop_new(activate_event_cb, &a);
@@ -187,4 +185,3 @@ struct testcase_t compat_libevent_tests[] = {
     TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_config.c b/src/test/test_config.c
index 855725411a..ebc0624fb2 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -16,7 +16,7 @@
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "test/test.h"
@@ -45,6 +45,7 @@
 #include "app/config/statefile.h"
 
 #include "test/test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "feature/dirclient/dir_server_st.h"
 #include "core/or/port_cfg_st.h"
@@ -54,6 +55,7 @@
 #include "lib/meminfo/meminfo.h"
 #include "lib/net/gethostname.h"
 #include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -1754,6 +1756,18 @@ add_default_fallback_dir_servers_known_default(void)
   n_add_default_fallback_dir_servers_known_default++;
 }
 
+/* Helper for test_config_adding_dir_servers(), which should be
+ * refactored: clear the fields in the options which the options object
+ * does not really own. */
+static void
+ads_clear_helper(or_options_t *options)
+{
+  options->DirAuthorities = NULL;
+  options->AlternateBridgeAuthority = NULL;
+  options->AlternateDirAuthority = NULL;
+  options->FallbackDir = NULL;
+}
+
 /* Test all the different combinations of adding dir servers */
 static void
 test_config_adding_dir_servers(void *arg)
@@ -1761,7 +1775,7 @@ test_config_adding_dir_servers(void *arg)
   (void)arg;
 
   /* allocate options */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
 
   /* Allocate and populate configuration lines:
    *
@@ -1884,7 +1898,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -1966,7 +1982,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2107,7 +2125,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2248,7 +2268,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2390,7 +2412,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2542,7 +2566,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2696,7 +2722,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2859,7 +2887,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3016,7 +3046,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3182,7 +3214,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3345,7 +3379,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3514,10 +3550,7 @@ test_config_adding_dir_servers(void *arg)
   tor_free(test_fallback_directory->value);
   tor_free(test_fallback_directory);
 
-  options->DirAuthorities = NULL;
-  options->AlternateBridgeAuthority = NULL;
-  options->AlternateDirAuthority = NULL;
-  options->FallbackDir = NULL;
+  ads_clear_helper(options);
   or_options_free(options);
 
   UNMOCK(add_default_fallback_dir_servers);
@@ -3532,7 +3565,7 @@ test_config_default_dir_servers(void *arg)
   int fallback_count = 0;
 
   /* new set of options should stop fallback parsing */
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 0;
   /* set old_options to NULL to force dir update */
   consider_adding_dir_servers(opts, NULL);
@@ -3546,7 +3579,7 @@ test_config_default_dir_servers(void *arg)
   /* if we disable the default fallbacks, there must not be any extra */
   tt_assert(fallback_count == trusted_count);
 
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 1;
   consider_adding_dir_servers(opts, opts);
   trusted_count = smartlist_len(router_get_trusted_dir_servers());
@@ -3606,7 +3639,7 @@ test_config_directory_fetch(void *arg)
   (void)arg;
 
   /* Test Setup */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
   routerinfo_t routerinfo;
   memset(&routerinfo, 0, sizeof(routerinfo));
   mock_router_pick_published_address_result = -1;
@@ -3618,9 +3651,10 @@ test_config_directory_fetch(void *arg)
        mock_router_my_exit_policy_is_reject_star);
   MOCK(advertised_server_mode, mock_advertised_server_mode);
   MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo);
+  or_options_free(options);
+  options = options_new();
 
   /* Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
   options->ClientOnly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
@@ -3629,7 +3663,8 @@ test_config_directory_fetch(void *arg)
             OP_EQ, 1);
 
   /* Bridge Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->UseBridges = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
@@ -3639,7 +3674,8 @@ test_config_directory_fetch(void *arg)
 
   /* Bridge Relays (Bridges) must act like clients, and use multiple
    * directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->BridgeRelay = 1;
   options->ORPort_set = 1;
   tt_assert(server_mode(options) == 1);
@@ -3650,7 +3686,8 @@ test_config_directory_fetch(void *arg)
 
   /* Clients set to FetchDirInfoEarly must fetch it from the authorities,
    * but can use multiple authorities for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->FetchDirInfoEarly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
@@ -3661,7 +3698,8 @@ test_config_directory_fetch(void *arg)
   /* OR servers only fetch the consensus from the authorities when they don't
    * know their own address, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
 
   mock_router_pick_published_address_result = -1;
@@ -3681,7 +3719,8 @@ test_config_directory_fetch(void *arg)
   /* Exit OR servers only fetch the consensus from the authorities when they
    * refuse unknown exits, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
   options->ExitRelay = 1;
   mock_router_pick_published_address_result = 0;
@@ -3711,7 +3750,8 @@ test_config_directory_fetch(void *arg)
    * advertising their dirport, and never use multiple directories for
    * bootstrap. This only applies if they are also OR servers.
    * (We don't care much about the behaviour of non-OR directory servers.) */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->DirPort_set = 1;
   options->ORPort_set = 1;
   options->DirCache = 1;
@@ -3765,7 +3805,7 @@ test_config_directory_fetch(void *arg)
             OP_EQ, 0);
 
  done:
-  tor_free(options);
+  or_options_free(options);
   UNMOCK(router_pick_published_address);
   UNMOCK(router_get_my_routerinfo);
   UNMOCK(advertised_server_mode);
@@ -4029,6 +4069,8 @@ test_config_parse_port_config__ports__ports_given(void *data)
 
   slout = smartlist_new();
 
+  mock_hostname_resolver();
+
   // Test error when encounters an invalid Port specification
   config_port_invalid = mock_config_line("DNSPort", "");
   ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
@@ -4725,6 +4767,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
 #endif /* defined(_WIN32) */
 
  done:
+  unmock_hostname_resolver();
   if (slout)
     SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_free(slout);
@@ -5063,7 +5106,7 @@ test_config_include_no_permission(void *data)
     chmod(dir, 0700);
   tor_free(dir);
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 static void
 test_config_include_recursion_before_after(void *data)
@@ -5778,7 +5821,7 @@ test_config_compute_max_mem_in_queues(void *data)
 #else
   /* We are on a 32-bit system. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1));
-#endif
+#endif /* SIZEOF_VOID_P >= 8 */
 
   /* We are able to detect the amount of RAM on the system. */
   total_system_memory_return = 0;
@@ -5819,7 +5862,7 @@ test_config_compute_max_mem_in_queues(void *data)
   /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
             MAX_DEFAULT_MEMORY_QUEUE_SIZE);
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
  done:
   UNMOCK(get_total_system_memory);
@@ -5863,6 +5906,7 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "is back here");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
   lp = lp->next;
+  tt_assert(!lp);
   config_free_lines(lines);
 
   /* Try with the "extended" flag enabled. */
@@ -5889,9 +5933,166 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR);
   lp = lp->next;
+  tt_assert(!lp);
+
+ done:
+  config_free_lines(lines);
+}
+
+static void
+test_config_kvline_parse(void *arg)
+{
+  (void)arg;
+
+  config_line_t *lines = NULL;
+  char *enc = NULL;
+
+  lines = kvline_parse("A=B CD=EF", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "A");
+  tt_str_op(lines->value, OP_EQ, "B");
+  tt_str_op(lines->next->key, OP_EQ, "CD");
+  tt_str_op(lines->next->value, OP_EQ, "EF");
+  enc = kvline_encode(lines, 0);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB CDE=F", 0);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "");
+  tt_str_op(lines->value, OP_EQ, "AB");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F");
+  tt_assert(lines);
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB CDE=F");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\"", 0);
+  tt_assert(!lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "C");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F G");
+  tt_str_op(lines->next->next->key, OP_EQ, "");
+  tt_str_op(lines->next->next->value, OP_EQ, "GHI");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB=", KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse(" AB ", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD DE FGH=I", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD DE FGH=I");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=\"CD E\" DE FGH=\"I\"", KV_OMIT_VALS|KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD E");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS|KV_QUOTED);
+  tt_str_op(enc, OP_EQ, "AB=\"CD E\" DE FGH=I");
 
  done:
   config_free_lines(lines);
+  tor_free(enc);
+}
+
+static void
+test_config_getinfo_config_names(void *arg)
+{
+  (void)arg;
+  char *answer = NULL;
+  const char *error = NULL;
+  int rv;
+
+  rv = getinfo_helper_config(NULL, "config/names", &answer, &error);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(error, OP_EQ, NULL);
+
+  // ContactInfo should be listed.
+  tt_assert(strstr(answer, "\nContactInfo String\n"));
+
+  // V1AuthoritativeDirectory should not be listed, since it is obsolete.
+  tt_assert(! strstr(answer, "V1AuthoritativeDirectory"));
+
+  // ___UsingTestNetworkDefaults should not be listed, since it is invisible.
+  tt_assert(! strstr(answer, "UsingTestNetworkDefaults"));
+
+ done:
+  tor_free(answer);
 }
 
 #define CONFIG_TEST(name, flags)                          \
@@ -5946,5 +6147,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(include_opened_file_list, 0),
   CONFIG_TEST(compute_max_mem_in_queues, 0),
   CONFIG_TEST(extended_fmt, 0),
+  CONFIG_TEST(kvline_parse, 0),
+  CONFIG_TEST(getinfo_config_names, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_confmgr.c b/src/test/test_confmgr.c
new file mode 100644
index 0000000000..d5c73b48e4
--- /dev/null
+++ b/src/test/test_confmgr.c
@@ -0,0 +1,325 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confparse.c's features that support multiple configuration
+ * formats and configuration objects.
+ */
+
+#define CONFPARSE_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confparse.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+/*
+ * Set up a few objects: a pasture_cfg is toplevel; it has a llama_cfg and an
+ * alpaca_cfg.
+ */
+
+typedef struct {
+  uint32_t magic;
+  char *address;
+  int opentopublic;
+  config_suite_t *subobjs;
+} pasture_cfg_t;
+
+typedef struct {
+  char *llamaname;
+  int cuteness;
+  uint32_t magic;
+  int eats_meat; /* deprecated; llamas are never carnivorous. */
+
+  char *description; // derived from other fields.
+} llama_cfg_t;
+
+typedef struct {
+  uint32_t magic;
+  int fuzziness;
+  char *alpacaname;
+  int n_wings; /* deprecated; alpacas don't have wings. */
+} alpaca_cfg_t;
+
+/*
+ * Make the above into configuration objects.
+ */
+
+static pasture_cfg_t pasture_cfg_t_dummy;
+static llama_cfg_t llama_cfg_t_dummy;
+static alpaca_cfg_t alpaca_cfg_t_dummy;
+
+#define PV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(pasture_cfg_t, #name, type, name, 0, dflt)
+#define LV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(llama_cfg_t, #name, type, name, 0, dflt)
+#define AV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(alpaca_cfg_t, #name, type, name, 0, dflt)
+static const config_var_t pasture_vars[] = {
+  PV(address, STRING, NULL),
+  PV(opentopublic, BOOL, "1"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t llama_vars[] =
+{
+  LV(llamaname, STRING, NULL),
+  LV(eats_meat, BOOL, NULL),
+  LV(cuteness, POSINT, "100"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t alpaca_vars[] =
+{
+  AV(alpacaname, STRING, NULL),
+  AV(fuzziness, POSINT, "50"),
+  AV(n_wings, POSINT, "0"),
+  END_OF_CONFIG_VARS
+};
+
+static config_deprecation_t llama_deprecations[] = {
+  { "eats_meat", "Llamas are herbivores." },
+  {NULL,NULL}
+};
+
+static config_deprecation_t alpaca_deprecations[] = {
+  { "n_wings", "Alpacas are quadrupeds." },
+  {NULL,NULL}
+};
+
+static int clear_llama_cfg_called = 0;
+static void
+clear_llama_cfg(const config_mgr_t *mgr, void *llamacfg)
+{
+  (void)mgr;
+  llama_cfg_t *lc = llamacfg;
+  tor_free(lc->description);
+  ++clear_llama_cfg_called;
+}
+
+static config_abbrev_t llama_abbrevs[] = {
+  { "gracia", "cuteness", 0, 0 },
+  { "gentillesse", "cuteness", 0, 0 },
+  { NULL, NULL, 0, 0 },
+};
+
+static const config_format_t pasture_fmt = {
+  sizeof(pasture_cfg_t),
+  {
+    "pasture_cfg_t",
+    8989,
+    offsetof(pasture_cfg_t, magic)
+  },
+  .vars = pasture_vars,
+  .config_suite_offset = offsetof(pasture_cfg_t, subobjs),
+};
+
+static const config_format_t llama_fmt = {
+  sizeof(llama_cfg_t),
+  {
+    "llama_cfg_t",
+    0x11aa11,
+    offsetof(llama_cfg_t, magic)
+  },
+  .vars = llama_vars,
+  .config_suite_offset = -1,
+  .deprecations = llama_deprecations,
+  .abbrevs = llama_abbrevs,
+  .clear_fn = clear_llama_cfg,
+};
+
+static const config_format_t alpaca_fmt = {
+  sizeof(alpaca_cfg_t),
+  {
+    "alpaca_cfg_t",
+    0xa15aca,
+    offsetof(alpaca_cfg_t, magic)
+  },
+  .vars = alpaca_vars,
+  .config_suite_offset = -1,
+  .deprecations = alpaca_deprecations,
+};
+
+#define LLAMA_IDX 0
+#define ALPACA_IDX 1
+
+static config_mgr_t *
+get_mgr(bool freeze)
+{
+  config_mgr_t *mgr = config_mgr_new(&pasture_fmt);
+  tt_int_op(LLAMA_IDX, OP_EQ, config_mgr_add_format(mgr, &llama_fmt));
+  tt_int_op(ALPACA_IDX, OP_EQ, config_mgr_add_format(mgr, &alpaca_fmt));
+  if (freeze)
+    config_mgr_freeze(mgr);
+  return mgr;
+
+ done:
+  config_mgr_free(mgr);
+  return NULL;
+}
+
+static void
+test_confmgr_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  smartlist_t *vars = NULL;
+  tt_ptr_op(mgr, OP_NE, NULL);
+
+  vars = config_mgr_list_vars(mgr);
+  tt_int_op(smartlist_len(vars), OP_EQ, 8); // 8 vars total.
+
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "CUTENESS"));
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "GRACIA"));
+  smartlist_free(vars);
+
+  vars = config_mgr_list_deprecated_vars(mgr); // 2 deprecated vars.
+  tt_int_op(smartlist_len(vars), OP_EQ, 2);
+  tt_assert(smartlist_contains_string(vars, "eats_meat"));
+  tt_assert(smartlist_contains_string(vars, "n_wings"));
+
+  tt_str_op("Llamas are herbivores.", OP_EQ,
+            config_find_deprecation(mgr, "EATS_MEAT"));
+  tt_str_op("Alpacas are quadrupeds.", OP_EQ,
+            config_find_deprecation(mgr, "N_WINGS"));
+
+ done:
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confmgr_magic(void *args)
+{
+  (void)args;
+  // Every time we build a manager, it is supposed to get a different magic
+  // number.  Let's test that.
+  config_mgr_t *mgr1 = get_mgr(true);
+  config_mgr_t *mgr2 = get_mgr(true);
+  config_mgr_t *mgr3 = get_mgr(true);
+
+  pasture_cfg_t *p1 = NULL, *p2 = NULL, *p3 = NULL;
+
+  tt_assert(mgr1);
+  tt_assert(mgr2);
+  tt_assert(mgr3);
+
+  p1 = config_new(mgr1);
+  p2 = config_new(mgr2);
+  p3 = config_new(mgr3);
+
+  tt_assert(p1);
+  tt_assert(p2);
+  tt_assert(p3);
+
+  // By chance, two managers get the same magic with P=2^-32.  Let's
+  // make sure that at least two of them are different, so that our
+  // odds of a false positive are 1/2^-64.
+  tt_assert((p1->magic != p2->magic) || (p2->magic != p3->magic));
+
+ done:
+  config_free(mgr1, p1);
+  config_free(mgr2, p2);
+  config_free(mgr3, p3);
+
+  config_mgr_free(mgr1);
+  config_mgr_free(mgr2);
+  config_mgr_free(mgr3);
+}
+
+static const char *simple_pasture =
+  "LLamaname hugo\n"
+  "Alpacaname daphne\n"
+  "gentillesse 42\n"
+  "address 123 Camelid ave\n";
+
+static void
+test_confmgr_parse(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, p); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_int_op(p->opentopublic, OP_EQ, 1);
+  tt_str_op(p->address, OP_EQ, "123 Camelid ave");
+
+  // We are using this API directly; modules outside confparse will, in the
+  // future, not.
+  const alpaca_cfg_t *ac = config_mgr_get_obj(mgr, p, ALPACA_IDX);
+  const llama_cfg_t *lc = config_mgr_get_obj(mgr, p, LLAMA_IDX);
+  tt_str_op(lc->llamaname, OP_EQ, "hugo");
+  tt_str_op(ac->alpacaname, OP_EQ, "daphne");
+  tt_int_op(lc->cuteness, OP_EQ, 42);
+  tt_int_op(ac->fuzziness, OP_EQ, 50);
+
+  // We set the description for the llama here, so that the clear function
+  // can clear it.  (Later we can do this in a verification function.)
+  clear_llama_cfg_called = 0;
+  llama_cfg_t *mut_lc = config_mgr_get_obj_mutable(mgr, p, LLAMA_IDX);
+  mut_lc->description = tor_strdup("A llama named Hugo.");
+  config_free(mgr, p);
+  tt_int_op(clear_llama_cfg_called, OP_EQ, 1);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_mgr_free(mgr);
+  tor_free(msg);
+}
+
+static void
+test_confmgr_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  pasture_cfg_t *defaults = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+  char *s = NULL;
+
+  config_init(mgr, p); // set defaults.
+  config_init(mgr, defaults); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  s = config_dump(mgr, defaults, p, 1, 0);
+  tt_str_op("address 123 Camelid ave\n"
+            "alpacaname daphne\n"
+            "cuteness 42\n"
+            "llamaname hugo\n", OP_EQ, s);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_free(mgr, defaults);
+  config_mgr_free(mgr);
+
+  tor_free(msg);
+  tor_free(s);
+}
+
+#define CONFMGR_TEST(name, flags)                       \
+  { #name, test_confmgr_ ## name, flags, NULL, NULL }
+
+struct testcase_t confmgr_tests[] = {
+  CONFMGR_TEST(init, 0),
+  CONFMGR_TEST(magic, 0),
+  CONFMGR_TEST(parse, 0),
+  CONFMGR_TEST(dump, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c
new file mode 100644
index 0000000000..5f29a22c10
--- /dev/null
+++ b/src/test/test_confparse.c
@@ -0,0 +1,1086 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confparse.c module that we use to parse various
+ * configuration/state file types.
+ */
+
+#define CONFPARSE_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "feature/nodelist/routerset.h"
+#include "lib/confmgt/confparse.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "lib/confmgt/unitparse.h"
+
+typedef struct test_struct_t {
+  uint32_t magic;
+  char *s;
+  char *fn;
+  int pos;
+  int i;
+  int deprecated_int;
+  uint64_t u64;
+  int interval;
+  int msec_interval;
+  uint64_t mem;
+  double dbl;
+  int boolean;
+  int autobool;
+  time_t time;
+  smartlist_t *csv;
+  int csv_interval;
+  config_line_t *lines;
+  config_line_t *mixed_lines;
+  routerset_t *routerset;
+  int hidden_int;
+  config_line_t *mixed_hidden_lines;
+
+  config_line_t *extra_lines;
+} test_struct_t;
+
+static test_struct_t test_struct_t_dummy;
+
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
+  VAR(#member, conftype, member, initvalue)
+#define OBSOLETE(varname)                       \
+  CONFIG_VAR_OBSOLETE(varname)
+
+static const config_var_t test_vars[] = {
+  V(s, STRING, "hello"),
+  V(fn, FILENAME, NULL),
+  V(pos, POSINT, NULL),
+  V(i, INT, "-10"),
+  V(deprecated_int, INT, "3"),
+  V(u64, UINT64, NULL),
+  V(interval, INTERVAL, "10 seconds"),
+  V(msec_interval, MSEC_INTERVAL, "150 msec"),
+  V(mem, MEMUNIT, "10 MB"),
+  V(dbl, DOUBLE, NULL),
+  V(boolean, BOOL, "0"),
+  V(autobool, AUTOBOOL, "auto"),
+  V(time, ISOTIME, NULL),
+  V(csv, CSV, NULL),
+  V(csv_interval, CSV_INTERVAL, "5 seconds"),
+  V(lines, LINELIST, NULL),
+  VAR("MixedLines", LINELIST_V, mixed_lines, NULL),
+  VAR("LineTypeA", LINELIST_S, mixed_lines, NULL),
+  VAR("LineTypeB", LINELIST_S, mixed_lines, NULL),
+  OBSOLETE("obsolete"),
+  {
+   .member = { .name = "routerset",
+               .type = CONFIG_TYPE_EXTENDED,
+               .type_def = &ROUTERSET_type_defn,
+               .offset = offsetof(test_struct_t, routerset),
+             },
+  },
+  VAR("__HiddenInt", POSINT, hidden_int, "0"),
+  VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL),
+  VAR("__HiddenLineA", LINELIST_S, mixed_hidden_lines, NULL),
+  VAR("VisibleLineB", LINELIST_S, mixed_hidden_lines, NULL),
+
+  END_OF_CONFIG_VARS,
+};
+
+static config_abbrev_t test_abbrevs[] = {
+  { "uint", "pos", 0, 0 },
+  { "float", "dbl", 0, 1 },
+  { NULL, NULL, 0, 0 }
+};
+
+static config_deprecation_t test_deprecation_notes[] = {
+  { "deprecated_int", "This integer is deprecated." },
+  { NULL, NULL }
+};
+
+static int
+test_validate_cb(void *old_options, void *options, void *default_options,
+                 int from_setconf, char **msg)
+{
+  (void)old_options;
+  (void)default_options;
+  (void)from_setconf;
+  (void)msg;
+  test_struct_t *ts = options;
+
+  if (ts->i == 0xbad) {
+    *msg = tor_strdup("bad value for i");
+    return -1;
+  }
+  return 0;
+}
+
+#define TEST_MAGIC 0x1337
+
+static const config_format_t test_fmt = {
+  sizeof(test_struct_t),
+  {
+   "test_struct_t",
+   TEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  test_abbrevs,
+  test_deprecation_notes,
+  test_vars,
+  test_validate_cb,
+  NULL,
+  NULL,
+  -1,
+};
+
+/* Make sure that config_init sets everything to the right defaults. */
+static void
+test_confparse_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_init(mgr, tst);
+
+  // Make sure that options are initialized right. */
+  tt_str_op(tst->s, OP_EQ, "hello");
+  tt_ptr_op(tst->fn, OP_EQ, NULL);
+  tt_int_op(tst->pos, OP_EQ, 0);
+  tt_int_op(tst->i, OP_EQ, -10);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, 0);
+  tt_int_op(tst->interval, OP_EQ, 10);
+  tt_int_op(tst->msec_interval, OP_EQ, 150);
+  tt_u64_op(tst->mem, OP_EQ, 10 * 1024 * 1024);
+  tt_double_op(tst->dbl, OP_LT, .0000000001);
+  tt_double_op(tst->dbl, OP_GT, -0.0000000001);
+  tt_int_op(tst->boolean, OP_EQ, 0);
+  tt_int_op(tst->autobool, OP_EQ, -1);
+  tt_i64_op(tst->time, OP_EQ, 0);
+  tt_ptr_op(tst->csv, OP_EQ, NULL);
+  tt_int_op(tst->csv_interval, OP_EQ, 5);
+  tt_ptr_op(tst->lines, OP_EQ, NULL);
+  tt_ptr_op(tst->mixed_lines, OP_EQ, NULL);
+  tt_int_op(tst->hidden_int, OP_EQ, 0);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static const char simple_settings[] =
+      "s this is a \n"
+      "fn /simple/test of the\n"
+      "uint 77\n" // this is an abbrev
+      "i 3\n"
+      "u64   1000000000000  \n"
+      "interval 5 minutes \n"
+      "msec_interval 5 minutes \n"
+      "mem 10\n"
+      "dbl 6.060842\n"
+      "BOOLEAN 1\n"
+      "aUtObOOl 0\n"
+      "time 2019-06-14 13:58:51\n"
+      "csv configuration, parsing  , system  \n"
+      "csv_interval 10 seconds, 5 seconds, 10 hours\n"
+      "lines hello\n"
+      "LINES world\n"
+      "linetypea i d\n"
+      "linetypeb i c\n"
+      "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+      "__hiddenint 11\n"
+      "__hiddenlineA XYZ\n"
+      "visiblelineB ABC\n";
+
+/* Return a configuration object set up from simple_settings above. */
+static test_struct_t *
+get_simple_config(const config_mgr_t *mgr)
+{
+  test_struct_t *result = NULL;
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(simple_settings, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  result = tst;
+  tst = NULL; // prevent free
+ done:
+  tor_free(msg);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  return result;
+}
+
+/* Make sure that config_assign can parse things. */
+static void
+test_confparse_assign_simple(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  tt_str_op(tst->s, OP_EQ, "this is a");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the");
+  tt_int_op(tst->pos, OP_EQ, 77);
+  tt_int_op(tst->i, OP_EQ, 3);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, UINT64_C(1000000000000));
+  tt_int_op(tst->interval, OP_EQ, 5 * 60);
+  tt_int_op(tst->msec_interval, OP_EQ, 5 * 60 * 1000);
+  tt_u64_op(tst->mem, OP_EQ, 10);
+  tt_double_op(tst->dbl, OP_LT, 6.060843);
+  tt_double_op(tst->dbl, OP_GT, 6.060841);
+  tt_int_op(tst->boolean, OP_EQ, 1);
+  tt_int_op(tst->autobool, OP_EQ, 0);
+  tt_i64_op(tst->time, OP_EQ, 1560520731);
+  tt_ptr_op(tst->csv, OP_NE, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 3);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "configuration");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "parsing");
+  tt_str_op(smartlist_get(tst->csv, 2), OP_EQ, "system");
+  tt_int_op(tst->csv_interval, OP_EQ, 10);
+  tt_int_op(tst->hidden_int, OP_EQ, 11);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(!tst->lines->next->next);
+
+  tt_assert(tst->mixed_lines);
+  tt_str_op(tst->mixed_lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(tst->mixed_lines->value, OP_EQ, "i d");
+  tt_assert(tst->mixed_lines->next);
+  tt_str_op(tst->mixed_lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(tst->mixed_lines->next->value, OP_EQ, "i c");
+  tt_assert(!tst->mixed_lines->next->next);
+
+  tt_assert(tst->mixed_hidden_lines);
+  tt_str_op(tst->mixed_hidden_lines->key, OP_EQ, "__HiddenLineA");
+  tt_str_op(tst->mixed_hidden_lines->value, OP_EQ, "XYZ");
+  tt_assert(tst->mixed_hidden_lines->next);
+  tt_str_op(tst->mixed_hidden_lines->next->key, OP_EQ, "VisibleLineB");
+  tt_str_op(tst->mixed_hidden_lines->next->value, OP_EQ, "ABC");
+  tt_assert(!tst->mixed_hidden_lines->next->next);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an obsolete option, and make sure we get a warning. */
+static void
+test_confparse_assign_obsolete(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("obsolete option here",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("Skipping obsolete configuration option");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an deprecated option, and make sure we get a warning
+ * but the assignment works anyway. */
+static void
+test_confparse_assign_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("deprecated_int 7",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("This integer is deprecated.");
+
+  tt_int_op(tst->deprecated_int, OP_EQ, 7);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to re-assign an option name that has been depreacted in favor of
+ * another. */
+static void
+test_confparse_assign_replaced(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("float 1000\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("use 'dbl' instead.");
+
+  tt_double_op(tst->dbl, OP_GT, 999.999);
+  tt_double_op(tst->dbl, OP_LT, 1000.001);
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a linelist value with no option. */
+static void
+test_confparse_assign_emptystring(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("lines\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("has no value");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a the same option twice; make sure we get a warning. */
+static void
+test_confparse_assign_twice(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("pos 10\n"
+                           "pos 99\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("used more than once");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+typedef struct badval_test_t {
+  const char *cfg;
+  const char *expect_msg;
+} badval_test_t;
+
+/* Try to set an option and make sure that we get a failure and an expected
+ * warning. */
+static void
+test_confparse_assign_badval(void *arg)
+{
+  const badval_test_t *bt = arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(bt->cfg, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  if (! strstr(msg, bt->expect_msg)) {
+    TT_DIE(("'%s' did not contain '%s'" , msg, bt->expect_msg));
+  }
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Various arguments for badval test.
+ *
+ * Note that the expected warnings here are _very_ truncated, since we
+ * are writing these tests before a refactoring that we expect will
+ * change them.
+ */
+static const badval_test_t bv_notint = { "pos X\n", "malformed" };
+static const badval_test_t bv_negint = { "pos -10\n", "out of bounds" };
+static const badval_test_t bv_badu64 = { "u64 u64\n", "malformed" };
+static const badval_test_t bv_dbl1 = { "dbl xxx\n", "Could not convert" };
+static const badval_test_t bv_dbl2 = { "dbl 1.0 xx\n", "Could not convert" };
+static const badval_test_t bv_dbl3 = {
+   "dbl 1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl4 = {
+   "dbl 1e1000\n", "too large to express" };
+static const badval_test_t bv_dbl5 = {
+   "dbl -1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl6 = {
+   "dbl -1e1000\n", "too large to express" };
+static const badval_test_t bv_badcsvi1 =
+  { "csv_interval 10 wl\n", "malformed" };
+static const badval_test_t bv_badcsvi2 =
+  { "csv_interval cl,10\n", "malformed" };
+static const badval_test_t bv_nonoption = { "fnord 10\n", "Unknown option" };
+static const badval_test_t bv_badmem = { "mem 3 trits\n", "malformed" };
+static const badval_test_t bv_badbool = { "boolean 7\n", "Unrecognized value"};
+static const badval_test_t bv_badabool =
+  { "autobool 7\n", "Unrecognized value" };
+static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" };
+static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" };
+static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" };
+static const badval_test_t bv_big_interval =
+  { "interval 1000 months", "too large" };
+
+/* Try config_dump(), and make sure it behaves correctly */
+static void
+test_confparse_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  char *dumped = NULL;
+
+  /* Minimal version. */
+  dumped = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  /* commented */
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 1);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "# deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+ done:
+  config_free(mgr, tst);
+  tor_free(dumped);
+  config_mgr_free(mgr);
+}
+
+/* Try confparse_reset_line(), and make sure it behaves correctly */
+static void
+test_confparse_reset(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  config_reset_line(mgr, tst, "interval", 0);
+  tt_int_op(tst->interval, OP_EQ, 0);
+
+  config_reset_line(mgr, tst, "interval", 1);
+  tt_int_op(tst->interval, OP_EQ, 10);
+
+  tt_ptr_op(tst->routerset, OP_NE, NULL);
+  config_reset_line(mgr, tst, "routerset", 0);
+  tt_ptr_op(tst->routerset, OP_EQ, NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, and make sure
+ * it behaves correctly. */
+static void
+test_confparse_reassign(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *rs = NULL;
+
+  int r = config_get_lines(
+         "s eleven\n"
+         "i 12\n"
+         "lines 13\n"
+         "csv 14,15\n"
+         "routerset 127.0.0.1\n",
+         &lines, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); // unchanged
+  tt_int_op(tst->pos, OP_EQ, 77); // unchanged
+  tt_int_op(tst->i, OP_EQ, 12);
+  tt_ptr_op(tst->lines, OP_NE, NULL);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "13");
+  tt_ptr_op(tst->lines->next, OP_EQ, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 2);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "14");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "15");
+
+  rs = routerset_to_string(tst->routerset);
+  tt_str_op(rs, OP_EQ, "127.0.0.1");
+
+  // Try again with the CLEAR_FIRST and USE_DEFAULTS flags
+  r = config_assign(mgr, tst, lines,
+                    CAL_CLEAR_FIRST|CAL_USE_DEFAULTS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  // tt_ptr_op(tst->fn, OP_EQ, NULL); //XXXX why is this not cleared?
+  // tt_int_op(tst->pos, OP_EQ, 0); //XXXX why is this not cleared?
+  tt_int_op(tst->i, OP_EQ, 12);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  tor_free(rs);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, using the +foo
+ * linelist-extending syntax. */
+static void
+test_confparse_reassign_extend(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  int r = config_get_lines(
+         "+lines 13\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(tst->lines->next->next);
+  tt_str_op(tst->lines->next->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->next->value, OP_EQ, "13");
+  tt_assert(tst->lines->next->next->next == NULL);
+  config_free_lines(lines);
+
+  r = config_get_lines(
+         "/lines\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+  config_free_lines(lines);
+
+  config_free(mgr, tst);
+  tst = get_simple_config(mgr);
+  r = config_get_lines(
+         "/lines away!\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Test out confparse_get_assigned(). */
+static void
+test_confparse_get_assigned(void *arg)
+{
+  (void)arg;
+
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+
+  lines = config_get_assigned_option(mgr, tst, "I", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "i");
+  tt_str_op(lines->value, OP_EQ, "3");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "this is a");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "obsolete", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "nonesuch", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "mixedlines", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(lines->value, OP_EQ, "i d");
+  tt_assert(lines->next);
+  tt_str_op(lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->next->value, OP_EQ, "i c");
+  tt_assert(lines->next->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "linetypeb", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->value, OP_EQ, "i c");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  tor_free(tst->s);
+  tst->s = tor_strdup("Hello\nWorld");
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "\"Hello\\nWorld\"");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  config_mgr_free(mgr);
+}
+
+/* Another variant, which accepts and stores unrecognized lines.*/
+#define ETEST_MAGIC 13371337
+
+static struct_member_t extra = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(test_struct_t, extra_lines),
+};
+
+static config_format_t etest_fmt = {
+  sizeof(test_struct_t),
+  {
+   "test_struct_t (with extra lines)",
+   ETEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  test_abbrevs,
+  test_deprecation_notes,
+  test_vars,
+  test_validate_cb,
+  NULL,
+  &extra,
+  -1,
+};
+
+/* Try out the feature where we can store unrecognized lines and dump them
+ * again.  (State files use this.) */
+static void
+test_confparse_extra_lines(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&etest_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *dump = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(
+      "unknotty addita\n"
+      "pos 99\n"
+      "wombat knish\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->extra_lines);
+
+  dump = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dump, OP_EQ,
+      "pos 99\n"
+      "unknotty addita\n"
+      "wombat knish\n");
+
+ done:
+  tor_free(msg);
+  tor_free(dump);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_unitparse(void *args)
+{
+  (void)args;
+  /* spot-check a few memunit values. */
+  int ok = 3;
+  tt_u64_op(config_parse_memunit("100 MB", &ok), OP_EQ, 100<<20);
+  tt_assert(ok);
+  tt_u64_op(config_parse_memunit("100 TB", &ok), OP_EQ, UINT64_C(100)<<40);
+  tt_assert(ok);
+  // This is a floating-point value, but note that 1.5 can be represented
+  // precisely.
+  tt_u64_op(config_parse_memunit("1.5 MB", &ok), OP_EQ, 3<<19);
+  tt_assert(ok);
+
+  /* Try some good intervals and msec intervals */
+  tt_int_op(config_parse_interval("2 days", &ok), OP_EQ, 48*3600);
+  tt_assert(ok);
+  tt_int_op(config_parse_interval("1.5 hour", &ok), OP_EQ, 5400);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("1 minute", &ok), OP_EQ, 60);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("2 days", &ok), OP_EQ, 48*3600*1000);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("10 msec", &ok), OP_EQ, 10);
+  tt_assert(ok);
+
+  /* Try a couple of unitless values. */
+  tt_int_op(config_parse_interval("10", &ok), OP_EQ, 10);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("15.0", &ok), OP_EQ, 15);
+  tt_assert(ok);
+
+  /* u64 overflow */
+  /* XXXX our implementation does not currently detect this. See bug 30920. */
+  /*
+  tt_u64_op(config_parse_memunit("20000000 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  */
+
+  /* i32 overflow */
+  tt_int_op(config_parse_interval("1000 months", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("4 weeks", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+
+  /* bad units */
+  tt_u64_op(config_parse_memunit("7 nybbles", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // XXXX these next two should return -1 according to the documentation.
+  tt_int_op(config_parse_interval("7 cowznofski", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("1 kalpa", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+ done:
+  ;
+}
+
+static void
+test_confparse_check_ok_fail(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  tst->pos = -10;
+  tt_assert(! config_check_ok(mgr, tst, LOG_INFO));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_vars(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_vars(mgr);
+  smartlist_t *varnames = smartlist_new();
+  char *joined = NULL;
+
+  tt_assert(vars);
+  SMARTLIST_FOREACH(vars, config_var_t *, cv,
+                    smartlist_add(varnames, (void*)cv->member.name));
+  smartlist_sort_strings(varnames);
+  joined = smartlist_join_strings(varnames, "::", 0, NULL);
+  tt_str_op(joined, OP_EQ,
+            "LineTypeA::"
+            "LineTypeB::"
+            "MixedHiddenLines::"
+            "MixedLines::"
+            "VisibleLineB::"
+            "__HiddenInt::"
+            "__HiddenLineA::"
+            "autobool::"
+            "boolean::"
+            "csv::"
+            "csv_interval::"
+            "dbl::"
+            "deprecated_int::"
+            "fn::"
+            "i::"
+            "interval::"
+            "lines::"
+            "mem::"
+            "msec_interval::"
+            "obsolete::"
+            "pos::"
+            "routerset::"
+            "s::"
+            "time::"
+            "u64");
+
+ done:
+  tor_free(joined);
+  smartlist_free(varnames);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_deprecated_vars(mgr);
+  char *joined = NULL;
+
+  tt_assert(vars);
+  smartlist_sort_strings(vars);
+  joined = smartlist_join_strings(vars, "::", 0, NULL);
+
+  tt_str_op(joined, OP_EQ, "deprecated_int");
+
+ done:
+  tor_free(joined);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_find_option_name(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+
+  // exact match
+  tt_str_op(config_find_option_name(mgr, "u64"), OP_EQ, "u64");
+  // case-insensitive match
+  tt_str_op(config_find_option_name(mgr, "S"), OP_EQ, "s");
+  tt_str_op(config_find_option_name(mgr, "linetypea"), OP_EQ, "LineTypeA");
+  // prefix match
+  tt_str_op(config_find_option_name(mgr, "deprec"), OP_EQ, "deprecated_int");
+  // explicit abbreviation
+  tt_str_op(config_find_option_name(mgr, "uint"), OP_EQ, "pos");
+  tt_str_op(config_find_option_name(mgr, "UINT"), OP_EQ, "pos");
+  // no match
+  tt_ptr_op(config_find_option_name(mgr, "absent"), OP_EQ, NULL);
+
+ done:
+  config_mgr_free(mgr);
+}
+
+#define CONFPARSE_TEST(name, flags)                          \
+  { #name, test_confparse_ ## name, flags, NULL, NULL }
+
+#define BADVAL_TEST(name)                               \
+  { "badval_" #name, test_confparse_assign_badval, 0,   \
+      &passthrough_setup, (void*)&bv_ ## name }
+
+struct testcase_t confparse_tests[] = {
+  CONFPARSE_TEST(init, 0),
+  CONFPARSE_TEST(assign_simple, 0),
+  CONFPARSE_TEST(assign_obsolete, 0),
+  CONFPARSE_TEST(assign_deprecated, 0),
+  CONFPARSE_TEST(assign_replaced, 0),
+  CONFPARSE_TEST(assign_emptystring, 0),
+  CONFPARSE_TEST(assign_twice, 0),
+  BADVAL_TEST(notint),
+  BADVAL_TEST(negint),
+  BADVAL_TEST(badu64),
+  BADVAL_TEST(dbl1),
+  BADVAL_TEST(dbl2),
+  BADVAL_TEST(dbl3),
+  BADVAL_TEST(dbl4),
+  BADVAL_TEST(dbl5),
+  BADVAL_TEST(dbl6),
+  BADVAL_TEST(badcsvi1),
+  BADVAL_TEST(badcsvi2),
+  BADVAL_TEST(nonoption),
+  BADVAL_TEST(badmem),
+  BADVAL_TEST(badbool),
+  BADVAL_TEST(badabool),
+  BADVAL_TEST(badtime),
+  BADVAL_TEST(virt),
+  BADVAL_TEST(rs),
+  BADVAL_TEST(big_interval),
+  CONFPARSE_TEST(dump, 0),
+  CONFPARSE_TEST(reset, 0),
+  CONFPARSE_TEST(reassign, 0),
+  CONFPARSE_TEST(reassign_extend, 0),
+  CONFPARSE_TEST(get_assigned, 0),
+  CONFPARSE_TEST(extra_lines, 0),
+  CONFPARSE_TEST(unitparse, 0),
+  CONFPARSE_TEST(check_ok_fail, 0),
+  CONFPARSE_TEST(list_vars, 0),
+  CONFPARSE_TEST(list_deprecated, 0),
+  CONFPARSE_TEST(find_option_name, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_connection.h b/src/test/test_connection.h
index 47a5599e5f..40121e6d38 100644
--- a/src/test/test_connection.h
+++ b/src/test/test_connection.h
@@ -1,6 +1,9 @@
 /* Copyright (c) 2014-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_CONNECTION_H
+#define TOR_TEST_CONNECTION_H
+
 /** Some constants used by test_connection and helpers */
 #define TEST_CONN_FAMILY        (AF_INET)
 #define TEST_CONN_ADDRESS       "127.0.0.1"
@@ -11,3 +14,4 @@
 void test_conn_lookup_addr_helper(const char *address,
                                   int family, tor_addr_t *addr);
 
+#endif /* !defined(TOR_TEST_CONNECTION_H) */
diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c
index 682ba5b970..7c4c92ea42 100644
--- a/src/test/test_consdiff.c
+++ b/src/test/test_consdiff.c
@@ -14,6 +14,39 @@
 #define tt_str_eq_line(a,b) \
   tt_assert(line_str_eq((b),(a)))
 
+static int
+consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area)
+{
+  size_t len = strlen(s);
+  return consensus_split_lines(out, s, len, area);
+}
+
+static int
+consensus_compute_digest_(const char *cons,
+                          consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
+static int
+consensus_compute_digest_as_signed_(const char *cons,
+                                    consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest_as_signed(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
 static void
 test_consdiff_smartlist_slice(void *arg)
 {
@@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg)
 
   /* Create a regular smartlist. */
   (void)arg;
-  consensus_split_lines(sl, "a\nd\nc\na\nb\n", area);
+  consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area);
 
   /* See that smartlist_slice_string_pos respects the bounds of the slice. */
   sls = smartlist_slice(sl, 2, 5);
@@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg)
   int e_lengths2[] = { 0, 1, 1, 2, 3, 4 };
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area);
-  consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area);
+  consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area);
+  consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area);
-  consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area);
-  consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area);
-  consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area);
+  consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area);
+  consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area);
+  consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area);
+  consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area);
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
   sls3 = smartlist_slice(sl3, 0, -1);
@@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\nb\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   /* Length of sls1 is 0. */
   sls1 = smartlist_slice(sl1, 0, 0);
@@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\na\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\na\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg)
   tt_assert(!bitarray_is_set(changed2, 3));
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "a\nb\na\nb\n", area);
+  consensus_split_lines_(sl2, "a\nb\na\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg)
   bitarray_clear(changed1, 3);
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "b\nb\nb\nb\n", area);
+  consensus_split_lines_(sl2, "b\nb\nb\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg)
   /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
-  consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(7, OP_EQ, smartlist_len(diff));
@@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg)
 
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "B\n", area);
-  consensus_split_lines(cons2, "A\nB\n", area);
+  consensus_split_lines_(cons1, "B\n", area);
+  consensus_split_lines_(cons2, "A\nB\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(3, OP_EQ, smartlist_len(diff));
@@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg)
   diff = smartlist_new();
   setup_capture_of_logs(LOG_WARN);
 
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
 
   /* Command without range. */
   smartlist_add_linecpy(diff, area, "a");
@@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_clear(diff);
 
   /* Test appending text, 'a'. */
-  consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
+  consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(8, OP_EQ, smartlist_len(cons2));
@@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test deleting text, 'd'. */
-  consensus_split_lines(diff, "4d\n1,2d\n", area);
+  consensus_split_lines_(diff, "4d\n1,2d\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(2, OP_EQ, smartlist_len(cons2));
@@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test changing text, 'c'. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(5, OP_EQ, smartlist_len(cons2));
@@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test 'a', 'd' and 'c' together. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(6, OP_EQ, smartlist_len(cons2));
@@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg)
       );
 
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons2_str, &digests2));
+      consensus_compute_digest_(cons2_str, &digests2));
 
-  consensus_split_lines(cons1, cons1_str, area);
-  consensus_split_lines(cons2, cons2_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
+  consensus_split_lines_(cons2, cons2_str, area);
 
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_EQ, diff);
@@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   smartlist_clear(cons1);
-  consensus_split_lines(cons1, cons1_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(11, OP_EQ, smartlist_len(diff));
@@ -991,8 +1024,8 @@ test_consdiff_apply_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons1_str, &digests1));
-  consensus_split_lines(cons1, cons1_str, area);
+      consensus_compute_digest_(cons1_str, &digests1));
+  consensus_split_lines_(cons1, cons1_str, area);
 
   /* diff doesn't have enough lines. */
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
@@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = {
   CONSDIFF_LEGACY(apply_diff),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c
index 254a5ba5d0..74226b8c52 100644
--- a/src/test/test_consdiffmgr.c
+++ b/src/test/test_consdiffmgr.c
@@ -21,6 +21,23 @@
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
+static char *
+consensus_diff_apply_(const char *c, const char *d)
+{
+  size_t c_len = strlen(c);
+  size_t d_len = strlen(d);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  char *c_tmp = tor_memdup(c, c_len);
+  char *d_tmp = tor_memdup(d, d_len);
+  char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len);
+  tor_free(c_tmp);
+  tor_free(d_tmp);
+  return result;
+}
+
 // ============================== Setup/teardown the consdiffmgr
 // These functions get run before/after each test in this module
 
@@ -153,7 +170,8 @@ lookup_diff_from(consensus_cache_entry_t **out,
                  const char *str1)
 {
   uint8_t digest[DIGEST256_LEN];
-  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+  if (router_get_networkstatus_v3_sha3_as_signed(digest,
+                                                 str1, strlen(str1))<0) {
     TT_FAIL(("Unable to compute sha3-as-signed"));
     return CONSDIFF_NOT_FOUND;
   }
@@ -175,14 +193,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav,
 
   consensus_cache_entry_incref(ent);
   size_t size;
-  char *diff_string = NULL;
-  int r = uncompress_or_copy(&diff_string, &size, ent);
+  const char *diff_string = NULL;
+  char *diff_owned = NULL;
+  int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent);
   consensus_cache_entry_decref(ent);
   if (diff_string == NULL || r < 0)
     return -1;
 
-  char *applied = consensus_diff_apply(str1, diff_string);
-  tor_free(diff_string);
+  char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size);
+  tor_free(diff_owned);
   if (applied == NULL)
     return -1;
 
@@ -282,7 +301,8 @@ test_consdiffmgr_add(void *arg)
   (void) arg;
   time_t now = approx_time();
 
-  char *body = NULL;
+  const char *body = NULL;
+  char *body_owned = NULL;
 
   consensus_cache_entry_t *ent = NULL;
   networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
@@ -324,7 +344,7 @@ test_consdiffmgr_add(void *arg)
   tt_assert(ent);
   consensus_cache_entry_incref(ent);
   size_t s;
-  r = uncompress_or_copy(&body, &s, ent);
+  r = uncompress_or_set_ptr(&body, &s, &body_owned, ent);
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(s, OP_EQ, 4);
   tt_mem_op(body, OP_EQ, "quux", 4);
@@ -337,7 +357,7 @@ test_consdiffmgr_add(void *arg)
   networkstatus_vote_free(ns_tmp);
   teardown_capture_of_logs();
   consensus_cache_entry_decref(ent);
-  tor_free(body);
+  tor_free(body_owned);
 }
 
 static void
@@ -370,7 +390,8 @@ test_consdiffmgr_make_diffs(void *arg)
   ns = fake_ns_new(FLAV_MICRODESC, now-3600);
   md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
   r = consdiffmgr_add_consensus(md_ns_body, ns);
-  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body,
+                                             strlen(md_ns_body));
   networkstatus_vote_free(ns);
   tt_int_op(r, OP_EQ, 0);
 
@@ -414,7 +435,7 @@ test_consdiffmgr_make_diffs(void *arg)
   r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
   tt_int_op(r, OP_EQ, 0);
   diff_text = tor_memdup_nulterm(diff_body, diff_size);
-  applied = consensus_diff_apply(md_ns_body, diff_text);
+  applied = consensus_diff_apply_(md_ns_body, diff_text);
   tt_assert(applied);
   tt_str_op(applied, OP_EQ, md_ns_body_2);
 
diff --git a/src/test/test_containers.c b/src/test/test_containers.c
index aedd2f7a89..67ba457975 100644
--- a/src/test/test_containers.c
+++ b/src/test/test_containers.c
@@ -96,6 +96,30 @@ test_container_smartlist_basic(void *arg)
   tor_free(v555);
 }
 
+/** Test SMARTLIST_FOREACH_REVERSE_BEGIN loop macro */
+static void
+test_container_smartlist_foreach_reverse(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  int i;
+
+  (void) arg;
+
+  /* Add integers to smartlist in increasing order */
+  for (i=0;i<100;i++) {
+    smartlist_add(sl, (void*)(uintptr_t)i);
+  }
+
+  /* Pop them out in reverse and test their value */
+  SMARTLIST_FOREACH_REVERSE_BEGIN(sl, void*, k) {
+    i--;
+    tt_ptr_op(k, OP_EQ, (void*)(uintptr_t)i);
+  } SMARTLIST_FOREACH_END(k);
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for smartlist-of-strings functionality. */
 static void
 test_container_smartlist_strings(void *arg)
@@ -582,6 +606,66 @@ test_container_smartlist_ints_eq(void *arg)
   smartlist_free(sl2);
 }
 
+static void
+test_container_smartlist_grow(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  int i;
+  const char *s[] = { "first", "2nd", "3rd" };
+
+  /* case 1: starting from empty. */
+  smartlist_grow(sl, 10);
+  tt_int_op(10, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 10; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 2: starting with a few elements, probably not reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 5);
+  tt_int_op(5, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  tt_ptr_op(smartlist_get(sl, 3), OP_EQ, NULL);
+  tt_ptr_op(smartlist_get(sl, 4), OP_EQ, NULL);
+
+  /* case 3: starting with a few elements, but reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 100);
+  tt_int_op(100, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  for (i = 3; i < 100; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 4: shrinking doesn't happen. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 1);
+  tt_int_op(3, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for bitarray code */
 static void
 test_container_bitarray(void *arg)
@@ -922,6 +1006,10 @@ test_container_smartlist_remove(void *arg)
   tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]);
   tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]);
   tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]);
+  /* Ordinary code should never look at this pointer; we're doing it here
+   * to make sure that we really cleared the pointer we removed.
+   */
+  tt_ptr_op(sl->list[4], OP_EQ, NULL);
 
  done:
   smartlist_free(sl);
@@ -1281,12 +1369,14 @@ test_container_smartlist_strings_eq(void *arg)
 struct testcase_t container_tests[] = {
   CONTAINER_LEGACY(smartlist_basic),
   CONTAINER_LEGACY(smartlist_strings),
+  CONTAINER_LEGACY(smartlist_foreach_reverse),
   CONTAINER_LEGACY(smartlist_overlap),
   CONTAINER_LEGACY(smartlist_digests),
   CONTAINER_LEGACY(smartlist_join),
   CONTAINER_LEGACY(smartlist_pos),
   CONTAINER(smartlist_remove, 0),
   CONTAINER(smartlist_ints_eq, 0),
+  CONTAINER(smartlist_grow, 0),
   CONTAINER_LEGACY(bitarray),
   CONTAINER_LEGACY(digestset),
   CONTAINER_LEGACY(strmap),
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index 5b406e159b..55eb79e448 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -1,11 +1,15 @@
 /* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define CONTROL_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "feature/client/bridges.h"
 #include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hs/hs_common.h"
 #include "feature/nodelist/networkstatus.h"
@@ -15,48 +19,255 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "lib/net/resolve.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
 #include "feature/dirclient/download_status_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/node_st.h"
 
+typedef struct {
+  const char *input;
+  const char *expected_parse;
+  const char *expected_error;
+} parser_testcase_t;
+
+typedef struct {
+  const control_cmd_syntax_t *syntax;
+  size_t n_testcases;
+  const parser_testcase_t *testcases;
+} parse_test_params_t;
+
+static char *
+control_cmd_dump_args(const control_cmd_args_t *result)
+{
+  buf_t *buf = buf_new();
+  buf_add_string(buf, "{ args=[");
+  if (result->args) {
+    if (smartlist_len(result->args)) {
+        buf_add_string(buf, " ");
+    }
+    SMARTLIST_FOREACH_BEGIN(result->args, const char *, s) {
+      const bool last = (s_sl_idx == smartlist_len(result->args)-1);
+      buf_add_printf(buf, "%s%s ",
+                     escaped(s),
+                     last ? "" : ",");
+    } SMARTLIST_FOREACH_END(s);
+  }
+  buf_add_string(buf, "]");
+  if (result->cmddata) {
+    buf_add_string(buf, ", obj=");
+    buf_add_string(buf, escaped(result->cmddata));
+  }
+  if (result->kwargs) {
+    buf_add_string(buf, ", { ");
+    const config_line_t *line;
+    for (line = result->kwargs; line; line = line->next) {
+      const bool last = (line->next == NULL);
+      buf_add_printf(buf, "%s=%s%s ", line->key, escaped(line->value),
+                     last ? "" : ",");
+    }
+    buf_add_string(buf, "}");
+  }
+  buf_add_string(buf, " }");
+
+  char *encoded = buf_extract(buf, NULL);
+  buf_free(buf);
+  return encoded;
+}
+
+static void
+test_controller_parse_cmd(void *arg)
+{
+  const parse_test_params_t *params = arg;
+  control_cmd_args_t *result = NULL;
+  char *error = NULL;
+  char *encoded = NULL;
+
+  for (size_t i = 0; i < params->n_testcases; ++i) {
+    const parser_testcase_t *t = &params->testcases[i];
+    result = control_cmd_parse_args("EXAMPLE",
+                                    params->syntax,
+                                    strlen(t->input),
+                                    t->input,
+                                    &error);
+    // A valid test should expect exactly one parse or error.
+    tt_int_op((t->expected_parse == NULL), OP_NE,
+              (t->expected_error == NULL));
+    // We get a result or an error, not both.
+    tt_int_op((result == NULL), OP_EQ, (error != NULL));
+    // We got the one we expected.
+    tt_int_op((result == NULL), OP_EQ, (t->expected_parse == NULL));
+
+    if (result) {
+      encoded = control_cmd_dump_args(result);
+      tt_str_op(encoded, OP_EQ, t->expected_parse);
+    } else {
+      tt_str_op(error, OP_EQ, t->expected_error);
+    }
+
+    tor_free(error);
+    tor_free(encoded);
+    control_cmd_args_free(result);
+  }
+
+ done:
+  tor_free(error);
+  tor_free(encoded);
+  control_cmd_args_free(result);
+}
+
+#define OK(inp, out) \
+  { inp "\r\n", out, NULL }
+#define ERR(inp, err) \
+  { inp "\r\n", NULL, err }
+
+#define TESTPARAMS(syntax, array)                \
+  { &syntax,                                     \
+      ARRAY_LENGTH(array),                       \
+      array }
+
+static const parser_testcase_t one_to_three_tests[] = {
+   ERR("", "Need at least 1 argument(s)"),
+   ERR("   \t", "Need at least 1 argument(s)"),
+   OK("hello", "{ args=[ \"hello\" ] }"),
+   OK("hello world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world      ", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello there world", "{ args=[ \"hello\", \"there\", \"world\" ] }"),
+   ERR("why hello there world", "Cannot accept more than 3 argument(s)"),
+   ERR("hello\r\nworld.\r\n.", "Unexpected body"),
+};
+
+static const control_cmd_syntax_t one_to_three_syntax = {
+   .min_args=1, .max_args=3
+};
+
+static const parse_test_params_t parse_one_to_three_params =
+  TESTPARAMS( one_to_three_syntax, one_to_three_tests );
+
+// =
+static const parser_testcase_t no_args_one_obj_tests[] = {
+  ERR("Hi there!\r\n.", "Cannot accept more than 0 argument(s)"),
+  ERR("", "Empty body"),
+  OK("\r\n", "{ args=[], obj=\"\\n\" }"),
+  OK("\r\nHello world\r\n", "{ args=[], obj=\"Hello world\\n\\n\" }"),
+  OK("\r\nHello\r\nworld\r\n", "{ args=[], obj=\"Hello\\nworld\\n\\n\" }"),
+  OK("\r\nHello\r\n..\r\nworld\r\n",
+     "{ args=[], obj=\"Hello\\n.\\nworld\\n\\n\" }"),
+};
+static const control_cmd_syntax_t no_args_one_obj_syntax = {
+   .min_args=0, .max_args=0,
+   .want_cmddata=true,
+};
+static const parse_test_params_t parse_no_args_one_obj_params =
+  TESTPARAMS( no_args_one_obj_syntax, no_args_one_obj_tests );
+
+static const parser_testcase_t no_args_kwargs_tests[] = {
+  OK("", "{ args=[] }"),
+  OK(" ", "{ args=[] }"),
+  OK("hello there=world", "{ args=[], { hello=\"\", there=\"world\" } }"),
+  OK("hello there=world today",
+     "{ args=[], { hello=\"\", there=\"world\", today=\"\" } }"),
+  ERR("=Foo", "Cannot parse keyword argument(s)"),
+};
+static const control_cmd_syntax_t no_args_kwargs_syntax = {
+   .min_args=0, .max_args=0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_VALS
+};
+static const parse_test_params_t parse_no_args_kwargs_params =
+  TESTPARAMS( no_args_kwargs_syntax, no_args_kwargs_tests );
+
+static const char *one_arg_kwargs_allow_keywords[] = {
+  "Hello", "world", NULL
+};
+static const parser_testcase_t one_arg_kwargs_tests[] = {
+  ERR("", "Need at least 1 argument(s)"),
+  OK("Hi", "{ args=[ \"Hi\" ] }"),
+  ERR("hello there=world", "Unrecognized keyword argument \"there\""),
+  OK("Hi HELLO=foo", "{ args=[ \"Hi\" ], { HELLO=\"foo\" } }"),
+  OK("Hi world=\"bar baz\" hello  ",
+     "{ args=[ \"Hi\" ], { world=\"bar baz\", hello=\"\" } }"),
+};
+static const control_cmd_syntax_t one_arg_kwargs_syntax = {
+   .min_args=1, .max_args=1,
+   .accept_keywords=true,
+   .allowed_keywords=one_arg_kwargs_allow_keywords,
+   .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+static const parse_test_params_t parse_one_arg_kwargs_params =
+  TESTPARAMS( one_arg_kwargs_syntax, one_arg_kwargs_tests );
+
+static char *reply_str = NULL;
+/* Mock for control_write_reply that copies the string for inspection
+ * by tests */
+static void
+mock_control_write_reply(control_connection_t *conn, int code, int c,
+                                const char *s)
+{
+  (void)conn;
+  (void)code;
+  (void)c;
+  tor_free(reply_str);
+  reply_str = tor_strdup(s);
+}
+
 static void
 test_add_onion_helper_keyarg_v3(void *arg)
 {
   int ret, hs_version;
   add_onion_secret_key_t pk;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   const char *key_new_alg = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit ED25519-V3 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
+  /* Test "BEST" key generation (Assumes BEST = ED25519-V3). */
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, NULL);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
+  tt_assert(key_new_blob);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
+
   /* Test discarding the private key. */
+  tor_free(reply_str);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
@@ -76,9 +287,10 @@ test_add_onion_helper_keyarg_v3(void *arg)
 
     tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk);
     tt_assert(key_blob);
+    tor_free(reply_str);
     ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg,
                                   &key_new_blob, &pk, &hs_version,
-                                  &err_msg);
+                                  NULL);
     tor_free(key_blob);
     tt_int_op(ret, OP_EQ, 0);
     tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
@@ -86,7 +298,7 @@ test_add_onion_helper_keyarg_v3(void *arg)
     tt_mem_op(pk.v3, OP_EQ, hex_sk, 64);
     tt_ptr_op(key_new_alg, OP_EQ, NULL);
     tt_ptr_op(key_new_blob, OP_EQ, NULL);
-    tt_ptr_op(err_msg, OP_EQ, NULL);
+    tt_ptr_op(reply_str, OP_EQ, NULL);
     tor_free(pk.v3); pk.v3 = NULL;
     tor_free(key_new_blob);
   }
@@ -94,7 +306,8 @@ test_add_onion_helper_keyarg_v3(void *arg)
  done:
   tor_free(pk.v3);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -105,72 +318,61 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_t *pk1 = NULL;
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   char *encoded = NULL;
   char *arg_str = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit RSA1024 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
-
-  /* Test "BEST" key generation (Assumes BEST = RSA1024). */
-  crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
-  tt_assert(pk.v2);
-  tt_str_op(key_new_alg, OP_EQ, "RSA1024");
-  tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test discarding the private key. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+  ret = add_onion_helper_keyarg("NEW:RSA1024", 1, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test generating a invalid key type. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a RSA1024 key. */
-  tor_free(err_msg);
+  tor_free(reply_str);
   pk1 = pk_generate(0);
   tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded));
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0);
 
   /* Test loading a invalid key type. */
@@ -179,36 +381,37 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_asprintf(&arg_str, "RSA512:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a invalid key. */
   tor_free(arg_str);
   crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(err_msg);
+  tor_free(reply_str);
   encoded[strlen(encoded)/2] = '\0';
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
  done:
   crypto_pk_free(pk1);
   crypto_pk_free(pk.v2);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
   tor_free(encoded);
   tor_free(arg_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -362,49 +565,52 @@ static void
 test_add_onion_helper_clientauth(void *arg)
 {
   rend_authorized_client_t *client = NULL;
-  char *err_msg = NULL;
   int created = 0;
 
   (void)arg;
 
+  MOCK(control_write_reply, mock_control_write_reply);
   /* Test "ClientName" only. */
-  client = add_onion_helper_clientauth("alice", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice", &created, NULL);
   tt_assert(client);
   tt_assert(created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test "ClientName:Blob" */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B",
-                                       &created, &err_msg);
+                                       &created, NULL);
   tt_assert(client);
   tt_assert(!created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test invalid client names */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("no*asterisks*allowed", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid auth cookie */
-  client = add_onion_helper_clientauth("alice:12345", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice:12345", &created, NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid syntax */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
  done:
   rend_authorized_client_free(client);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 /* Mocks and data/variables used for GETINFO download status tests */
@@ -1543,7 +1749,7 @@ test_current_time(void *arg)
 static size_t n_nodelist_get_list = 0;
 static smartlist_t *nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 mock_nodelist_get_list(void)
 {
   n_nodelist_get_list++;
@@ -1614,7 +1820,15 @@ test_getinfo_md_all(void *arg)
   return;
 }
 
+#define PARSER_TEST(type)                                             \
+  { "parse/" #type, test_controller_parse_cmd, 0, &passthrough_setup, \
+      (void*)&parse_ ## type ## _params }
+
 struct testcase_t controller_tests[] = {
+  PARSER_TEST(one_to_three),
+  PARSER_TEST(no_args_one_obj),
+  PARSER_TEST(no_args_kwargs),
+  PARSER_TEST(one_arg_kwargs),
   { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0,
     NULL, NULL },
   { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0,
diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c
index 70d36e53d4..0d276ef8a6 100644
--- a/src/test/test_controller_events.c
+++ b/src/test/test_controller_events.c
@@ -4,13 +4,21 @@
 #define CONNECTION_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "app/main/subsysmgr.h"
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitlist.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
 
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -351,10 +359,10 @@ test_cntev_dirboot_defer_desc(void *arg)
   /* This event should get deferred */
   control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
   assert_bootmsg("45 TAG=requesting_descriptors");
@@ -374,29 +382,215 @@ test_cntev_dirboot_defer_orconn(void *arg)
   control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
   assert_bootmsg("0 TAG=starting");
   /* This event should get deferred */
-  control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+  control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
-  assert_bootmsg("80 TAG=conn_or");
+  assert_bootmsg("75 TAG=enough_dirinfo");
  done:
   tor_free(saved_event_str);
   UNMOCK(queue_control_event_string);
 }
 
-#define TEST(name, flags)                                               \
+static void
+test_cntev_signal(void *arg)
+{
+  (void)arg;
+  int rv;
+
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+
+  /* Nothing is listening for signals, so no event should be queued. */
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+
+  /* Now try with signals included in the event mask. */
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_GOT_SIGNAL));
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL RELOAD\r\n");
+
+  rv = control_event_signal(SIGACTIVE);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL ACTIVE\r\n");
+
+  /* Try a signal that doesn't exist. */
+  setup_full_capture_of_logs(LOG_WARN);
+  tor_free(saved_event_str);
+  rv = control_event_signal(99999);
+  tt_int_op(-1, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+  expect_single_log_msg_containing("Unrecognized signal 99999");
+
+ done:
+  tor_free(saved_event_str);
+ teardown_capture_of_logs();
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+setup_orconn_state(orconn_state_msg_t *msg, uint64_t gid, uint64_t chan,
+                   int proxy_type)
+{
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->proxy_type = proxy_type;
+}
+
+static void
+send_orconn_state(const orconn_state_msg_t *msg_in, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  msg->state = state;
+  orconn_state_publish(msg);
+}
+
+static void
+send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->onehop = onehop;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_cntev_orconn_state(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_NONE);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  send_ocirc_chan(1, 1, true);
+  assert_bootmsg("5 TAG=conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  /* It doesn't know it's an origin circuit yet */
+  assert_bootmsg("15 TAG=handshake_done");
+  send_ocirc_chan(2, 2, false);
+  assert_bootmsg("80 TAG=ap_conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("85 TAG=ap_conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("89 TAG=ap_handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("90 TAG=ap_handshake_done");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_pt(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("1 TAG=conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("2 TAG=conn_done_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("76 TAG=ap_conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("77 TAG=ap_conn_done_pt");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_proxy(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_CONNECT);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("3 TAG=conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("4 TAG=conn_done_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("78 TAG=ap_conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("79 TAG=ap_conn_done_proxy");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+#define TEST(name, flags)                               \
   { #name, test_cntev_ ## name, flags, 0, NULL }
 
+#define T_PUBSUB(name, setup)                                           \
+  { #name, test_cntev_ ## name, TT_FORK, &helper_pubsub_setup, NULL }
+
 struct testcase_t controller_event_tests[] = {
   TEST(sum_up_cell_stats, TT_FORK),
   TEST(append_cell_stats, TT_FORK),
   TEST(format_cell_stats, TT_FORK),
   TEST(event_mask, TT_FORK),
-  TEST(dirboot_defer_desc, TT_FORK),
-  TEST(dirboot_defer_orconn, TT_FORK),
+  T_PUBSUB(dirboot_defer_desc, TT_FORK),
+  T_PUBSUB(dirboot_defer_orconn, TT_FORK),
+  T_PUBSUB(signal, TT_FORK),
+  T_PUBSUB(orconn_state, TT_FORK),
+  T_PUBSUB(orconn_state_pt, TT_FORK),
+  T_PUBSUB(orconn_state_proxy, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 5af0cce130..ed39f41560 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -32,7 +32,7 @@
 DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/dh.h>
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Run unit tests for Diffie-Hellman functionality. */
 static void
@@ -190,7 +190,7 @@ test_crypto_dh(void *arg)
     DH_get0_key(dh4, &pk, &sk);
 #else
     pk = dh4->pub_key;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
     tt_assert(pk);
     tt_int_op(BN_num_bytes(pk), OP_LE, DH1024_KEY_LEN);
     tt_int_op(BN_num_bytes(pk), OP_GT, 0);
@@ -207,7 +207,7 @@ test_crypto_dh(void *arg)
     tt_int_op(s1len, OP_GT, 0);
     tt_mem_op(s1, OP_EQ, s2, s1len);
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
  done:
   crypto_dh_free(dh1);
@@ -219,7 +219,7 @@ test_crypto_dh(void *arg)
     DH_free(dh4);
   if (pubkey_tmp)
     BN_free(pubkey_tmp);
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 }
 
 static void
@@ -248,174 +248,12 @@ test_crypto_openssl_version(void *arg)
   tt_int_op(a, OP_GE, 0);
   tt_int_op(b, OP_GE, 0);
   tt_int_op(c, OP_GE, 0);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
  done:
   ;
 }
 
-/** Run unit tests for our random number generation function and its wrappers.
- */
-static void
-test_crypto_rng(void *arg)
-{
-  int i, j, allok;
-  char data1[100], data2[100];
-  double d;
-  char *h=NULL;
-
-  /* Try out RNG. */
-  (void)arg;
-  tt_assert(! crypto_seed_rng());
-  crypto_rand(data1, 100);
-  crypto_rand(data2, 100);
-  tt_mem_op(data1,OP_NE, data2,100);
-  allok = 1;
-  for (i = 0; i < 100; ++i) {
-    uint64_t big;
-    char *host;
-    j = crypto_rand_int(100);
-    if (j < 0 || j >= 100)
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(1)<<40);
-    if (big >= (UINT64_C(1)<<40))
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(5));
-    if (big >= 5)
-      allok = 0;
-    d = crypto_rand_double();
-    tt_assert(d >= 0);
-    tt_assert(d < 1.0);
-    host = crypto_random_hostname(3,8,"www.",".onion");
-    if (strcmpstart(host,"www.") ||
-        strcmpend(host,".onion") ||
-        strlen(host) < 13 ||
-        strlen(host) > 18)
-      allok = 0;
-    tor_free(host);
-  }
-
-  /* Make sure crypto_random_hostname clips its inputs properly. */
-  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
-  tt_assert(! strcmpstart(h,"www."));
-  tt_assert(! strcmpend(h,".onion"));
-  tt_int_op(63+4+6, OP_EQ, strlen(h));
-
-  tt_assert(allok);
- done:
-  tor_free(h);
-}
-
-static void
-test_crypto_rng_range(void *arg)
-{
-  int got_smallest = 0, got_largest = 0;
-  int i;
-
-  (void)arg;
-  for (i = 0; i < 1000; ++i) {
-    int x = crypto_rand_int_range(5,9);
-    tt_int_op(x, OP_GE, 5);
-    tt_int_op(x, OP_LT, 9);
-    if (x == 5)
-      got_smallest = 1;
-    if (x == 8)
-      got_largest = 1;
-  }
-  /* These fail with probability 1/10^603. */
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  got_smallest = got_largest = 0;
-  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
-  for (i = 0; i < 1000; ++i) {
-    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
-    tt_u64_op(x, OP_GE, ten_billion);
-    tt_u64_op(x, OP_LT, ten_billion+10);
-    if (x == ten_billion)
-      got_smallest = 1;
-    if (x == ten_billion+9)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  const time_t now = time(NULL);
-  for (i = 0; i < 2000; ++i) {
-    time_t x = crypto_rand_time_range(now, now+60);
-    tt_i64_op(x, OP_GE, now);
-    tt_i64_op(x, OP_LT, now+60);
-    if (x == now)
-      got_smallest = 1;
-    if (x == now+59)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
- done:
-  ;
-}
-
-static void
-test_crypto_rng_strongest(void *arg)
-{
-  const char *how = arg;
-  int broken = 0;
-
-  if (how == NULL) {
-    ;
-  } else if (!strcmp(how, "nosyscall")) {
-    break_strongest_rng_syscall = 1;
-  } else if (!strcmp(how, "nofallback")) {
-    break_strongest_rng_fallback = 1;
-  } else if (!strcmp(how, "broken")) {
-    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
-  }
-
-#define N 128
-  uint8_t combine_and[N];
-  uint8_t combine_or[N];
-  int i, j;
-
-  memset(combine_and, 0xff, N);
-  memset(combine_or, 0, N);
-
-  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
-    uint8_t output[N];
-    memset(output, 0, N);
-    if (how == NULL) {
-      /* this one can't fail. */
-      crypto_strongest_rand(output, sizeof(output));
-    } else {
-      int r = crypto_strongest_rand_raw(output, sizeof(output));
-      if (r == -1) {
-        if (broken) {
-          goto done; /* we're fine. */
-        }
-        /* This function is allowed to break, but only if it always breaks. */
-        tt_int_op(i, OP_EQ, 0);
-        tt_skip();
-      } else {
-        tt_assert(! broken);
-      }
-    }
-    for (j = 0; j < N; ++j) {
-      combine_and[j] &= output[j];
-      combine_or[j] |= output[j];
-    }
-  }
-
-  for (j = 0; j < N; ++j) {
-    tt_int_op(combine_and[j], OP_EQ, 0);
-    tt_int_op(combine_or[j], OP_EQ, 0xff);
-  }
- done:
-  ;
-#undef N
-}
-
 /** Run unit tests for our AES128 functionality */
 static void
 test_crypto_aes128(void *arg)
@@ -551,7 +389,7 @@ test_crypto_aes128(void *arg)
                                    "\xff\xff\xff\xff\xff\xff\xff\xff"
                                    "\xff\xff\xff\xff\xff\xff\xff\xff");
   crypto_cipher_crypt_inplace(env1, data2, 64);
-  tt_assert(tor_mem_is_zero(data2, 64));
+  tt_assert(fast_mem_is_zero(data2, 64));
 
  done:
   tor_free(mem_op_hex_tmp);
@@ -1173,13 +1011,19 @@ test_crypto_sha3_xof(void *arg)
   crypto_xof_free(xof);
   memset(out, 0, sizeof(out));
 
+  /* Test one-function absorb/squeeze. */
+  crypto_xof(out, sizeof(out), msg, sizeof(msg));
+  test_memeq_hex(out, squeezed_hex);
+  memset(out, 0, sizeof(out));
+
   /* Test incremental absorb/squeeze. */
   xof = crypto_xof_new();
   tt_assert(xof);
   for (size_t i = 0; i < sizeof(msg); i++)
     crypto_xof_add_bytes(xof, msg + i, 1);
-  for (size_t i = 0; i < sizeof(out); i++)
+  for (size_t i = 0; i < sizeof(out); i++) {
     crypto_xof_squeeze_bytes(xof, out + i, 1);
+  }
   test_memeq_hex(out, squeezed_hex);
 
  done:
@@ -1903,13 +1747,13 @@ test_crypto_base32_decode(void *arg)
   /* Encode and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Encode, uppercase, and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   tor_strupper(encoded);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Change encoded string and decode. */
   if (encoded[0] == 'A' || encoded[0] == 'a')
@@ -1917,12 +1761,12 @@ test_crypto_base32_decode(void *arg)
   else
     encoded[0] = 'A';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_NE, decoded, 60);
   /* Bad encodings. */
   encoded[0] = '!';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(0, OP_GT, res);
+  tt_int_op(res, OP_LT, 0);
 
  done:
   ;
@@ -2117,7 +1961,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "e0544770bc7de853b38f9100489e3e79";
   const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88"
                                 "bd13861475516bc2cd2b6e005e805064";
-#else /* !(defined(SLOW_CURVE25519_TEST)) */
+#else /* !defined(SLOW_CURVE25519_TEST) */
   const int loop_max=200;
   const char e1_expected[]    = "bc7112cde03f97ef7008cad1bdc56be3"
                                 "c6a1037d74cceb3712e9206871dcf654";
@@ -2269,7 +2113,7 @@ test_crypto_curve25519_encode(void *arg)
 
   curve25519_secret_key_generate(&seckey, 0);
   curve25519_public_key_generate(&key1, &seckey);
-  tt_int_op(0, OP_EQ, curve25519_public_to_base64(buf, &key1));
+  curve25519_public_to_base64(buf, &key1);
   tt_int_op(CURVE25519_BASE64_PADDED_LEN, OP_EQ, strlen(buf));
 
   tt_int_op(0, OP_EQ, curve25519_public_from_base64(&key2, buf));
@@ -2328,7 +2172,7 @@ test_crypto_curve25519_persist(void *arg)
   tt_u64_op((uint64_t)st.st_size, OP_EQ,
             32+CURVE25519_PUBKEY_LEN+CURVE25519_SECKEY_LEN);
   tt_assert(fast_memeq(content, "== c25519v1: testing ==", taglen));
-  tt_assert(tor_mem_is_zero(content+taglen, 32-taglen));
+  tt_assert(fast_mem_is_zero(content+taglen, 32-taglen));
   cp = content + 32;
   tt_mem_op(keypair.seckey.secret_key,OP_EQ,
              cp,
@@ -2649,13 +2493,13 @@ test_crypto_ed25519_encode(void *arg)
 
   /* Test roundtrip. */
   tt_int_op(0, OP_EQ, ed25519_keypair_generate(&kp, 0));
-  tt_int_op(0, OP_EQ, ed25519_public_to_base64(buf, &kp.pubkey));
+  ed25519_public_to_base64(buf, &kp.pubkey);
   tt_int_op(ED25519_BASE64_LEN, OP_EQ, strlen(buf));
   tt_int_op(0, OP_EQ, ed25519_public_from_base64(&pk, buf));
   tt_mem_op(kp.pubkey.pubkey, OP_EQ, pk.pubkey, ED25519_PUBKEY_LEN);
 
   tt_int_op(0, OP_EQ, ed25519_sign(&sig1, (const uint8_t*)"ABC", 3, &kp));
-  tt_int_op(0, OP_EQ, ed25519_signature_to_base64(buf, &sig1));
+  ed25519_signature_to_base64(buf, &sig1);
   tt_int_op(0, OP_EQ, ed25519_signature_from_base64(&sig2, buf));
   tt_mem_op(sig1.sig, OP_EQ, sig2.sig, ED25519_SIG_LEN);
 
@@ -3178,15 +3022,6 @@ test_crypto_failure_modes(void *arg)
 
 struct testcase_t crypto_tests[] = {
   CRYPTO_LEGACY(formats),
-  CRYPTO_LEGACY(rng),
-  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
-  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
-  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nosyscall" },
-  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nofallback" },
-  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"broken" },
   { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL },
   { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" },
   { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" },
diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c
new file mode 100644
index 0000000000..6b7749a889
--- /dev/null
+++ b/src/test/test_crypto_rng.c
@@ -0,0 +1,332 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#define CRYPTO_RAND_PRIVATE
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+/** Run unit tests for our random number generation function and its wrappers.
+ */
+static void
+test_crypto_rng(void *arg)
+{
+  int i, j, allok;
+  char data1[100], data2[100];
+  double d;
+  char *h=NULL;
+
+  /* Try out RNG. */
+  (void)arg;
+  tt_assert(! crypto_seed_rng());
+  crypto_rand(data1, 100);
+  crypto_rand(data2, 100);
+  tt_mem_op(data1,OP_NE, data2,100);
+  allok = 1;
+  for (i = 0; i < 100; ++i) {
+    uint64_t big;
+    char *host;
+    j = crypto_rand_int(100);
+    if (j < 0 || j >= 100)
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(1)<<40);
+    if (big >= (UINT64_C(1)<<40))
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(5));
+    if (big >= 5)
+      allok = 0;
+    d = crypto_rand_double();
+    tt_assert(d >= 0);
+    tt_assert(d < 1.0);
+    host = crypto_random_hostname(3,8,"www.",".onion");
+    if (strcmpstart(host,"www.") ||
+        strcmpend(host,".onion") ||
+        strlen(host) < 13 ||
+        strlen(host) > 18)
+      allok = 0;
+    tor_free(host);
+  }
+
+  /* Make sure crypto_random_hostname clips its inputs properly. */
+  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
+  tt_assert(! strcmpstart(h,"www."));
+  tt_assert(! strcmpend(h,".onion"));
+  tt_int_op(63+4+6, OP_EQ, strlen(h));
+
+  tt_assert(allok);
+ done:
+  tor_free(h);
+}
+
+static void
+test_crypto_rng_range(void *arg)
+{
+  int got_smallest = 0, got_largest = 0;
+  int i;
+
+  (void)arg;
+  for (i = 0; i < 1000; ++i) {
+    int x = crypto_rand_int_range(5,9);
+    tt_int_op(x, OP_GE, 5);
+    tt_int_op(x, OP_LT, 9);
+    if (x == 5)
+      got_smallest = 1;
+    if (x == 8)
+      got_largest = 1;
+  }
+  /* These fail with probability 1/10^603. */
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  got_smallest = got_largest = 0;
+  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
+  for (i = 0; i < 1000; ++i) {
+    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
+    tt_u64_op(x, OP_GE, ten_billion);
+    tt_u64_op(x, OP_LT, ten_billion+10);
+    if (x == ten_billion)
+      got_smallest = 1;
+    if (x == ten_billion+9)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  const time_t now = time(NULL);
+  for (i = 0; i < 2000; ++i) {
+    time_t x = crypto_rand_time_range(now, now+60);
+    tt_i64_op(x, OP_GE, now);
+    tt_i64_op(x, OP_LT, now+60);
+    if (x == now)
+      got_smallest = 1;
+    if (x == now+59)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+ done:
+  ;
+}
+
+static void
+test_crypto_rng_strongest(void *arg)
+{
+  const char *how = arg;
+  int broken = 0;
+
+  if (how == NULL) {
+    ;
+  } else if (!strcmp(how, "nosyscall")) {
+    break_strongest_rng_syscall = 1;
+  } else if (!strcmp(how, "nofallback")) {
+    break_strongest_rng_fallback = 1;
+  } else if (!strcmp(how, "broken")) {
+    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
+  }
+
+#define N 128
+  uint8_t combine_and[N];
+  uint8_t combine_or[N];
+  int i, j;
+
+  memset(combine_and, 0xff, N);
+  memset(combine_or, 0, N);
+
+  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
+    uint8_t output[N];
+    memset(output, 0, N);
+    if (how == NULL) {
+      /* this one can't fail. */
+      crypto_strongest_rand(output, sizeof(output));
+    } else {
+      int r = crypto_strongest_rand_raw(output, sizeof(output));
+      if (r == -1) {
+        if (broken) {
+          goto done; /* we're fine. */
+        }
+        /* This function is allowed to break, but only if it always breaks. */
+        tt_int_op(i, OP_EQ, 0);
+        tt_skip();
+      } else {
+        tt_assert(! broken);
+      }
+    }
+    for (j = 0; j < N; ++j) {
+      combine_and[j] &= output[j];
+      combine_or[j] |= output[j];
+    }
+  }
+
+  for (j = 0; j < N; ++j) {
+    tt_int_op(combine_and[j], OP_EQ, 0);
+    tt_int_op(combine_or[j], OP_EQ, 0xff);
+  }
+ done:
+  ;
+#undef N
+}
+
+static void
+test_crypto_rng_fast(void *arg)
+{
+  (void)arg;
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  tt_assert(rng);
+
+  /* Rudimentary black-block test to make sure that our prng outputs
+   * have all bits sometimes on and all bits sometimes off. */
+  uint64_t m1 = 0, m2 = ~(uint64_t)0;
+  const int N = 128;
+
+  for (int i=0; i < N; ++i) {
+    uint64_t v;
+    crypto_fast_rng_getbytes(rng, (void*)&v, sizeof(v));
+    m1 |= v;
+    m2 &= v;
+  }
+
+  tt_u64_op(m1, OP_EQ, ~(uint64_t)0);
+  tt_u64_op(m2, OP_EQ, 0);
+
+  /* Check range functions. */
+  int counts[5];
+  memset(counts, 0, sizeof(counts));
+  for (int i=0; i < N; ++i) {
+    unsigned u = crypto_fast_rng_get_uint(rng, 5);
+    tt_int_op(u, OP_GE, 0);
+    tt_int_op(u, OP_LT, 5);
+    counts[u]++;
+
+    uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+
+    double d = crypto_fast_rng_get_double(rng);
+    tt_assert(d >= 0.0);
+    tt_assert(d < 1.0);
+  }
+
+  /* All values should have come up once. */
+  for (int i=0; i<5; ++i) {
+    tt_int_op(counts[i], OP_GT, 0);
+  }
+
+  /* per-thread rand_fast shouldn't crash or leak. */
+  crypto_fast_rng_t *t_rng = get_thread_fast_rng();
+  for (int i = 0; i < N; ++i) {
+    uint64_t u64 = crypto_fast_rng_get_uint64(t_rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+  }
+
+ done:
+  crypto_fast_rng_free(rng);
+}
+
+static void
+test_crypto_rng_fast_whitebox(void *arg)
+{
+  (void)arg;
+  const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream();
+  char *buf = tor_malloc_zero(buflen);
+  char *buf2 = tor_malloc_zero(buflen);
+  char *buf3 = NULL, *buf4 = NULL;
+
+  crypto_cipher_t *cipher = NULL, *cipher2 = NULL;
+  uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed, 0, sizeof(seed));
+
+  /* Start with a prng with zero key and zero IV. */
+  crypto_fast_rng_t *rng = crypto_fast_rng_new_from_seed(seed);
+  tt_assert(rng);
+
+  /* We'll use a stream cipher to keep in sync */
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+
+  /* The first 48 bytes are used for the next seed -- let's make sure we have
+   * them.
+   */
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+
+  /* if we get 128 bytes, they should match the bytes from the aes256-counter
+   * stream, starting at position 48.
+   */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 128);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* Try that again, with an odd number of bytes. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 199);
+  memset(buf2, 0, 199);
+  crypto_cipher_crypt_inplace(cipher, buf2, 199);
+  tt_mem_op(buf, OP_EQ, buf2, 199);
+
+  /* Make sure that refilling works as expected: skip all but the last 5 bytes
+   * of this steam. */
+  size_t skip = buflen - (199+128) - 5;
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, skip);
+  crypto_cipher_crypt_inplace(cipher, buf2, skip);
+
+  /* Now get the next 128 bytes. The first 5 will come from this stream, and
+   * the next 5 will come from the stream keyed by the new value of 'seed'. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 5);
+  crypto_cipher_free(cipher);
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, buf2+5, 128-5);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* And check the next 7 bytes to make sure we didn't discard anything. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 7);
+  memset(buf2, 0, 7);
+  crypto_cipher_crypt_inplace(cipher, buf2, 7);
+  tt_mem_op(buf, OP_EQ, buf2, 7);
+
+  /* Now try the optimization for long outputs. */
+  buf3 = tor_malloc(65536);
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf3, 65536);
+
+  buf4 = tor_malloc_zero(65536);
+  uint8_t seed2[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed2, 0, sizeof(seed2));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed2, sizeof(seed2));
+  cipher2 = crypto_cipher_new_with_iv_and_bits(seed2, seed2+32, 256);
+  crypto_cipher_crypt_inplace(cipher2, buf4, 65536);
+  tt_mem_op(buf3, OP_EQ, buf4, 65536);
+
+ done:
+  crypto_fast_rng_free(rng);
+  crypto_cipher_free(cipher);
+  crypto_cipher_free(cipher2);
+  tor_free(buf);
+  tor_free(buf2);
+  tor_free(buf3);
+  tor_free(buf4);
+}
+
+struct testcase_t crypto_rng_tests[] = {
+  { "rng", test_crypto_rng, 0, NULL, NULL },
+  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
+  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
+  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nosyscall" },
+  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nofallback" },
+  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"broken" },
+  { "fast", test_crypto_rng_fast, 0, NULL, NULL },
+  { "fast_whitebox", test_crypto_rng_fast_whitebox, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c
index e24aee8930..3b20dfa587 100644
--- a/src/test/test_crypto_slow.c
+++ b/src/test/test_crypto_slow.c
@@ -109,7 +109,7 @@ run_s2k_tests(const unsigned flags, const unsigned type,
             secret_to_key_derivekey(buf3, sizeof(buf3), buf, speclen,
                                     pw1, strlen(pw1)));
   tt_mem_op(buf2, OP_EQ, buf3, sizeof(buf3));
-  tt_assert(!tor_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
+  tt_assert(!fast_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
 
  done:
   ;
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 0e44c47f3f..6329ff7750 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -8,7 +8,7 @@
 
 #define BWAUTH_PRIVATE
 #define CONFIG_PRIVATE
-#define CONTROL_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
 #define DIRSERV_PRIVATE
@@ -26,13 +26,13 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/connection.h"
 #include "core/or/relay.h"
 #include "core/or/versions.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_getinfo.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/dsigs_parse.h"
@@ -91,9 +91,29 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 #define NS_MODULE dir
 
+static networkstatus_t *
+networkstatus_parse_vote_from_string_(const char *s,
+                                      const char **eos_out,
+                                      enum networkstatus_type_t ns_type)
+{
+  size_t len = strlen(s);
+  // memdup so that it won't be nul-terminated.
+  char *tmp = tor_memdup(s, len);
+  networkstatus_t *result =
+    networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type);
+  if (eos_out && *eos_out) {
+    *eos_out = s + (*eos_out - tmp);
+  }
+  tor_free(tmp);
+  return result;
+}
+
 static void
 test_dir_nicknames(void *arg)
 {
@@ -142,6 +162,269 @@ test_dir_nicknames(void *arg)
   ;
 }
 
+/* Allocate and return a new routerinfo, with the fields set from the
+ * arguments to this function.
+ *
+ * Also sets:
+ *  - random RSA identity and onion keys,
+ *  - the platform field using get_platform_str(), and
+ *  - supports_tunnelled_dir_requests to 1.
+ *
+ * If rsa_onion_keypair_out is not NULL, it is set to the onion keypair.
+ * The caller must free this keypair.
+ */
+static routerinfo_t *
+basic_routerinfo_new(const char *nickname, uint32_t ipv4_addr,
+                     uint16_t or_port, uint16_t dir_port,
+                     uint32_t bandwidthrate, uint32_t bandwidthburst,
+                     uint32_t bandwidthcapacity,
+                     time_t published_on,
+                     crypto_pk_t **rsa_onion_keypair_out)
+{
+  char platform[256];
+
+  tor_assert(nickname);
+
+  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
+  /* These keys are random: idx is ignored. */
+  pk1 = pk_generate(0);
+  pk2 = pk_generate(1);
+
+  tor_assert(pk1);
+  tor_assert(pk2);
+
+  get_platform_str(platform, sizeof(platform));
+
+  routerinfo_t *r1 = tor_malloc_zero(sizeof(routerinfo_t));
+
+  r1->nickname = tor_strdup(nickname);
+  r1->platform = tor_strdup(platform);
+
+  r1->addr = ipv4_addr;
+  r1->or_port = or_port;
+  r1->dir_port = dir_port;
+  r1->supports_tunnelled_dir_requests = 1;
+
+  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
+  r1->identity_pkey = pk2;
+
+  r1->bandwidthrate = bandwidthrate;
+  r1->bandwidthburst = bandwidthburst;
+  r1->bandwidthcapacity = bandwidthcapacity;
+
+  r1->cache_info.published_on = published_on;
+
+  if (rsa_onion_keypair_out) {
+    *rsa_onion_keypair_out = pk1;
+  } else {
+    crypto_pk_free(pk1);
+  }
+
+  return r1;
+}
+
+/* Allocate and return a new string containing a "router" line for r1. */
+static char *
+get_new_router_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "router %s %s %d 0 %d\n",
+               r1->nickname, fmt_addr32(r1->addr),
+               r1->or_port, r1->dir_port);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "platform" line for the
+ * current Tor version and OS. */
+static char *
+get_new_platform_line(void)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "platform Tor %s on %s\n",
+               VERSION, get_uname());
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "published" line for r1.
+ * r1->cache_info.published_on must be between 0 and 59 seconds. */
+static char *
+get_new_published_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_assert(r1->cache_info.published_on >= 0);
+  tor_assert(r1->cache_info.published_on <= 59);
+
+  tor_asprintf(&line,
+               "published 1970-01-01 00:00:%02u\n",
+               (unsigned)r1->cache_info.published_on);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "fingerprint" line for r1. */
+static char *
+get_new_fingerprint_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+  char fingerprint[FINGERPRINT_LEN+1];
+
+  tor_assert(r1);
+
+  tor_assert(!crypto_pk_get_fingerprint(r1->identity_pkey, fingerprint, 1));
+  tor_assert(strlen(fingerprint) > 0);
+
+  tor_asprintf(&line,
+               "fingerprint %s\n",
+               fingerprint);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "uptime" line with uptime t.
+ *
+ * You should pass a hard-coded value to this function, because even if we made
+ * it reflect uptime, that still wouldn't make it right, because the two
+ * descriptors might be made on different seconds.
+ */
+static char *
+get_new_uptime_line(time_t t)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "uptime %u\n",
+               (unsigned)t);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "bandwidth" line for r1.
+ */
+static char *
+get_new_bandwidth_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "bandwidth %u %u %u\n",
+               r1->bandwidthrate,
+               r1->bandwidthburst,
+               r1->bandwidthcapacity);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a key_name block for the
+ * RSA key pk1.
+ */
+static char *
+get_new_rsa_key_block(const char *key_name, crypto_pk_t *pk1)
+{
+  char *block = NULL;
+  char *pk1_str = NULL;
+  size_t pk1_str_len = 0;
+
+  tor_assert(key_name);
+  tor_assert(pk1);
+
+  tor_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
+                                                   &pk1_str_len));
+  tor_assert(pk1_str);
+  tor_assert(pk1_str_len);
+
+  tor_asprintf(&block,
+               "%s\n%s",
+               key_name,
+               pk1_str);
+  tor_free(pk1_str);
+
+  tor_assert(block);
+  return block;
+}
+
+/* Allocate and return a new string containing an "onion-key" block for the
+ * router r1.
+ */
+static char *
+get_new_onion_key_block(const routerinfo_t *r1)
+{
+  char *block = NULL;
+  tor_assert(r1);
+  crypto_pk_t *pk_tmp = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                  r1->onion_pkey_len);
+  block = get_new_rsa_key_block("onion-key", pk_tmp);
+  crypto_pk_free(pk_tmp);
+  return block;
+}
+
+/* Allocate and return a new string containing an "signing-key" block for the
+ * router r1.
+ */
+static char *
+get_new_signing_key_block(const routerinfo_t *r1)
+{
+  tor_assert(r1);
+  return get_new_rsa_key_block("signing-key", r1->identity_pkey);
+}
+
+/* Allocate and return a new string containing an "ntor-onion-key" line for
+ * the curve25519 public key ntor_onion_pubkey.
+ */
+static char *
+get_new_ntor_onion_key_line(const curve25519_public_key_t *ntor_onion_pubkey)
+{
+  char *line = NULL;
+  char cert_buf[256];
+  int rv = 0;
+
+  tor_assert(ntor_onion_pubkey);
+
+  rv = base64_encode(cert_buf, sizeof(cert_buf),
+                     (const char*)ntor_onion_pubkey->public_key, 32,
+                     BASE64_ENCODE_MULTILINE);
+  tor_assert(rv > 0);
+  tor_assert(strlen(cert_buf) > 0);
+
+  tor_asprintf(&line,
+               "ntor-onion-key %s",
+               cert_buf);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "bridge-distribution-request"
+ * line for options.
+ */
+static char *
+get_new_bridge_distribution_request_line(const or_options_t *options)
+{
+  if (options->BridgeRelay) {
+    return tor_strdup("bridge-distribution-request any\n");
+  } else {
+    return tor_strdup("");
+  }
+}
+
 static smartlist_t *mocked_configured_ports = NULL;
 
 /** Returns mocked_configured_ports */
@@ -151,71 +434,510 @@ mock_get_configured_ports(void)
   return mocked_configured_ports;
 }
 
-/** Run unit tests for router descriptor generation logic. */
+static tor_cert_t *
+mock_tor_cert_dup_null(const tor_cert_t *cert)
+{
+  (void)cert;
+  return NULL;
+}
+
+static crypto_pk_t *mocked_server_identitykey = NULL;
+
+/* Returns mocked_server_identitykey with no checks. */
+static crypto_pk_t *
+mock_get_server_identity_key(void)
+{
+  return mocked_server_identitykey;
+}
+
+static crypto_pk_t *mocked_onionkey = NULL;
+
+/* Returns mocked_onionkey with no checks. */
+static crypto_pk_t *
+mock_get_onion_key(void)
+{
+  return mocked_onionkey;
+}
+
+static routerinfo_t *mocked_routerinfo = NULL;
+
+/* Returns 0 and sets ri_out to mocked_routerinfo.
+ * ri_out must not be NULL. There are no other checks. */
+static int
+mock_router_build_fresh_unsigned_routerinfo(routerinfo_t **ri_out)
+{
+  tor_assert(ri_out);
+  *ri_out = mocked_routerinfo;
+  return 0;
+}
+
+static ed25519_keypair_t *mocked_master_signing_key = NULL;
+
+/* Returns mocked_master_signing_key with no checks. */
+static const ed25519_keypair_t *
+mock_get_master_signing_keypair(void)
+{
+  return mocked_master_signing_key;
+}
+
+static struct tor_cert_st *mocked_signing_key_cert = NULL;
+
+/* Returns mocked_signing_key_cert with no checks. */
+static const struct tor_cert_st *
+mock_get_master_signing_key_cert(void)
+{
+  return mocked_signing_key_cert;
+}
+
+static curve25519_keypair_t *mocked_curve25519_onion_key = NULL;
+
+/* Returns mocked_curve25519_onion_key with no checks. */
+static const curve25519_keypair_t *
+mock_get_current_curve25519_keypair(void)
+{
+  return mocked_curve25519_onion_key;
+}
+
+/* Unmock get_configured_ports() and free mocked_configured_ports. */
+static void
+cleanup_mock_configured_ports(void)
+{
+  UNMOCK(get_configured_ports);
+
+  if (mocked_configured_ports) {
+    SMARTLIST_FOREACH(mocked_configured_ports, port_cfg_t *, p, tor_free(p));
+    smartlist_free(mocked_configured_ports);
+  }
+}
+
+/* Mock get_configured_ports() with a list containing or_port and dir_port.
+ * If a port is 0, don't set it.
+ * Only sets the minimal data required for the tests to pass. */
+static void
+setup_mock_configured_ports(uint16_t or_port, uint16_t dir_port)
+{
+  cleanup_mock_configured_ports();
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  MOCK(get_configured_ports, mock_get_configured_ports);
+  mocked_configured_ports = smartlist_new();
+
+  if (or_port) {
+    port_cfg_t *or_port_cfg = tor_malloc_zero(sizeof(*or_port_cfg));
+    or_port_cfg->type = CONN_TYPE_OR_LISTENER;
+    or_port_cfg->addr.family = AF_INET;
+    or_port_cfg->port = or_port;
+    smartlist_add(mocked_configured_ports, or_port_cfg);
+  }
+
+  if (dir_port) {
+    port_cfg_t *dir_port_cfg = tor_malloc_zero(sizeof(*dir_port_cfg));
+    dir_port_cfg->type = CONN_TYPE_DIR_LISTENER;
+    dir_port_cfg->addr.family = AF_INET;
+    dir_port_cfg->port = dir_port;
+    smartlist_add(mocked_configured_ports, dir_port_cfg);
+  }
+}
+
+/* Clean up the data structures and unmock the functions needed for generating
+ * a fresh descriptor. */
+static void
+cleanup_mocks_for_fresh_descriptor(void)
+{
+  tor_free(get_options_mutable()->Nickname);
+
+  mocked_server_identitykey = NULL;
+  UNMOCK(get_server_identity_key);
+
+  crypto_pk_free(mocked_onionkey);
+  UNMOCK(get_onion_key);
+}
+
+/* Mock the data structures and functions needed for generating a fresh
+ * descriptor.
+ *
+ * Sets options->Nickname from r1->nickname.
+ * Mocks get_server_identity_key() with r1->identity_pkey.
+ *
+ * If rsa_onion_keypair is not NULL, it is used to mock get_onion_key().
+ * Otherwise, the public key in r1->onion_pkey is used to mock get_onion_key().
+ */
+static void
+setup_mocks_for_fresh_descriptor(const routerinfo_t *r1,
+                                 crypto_pk_t *rsa_onion_keypair)
+{
+  cleanup_mocks_for_fresh_descriptor();
+
+  tor_assert(r1);
+
+  /* router_build_fresh_signed_extrainfo() requires options->Nickname */
+  get_options_mutable()->Nickname = tor_strdup(r1->nickname);
+
+  /* router_build_fresh_signed_extrainfo() requires get_server_identity_key().
+   * Use the same one as the call to router_dump_router_to_string() above.
+   */
+  mocked_server_identitykey = r1->identity_pkey;
+  MOCK(get_server_identity_key, mock_get_server_identity_key);
+
+  /* router_dump_and_sign_routerinfo_descriptor_body() requires
+   * get_onion_key(). Use the same one as r1.
+   */
+  if (rsa_onion_keypair) {
+    mocked_onionkey = crypto_pk_dup_key(rsa_onion_keypair);
+  } else {
+    mocked_onionkey = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                r1->onion_pkey_len);
+  }
+  MOCK(get_onion_key, mock_get_onion_key);
+}
+
+/* Set options based on arg.
+ *
+ * b: BridgeRelay 1
+ * e: ExtraInfoStatistics 1
+ * s: sets all the individual statistics options to 1
+ *
+ * Always sets AssumeReachable to 1.
+ *
+ * Does not set ServerTransportPlugin, because it's parsed before use.
+ *
+ * Does not set BridgeRecordUsageByCountry, because the tests don't have access
+ * to a GeoIPFile or GeoIPv6File. */
 static void
-test_dir_formats(void *arg)
+setup_dir_formats_options(const char *arg, or_options_t *options)
+{
+  /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */
+  options->AssumeReachable = 1;
+
+  if (strchr(arg, 'b')) {
+    options->BridgeRelay = 1;
+  }
+
+  if (strchr(arg, 'e')) {
+    options->ExtraInfoStatistics = 1;
+  }
+
+  if (strchr(arg, 's')) {
+    options->DirReqStatistics = 1;
+    options->HiddenServiceStatistics = 1;
+    options->EntryStatistics = 1;
+    options->CellStatistics = 1;
+    options->ExitPortStatistics = 1;
+    options->ConnDirectionStatistics = 1;
+    options->PaddingStatistics = 1;
+  }
+}
+
+/* Check that routerinfos r1 and rp1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_ROUTERINFO_CONSISTENCY(r1, rp1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(rp1); \
+\
+  tt_int_op(rp1->addr,OP_EQ, r1->addr); \
+  tt_int_op(rp1->or_port,OP_EQ, r1->or_port); \
+  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); \
+  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); \
+  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); \
+  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); \
+  crypto_pk_t *rp1_onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, \
+                                                      rp1->onion_pkey_len); \
+  crypto_pk_t *r1_onion_pkey = router_get_rsa_onion_pkey(r1->onion_pkey, \
+                                                      r1->onion_pkey_len); \
+  tt_int_op(crypto_pk_cmp_keys(rp1_onion_pkey, r1_onion_pkey), OP_EQ, 0); \
+  crypto_pk_free(rp1_onion_pkey); \
+  crypto_pk_free(r1_onion_pkey); \
+  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, r1->identity_pkey), \
+            OP_EQ, 0); \
+  tt_int_op(rp1->supports_tunnelled_dir_requests, OP_EQ, \
+            r1->supports_tunnelled_dir_requests); \
+STMT_END
+
+/* Check that routerinfo r1 and extrainfo e1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_EXTRAINFO_CONSISTENCY(r1, e1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(e1); \
+\
+  tt_str_op(e1->nickname, OP_EQ, r1->nickname); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA-only
+ * router. Tor versions without ed25519 (0.2.6 and earlier) are no longer
+ * officially supported, but the authorities still accept their descriptors.
+ */
+static void
+test_dir_formats_rsa(void *arg)
 {
   char *buf = NULL;
-  char buf2[8192];
-  char platform[256];
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *pk1_str = NULL, *pk2_str = NULL, *cp;
-  size_t pk1_str_len, pk2_str_len;
-  routerinfo_t *r1=NULL, *r2=NULL;
-  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
-  routerinfo_t *rp1 = NULL, *rp2 = NULL;
-  addr_policy_t *ex1, *ex2;
-  routerlist_t *dir1 = NULL, *dir2 = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
   uint8_t *rsa_cc = NULL;
-  or_options_t *options = get_options_mutable();
-  const addr_policy_t *p;
-  time_t now = time(NULL);
-  port_cfg_t orport, dirport;
-  char cert_buf[256];
 
-  (void)arg;
-  pk1 = pk_generate(0);
-  pk2 = pk_generate(1);
+  routerinfo_t *r1 = NULL;
+  extrainfo_t *e1 = NULL;
+  routerinfo_t *rp1 = NULL;
+  extrainfo_t *ep1 = NULL;
+
+  smartlist_t *chunks = NULL;
+  const char *msg = NULL;
+  int rv = -1;
 
-  tt_assert(pk1 && pk2);
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
 
   hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
 
-  get_platform_str(platform, sizeof(platform));
-  r1 = tor_malloc_zero(sizeof(routerinfo_t));
-  r1->addr = 0xc0a80001u; /* 192.168.0.1 */
-  r1->cache_info.published_on = 0;
-  r1->or_port = 9000;
-  r1->dir_port = 9003;
-  r1->supports_tunnelled_dir_requests = 1;
-  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
-  r1->ipv6_orport = 9999;
-  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
-  /* Fake just enough of an ntor key to get by */
+  /* r1 is a minimal, RSA-only descriptor, with DirPort and IPv6 */
+  r1 = basic_routerinfo_new("Magri", 0xc0a80001u /* 192.168.0.1 */,
+                            9000, 9003,
+                            1000, 5000, 10000,
+                            0,
+                            NULL);
+
+ /* Fake just enough of an ntor key to get by */
   curve25519_keypair_t r1_onion_keypair;
   curve25519_keypair_generate(&r1_onion_keypair, 0);
   r1->onion_curve25519_pkey = tor_memdup(&r1_onion_keypair.pubkey,
                                          sizeof(curve25519_public_key_t));
-  r1->identity_pkey = crypto_pk_dup_key(pk2);
-  r1->bandwidthrate = 1000;
-  r1->bandwidthburst = 5000;
-  r1->bandwidthcapacity = 10000;
+
+  /* Now add IPv6 */
+  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
+  r1->ipv6_orport = 9999;
+
   r1->exit_policy = NULL;
-  r1->nickname = tor_strdup("Magri");
-  r1->platform = tor_strdup(platform);
 
-  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex1->policy_type = ADDR_POLICY_ACCEPT;
-  tor_addr_from_ipv4h(&ex1->addr, 0);
-  ex1->maskbits = 0;
-  ex1->prt_min = ex1->prt_max = 80;
-  ex2->policy_type = ADDR_POLICY_REJECT;
-  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
-  ex2->maskbits = 8;
-  ex2->prt_min = ex2->prt_max = 24;
-  r2 = tor_malloc_zero(sizeof(routerinfo_t));
-  r2->addr = 0x0a030201u; /* 10.3.2.1 */
+  /* XXXX+++ router_dump_to_string should really take this from ri. */
+  options->ContactInfo = tor_strdup("Magri White "
+                                    "<magri@elsewhere.example.com>");
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  tor_free(options->ContactInfo);
+  cleanup_mock_configured_ports();
+
+  /* Synthesise a router descriptor, without the signature */
+  chunks = smartlist_new();
+
+  smartlist_add(chunks, get_new_router_line(r1));
+  smartlist_add_strdup(chunks, "or-address [1:2:3:4::]:9999\n");
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r1));
+  smartlist_add(chunks, get_new_fingerprint_line(r1));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r1));
+
+  smartlist_add(chunks, get_new_onion_key_block(r1));
+  smartlist_add(chunks, get_new_signing_key_block(r1));
+
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
+
+  smartlist_add_strdup(chunks, "contact Magri White "
+                               "<magri@elsewhere.example.com>\n");
+
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r1_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "reject *:*\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
+
+  smartlist_add_strdup(chunks, "router-signature\n");
+
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+                             * twice */
+
+  tt_str_op(buf,OP_EQ, buf2);
+  tor_free(buf);
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  cleanup_mock_configured_ports();
+
+  /* Now, try to parse buf */
+  cp = buf;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  tor_free(buf);
+  routerinfo_free(rp1);
+
+  /* Test extrainfo creation.
+   * We avoid calling router_build_fresh_unsigned_routerinfo(), because it's
+   * too complex. Instead, we re-use the manually-created routerinfos.
+   */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r1, NULL);
+
+  /* router_build_fresh_signed_extrainfo() passes the result of
+   * get_master_signing_key_cert() directly to tor_cert_dup(), which fails on
+   * NULL. But we want a NULL ei->cache_info.signing_key_cert to test the
+   * non-ed key path.
+   */
+  MOCK(tor_cert_dup, mock_tor_cert_dup_null);
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  /* Test some of the low-level static functions. */
+  e1 = router_build_fresh_signed_extrainfo(r1);
+  tt_assert(e1);
+  router_update_routerinfo_from_extrainfo(r1, e1);
+  rv = router_dump_and_sign_routerinfo_descriptor_body(r1);
+  tt_assert(rv == 0);
+  msg = "";
+  rv = routerinfo_incompatible_with_extrainfo(r1->identity_pkey, e1,
+                                              &r1->cache_info, &msg);
+  /* If they are incompatible, fail and show the msg string */
+  tt_str_op(msg, OP_EQ, "");
+  tt_assert(rv == 0);
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  UNMOCK(tor_cert_dup);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, e1);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r1->cache_info.signed_descriptor_body);
+  cp = r1->cache_info.signed_descriptor_body;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  routerinfo_free(rp1);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e1->cache_info.signed_descriptor_body);
+  cp = e1->cache_info.signed_descriptor_body;
+  ep1 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, ep1);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep1);
+
+ done:
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->ContactInfo);
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r1);
+  routerinfo_free(rp1);
+
+  extrainfo_free(e1);
+  extrainfo_free(ep1);
+
+  tor_free(rsa_cc);
+
+  tor_free(buf);
+  tor_free(buf2);
+}
+
+/* Check that the exit policy in rp2 is as expected. */
+#define CHECK_PARSED_EXIT_POLICY(rp2) \
+STMT_BEGIN \
+  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); \
+ \
+  p = smartlist_get(rp2->exit_policy, 0); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); \
+  tt_assert(tor_addr_is_null(&p->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 0); \
+  tt_int_op(p->prt_min,OP_EQ, 80); \
+  tt_int_op(p->prt_max,OP_EQ, 80); \
+ \
+  p = smartlist_get(rp2->exit_policy, 1); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); \
+  tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 8); \
+  tt_int_op(p->prt_min,OP_EQ, 24); \
+  tt_int_op(p->prt_max,OP_EQ, 24); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA + ed25519
+ * router.
+ */
+static void
+test_dir_formats_rsa_ed25519(void *arg)
+{
+  char *buf = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
+  crypto_pk_t *r2_onion_pkey = NULL;
+  char cert_buf[256];
+  uint8_t *rsa_cc = NULL;
+  time_t now = time(NULL);
+
+  routerinfo_t *r2 = NULL;
+  extrainfo_t *e2 = NULL;
+  routerinfo_t *r2_out = NULL;
+  routerinfo_t *rp2 = NULL;
+  extrainfo_t *ep2 = NULL;
+  addr_policy_t *ex1, *ex2;
+  const addr_policy_t *p;
+
+  smartlist_t *chunks = NULL;
+  int rv = -1;
+
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
+
+  hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
+
+  /* r2 is a RSA + ed25519 descriptor, with an exit policy, but no DirPort or
+   * IPv6 */
+  r2 = basic_routerinfo_new("Fred", 0x0a030201u /* 10.3.2.1 */,
+                            9005, 0,
+                            3000, 3000, 3000,
+                            5,
+                            &r2_onion_pkey);
+
+  /* Fake just enough of an ntor key to get by */
+  curve25519_keypair_t r2_onion_keypair;
+  curve25519_keypair_generate(&r2_onion_keypair, 0);
+  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
+                                         sizeof(curve25519_public_key_t));
+
+  /* Now add relay ed25519 keys
+   * We can't use init_mock_ed_keys() here, because the keys are seeded */
   ed25519_keypair_t kp1, kp2;
   ed25519_secret_key_from_seed(&kp1.seckey,
                           (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
@@ -228,157 +950,78 @@ test_dir_formats(void *arg)
                                          &kp2.pubkey,
                                          now, 86400,
                                          CERT_FLAG_INCLUDE_SIGNING_KEY);
-  r2->platform = tor_strdup(platform);
-  r2->cache_info.published_on = 5;
-  r2->or_port = 9005;
-  r2->dir_port = 0;
-  r2->supports_tunnelled_dir_requests = 1;
-  router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len);
-  curve25519_keypair_t r2_onion_keypair;
-  curve25519_keypair_generate(&r2_onion_keypair, 0);
-  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
-                                         sizeof(curve25519_public_key_t));
-  r2->identity_pkey = crypto_pk_dup_key(pk1);
-  r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000;
+
+  /* Now add an exit policy */
+  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex1->policy_type = ADDR_POLICY_ACCEPT;
+  tor_addr_from_ipv4h(&ex1->addr, 0);
+  ex1->maskbits = 0;
+  ex1->prt_min = ex1->prt_max = 80;
+  ex2->policy_type = ADDR_POLICY_REJECT;
+  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
+  ex2->maskbits = 8;
+  ex2->prt_min = ex2->prt_max = 24;
+
   r2->exit_policy = smartlist_new();
   smartlist_add(r2->exit_policy, ex1);
   smartlist_add(r2->exit_policy, ex2);
-  r2->nickname = tor_strdup("Fred");
-
-  tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
-                                                    &pk1_str_len));
-  tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str,
-                                                    &pk2_str_len));
-
-  /* XXXX+++ router_dump_to_string should really take this from ri.*/
-  options->ContactInfo = tor_strdup("Magri White "
-                                    "<magri@elsewhere.example.com>");
-  /* Skip reachability checks for DirPort and tunnelled-dir-server */
-  options->AssumeReachable = 1;
 
-  /* Fake just enough of an ORPort and DirPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
-
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9000;
-  smartlist_add(mocked_configured_ports, &orport);
-
-  memset(&dirport, 0, sizeof(dirport));
-  dirport.type = CONN_TYPE_DIR_LISTENER;
-  dirport.addr.family = AF_INET;
-  dirport.port = 9003;
-  smartlist_add(mocked_configured_ports, &dirport);
-
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  tor_free(options->ContactInfo);
+  buf = router_dump_router_to_string(r2,
+                                     r2->identity_pkey, r2_onion_pkey,
+                                     &r2_onion_keypair, &kp2);
   tt_assert(buf);
 
-  strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n"
-          "or-address [1:2:3:4::]:9999\n"
-          "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:00\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-  /* XXX the "0" above is hard-coded, but even if we made it reflect
-   * uptime, that still wouldn't make it right, because the two
-   * descriptors might be made on different seconds... hm. */
-         "bandwidth 1000 5000 10000\n"
-          "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "contact Magri White <magri@elsewhere.example.com>\n",
-          sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r1_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "reject *:*\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\nrouter-signature\n", sizeof(buf2));
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
-                             * twice */
+  cleanup_mock_configured_ports();
 
-  tt_str_op(buf,OP_EQ, buf2);
-  tor_free(buf);
+  chunks = smartlist_new();
 
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-  tt_assert(buf);
-  cp = buf;
-  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp1);
-  tt_int_op(rp1->addr,OP_EQ, r1->addr);
-  tt_int_op(rp1->or_port,OP_EQ, r1->or_port);
-  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port);
-  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate);
-  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst);
-  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity);
-  crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey,
-                                                      rp1->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0);
-  tt_assert(rp1->supports_tunnelled_dir_requests);
-  //tt_assert(rp1->exit_policy == NULL);
-  tor_free(buf);
+  /* Synthesise a router descriptor, without the signatures */
+  smartlist_add(chunks, get_new_router_line(r2));
 
-  strlcpy(buf2,
-          "router Fred 10.3.2.1 9005 0 0\n"
-          "identity-ed25519\n"
-          "-----BEGIN ED25519 CERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks,
+                       "identity-ed25519\n"
+                       "-----BEGIN ED25519 CERT-----\n");
   base64_encode(cert_buf, sizeof(cert_buf),
                 (const char*)r2->cache_info.signing_key_cert->encoded,
                 r2->cache_info.signing_key_cert->encoded_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END ED25519 CERT-----\n", sizeof(buf2));
-  strlcat(buf2, "master-key-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END ED25519 CERT-----\n");
+
+  smartlist_add_strdup(chunks, "master-key-ed25519 ");
   {
     char k[ED25519_BASE64_LEN+1];
-    tt_int_op(ed25519_public_to_base64(k,
-                          &r2->cache_info.signing_key_cert->signing_key),
-              OP_GE, 0);
-    strlcat(buf2, k, sizeof(buf2));
-    strlcat(buf2, "\n", sizeof(buf2));
+    ed25519_public_to_base64(k, &r2->cache_info.signing_key_cert->signing_key);
+    smartlist_add_strdup(chunks, k);
+    smartlist_add_strdup(chunks, "\n");
   }
-  strlcat(buf2, "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:05\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk1, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-          "bandwidth 3000 3000 3000\n", sizeof(buf2));
-  strlcat(buf2, "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r2));
+  smartlist_add(chunks, get_new_fingerprint_line(r2));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r2));
+
+  smartlist_add(chunks, get_new_onion_key_block(r2));
+  smartlist_add(chunks, get_new_signing_key_block(r2));
+
   int rsa_cc_len;
-  rsa_cc = make_tap_onion_key_crosscert(pk2,
+  rsa_cc = make_tap_onion_key_crosscert(r2_onion_pkey,
                                         &kp1.pubkey,
-                                        pk1,
+                                        r2->identity_pkey,
                                         &rsa_cc_len);
   tt_assert(rsa_cc);
   base64_encode(cert_buf, sizeof(cert_buf), (char*)rsa_cc, rsa_cc_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, "onion-key-crosscert\n"
-          "-----BEGIN CROSSCERT-----\n", sizeof(buf2));
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END CROSSCERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks, "onion-key-crosscert\n"
+                       "-----BEGIN CROSSCERT-----\n");
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END CROSSCERT-----\n");
   int ntor_cc_sign;
   {
     tor_cert_t *ntor_cc = NULL;
@@ -393,112 +1036,165 @@ test_dir_formats(void *arg)
                 BASE64_ENCODE_MULTILINE);
     tor_cert_free(ntor_cc);
   }
-  tor_snprintf(buf2+strlen(buf2), sizeof(buf2)-strlen(buf2),
+  smartlist_add_asprintf(chunks,
                "ntor-onion-key-crosscert %d\n"
                "-----BEGIN ED25519 CERT-----\n"
                "%s"
                "-----END ED25519 CERT-----\n", ntor_cc_sign, cert_buf);
 
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r2_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "accept *:80\nreject 18.0.0.0/8:24\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\n", sizeof(buf2));
-  strlcat(buf2, "router-sig-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
 
-  /* Fake just enough of an ORPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r2_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "accept *:80\nreject 18.0.0.0/8:24\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
 
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9005;
-  smartlist_add(mocked_configured_ports, &orport);
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
 
-  buf = router_dump_router_to_string(r2, pk1, pk2, &r2_onion_keypair, &kp2);
-  tt_assert(buf);
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare either sig; they're never the same
                              * twice */
 
   tt_str_op(buf, OP_EQ, buf2);
   tor_free(buf);
 
-  buf = router_dump_router_to_string(r2, pk1, NULL, NULL, NULL);
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  buf = router_dump_router_to_string(r2, r2->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  cleanup_mock_configured_ports();
 
-  /* Reset for later */
+  /* Now, try to parse buf */
   cp = buf;
   rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp2);
-  tt_int_op(rp2->addr,OP_EQ, r2->addr);
-  tt_int_op(rp2->or_port,OP_EQ, r2->or_port);
-  tt_int_op(rp2->dir_port,OP_EQ, r2->dir_port);
-  tt_int_op(rp2->bandwidthrate,OP_EQ, r2->bandwidthrate);
-  tt_int_op(rp2->bandwidthburst,OP_EQ, r2->bandwidthburst);
-  tt_int_op(rp2->bandwidthcapacity,OP_EQ, r2->bandwidthcapacity);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
   tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
              r2->onion_curve25519_pkey->public_key,
              CURVE25519_PUBKEY_LEN);
-  onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey,
-                                         rp2->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0);
-  tt_assert(rp2->supports_tunnelled_dir_requests);
-
-  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2);
-
-  p = smartlist_get(rp2->exit_policy, 0);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT);
-  tt_assert(tor_addr_is_null(&p->addr));
-  tt_int_op(p->maskbits,OP_EQ, 0);
-  tt_int_op(p->prt_min,OP_EQ, 80);
-  tt_int_op(p->prt_max,OP_EQ, 80);
-
-  p = smartlist_get(rp2->exit_policy, 1);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT);
-  tt_assert(tor_addr_eq(&p->addr, &ex2->addr));
-  tt_int_op(p->maskbits,OP_EQ, 8);
-  tt_int_op(p->prt_min,OP_EQ, 24);
-  tt_int_op(p->prt_max,OP_EQ, 24);
-
-#if 0
-  /* Okay, now for the directories. */
-  {
-    fingerprint_list = smartlist_new();
-    crypto_pk_get_fingerprint(pk2, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
-    crypto_pk_get_fingerprint(pk1, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  tor_free(buf);
+  routerinfo_free(rp2);
+
+  /* Test extrainfo creation. */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r2, r2_onion_pkey);
+
+  /* router_build_fresh_descriptor() requires
+   * router_build_fresh_unsigned_routerinfo(), but the implementation is
+   * too complex. Instead, we re-use r2.
+   */
+  mocked_routerinfo = r2;
+  MOCK(router_build_fresh_unsigned_routerinfo,
+       mock_router_build_fresh_unsigned_routerinfo);
+
+  /* r2 uses ed25519, so we need to mock the ed key functions */
+  mocked_master_signing_key = &kp2;
+  MOCK(get_master_signing_keypair, mock_get_master_signing_keypair);
+
+  mocked_signing_key_cert = r2->cache_info.signing_key_cert;
+  MOCK(get_master_signing_key_cert, mock_get_master_signing_key_cert);
+
+  mocked_curve25519_onion_key = &r2_onion_keypair;
+  MOCK(get_current_curve25519_keypair, mock_get_current_curve25519_keypair);
+
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
+
+  /* Test the high-level interface. */
+  rv = router_build_fresh_descriptor(&r2_out, &e2);
+  if (rv < 0) {
+    /* router_build_fresh_descriptor() frees r2 on failure. */
+    r2 = NULL;
+    /* Get rid of an alias to rp2 */
+    r2_out = NULL;
   }
+  tt_assert(rv == 0);
+  tt_assert(r2_out);
+  tt_assert(e2);
+  /* Guaranteed by mock_router_build_fresh_unsigned_routerinfo() */
+  tt_ptr_op(r2_out, OP_EQ, r2);
+  /* Get rid of an alias to r2 */
+  r2_out = NULL;
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  mocked_routerinfo = NULL;
+  UNMOCK(router_build_fresh_unsigned_routerinfo);
+  mocked_master_signing_key = NULL;
+  UNMOCK(get_master_signing_keypair);
+  mocked_signing_key_cert = NULL;
+  UNMOCK(get_master_signing_key_cert);
+  mocked_curve25519_onion_key = NULL;
+  UNMOCK(get_current_curve25519_keypair);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, e2);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r2->cache_info.signed_descriptor_body);
+  cp = r2->cache_info.signed_descriptor_body;
+  rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
 
-#endif /* 0 */
-  dirserv_free_fingerprint_list();
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
+  tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
+            r2->onion_curve25519_pkey->public_key,
+            CURVE25519_PUBKEY_LEN);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  routerinfo_free(rp2);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e2->cache_info.signed_descriptor_body);
+  cp = e2->cache_info.signed_descriptor_body;
+  ep2 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, ep2);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep2);
 
  done:
-  if (r1)
-    routerinfo_free(r1);
-  if (r2)
-    routerinfo_free(r2);
-  if (rp2)
-    routerinfo_free(rp2);
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r2);
+  routerinfo_free(r2_out);
+  routerinfo_free(rp2);
+
+  extrainfo_free(e2);
+  extrainfo_free(ep2);
 
   tor_free(rsa_cc);
+  crypto_pk_free(r2_onion_pkey);
+
   tor_free(buf);
-  tor_free(pk1_str);
-  tor_free(pk2_str);
-  if (pk1) crypto_pk_free(pk1);
-  if (pk2) crypto_pk_free(pk2);
-  if (rp1) routerinfo_free(rp1);
-  tor_free(dir1); /* XXXX And more !*/
-  tor_free(dir2); /* And more !*/
+  tor_free(buf2);
 }
 
 #include "failing_routerdescs.inc"
@@ -1767,7 +2463,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, "", 0);
   setup_capture_of_logs(LOG_WARN);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
   expect_log_msg("Empty bandwidth file\n");
   teardown_capture_of_logs();
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1783,7 +2480,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op("", OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1794,7 +2493,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, header_lines_v100, 0);
   bw_file_headers = smartlist_new();
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1807,7 +2508,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1818,7 +2521,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100);
   write_str_to_file(fname, content, 0);
   tor_free(content);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
 
   /* Test bandwidth file including v1.1.0 bandwidth headers and
    * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */
@@ -1828,7 +2532,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1844,7 +2550,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1861,7 +2569,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1872,7 +2582,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110_no_terminator, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1883,7 +2595,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1898,7 +2612,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1913,7 +2629,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1929,7 +2647,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1946,7 +2666,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1964,7 +2686,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1985,7 +2709,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   /* force bw_file_headers to be bigger than
@@ -2014,7 +2740,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
@@ -2032,11 +2759,13 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_not_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
  done:
+  unlink(fname);
   tor_free(fname);
   tor_free(header_lines_v100);
   tor_free(header_lines_v110_no_terminator);
@@ -2863,11 +3592,17 @@ test_a_networkstatus(
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   /* Parse certificates and keys. */
-  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
   tt_assert(cert1);
-  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                           strlen(AUTHORITY_CERT_2),
+                                           NULL);
   tt_assert(cert2);
-  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                           strlen(AUTHORITY_CERT_3),
+                                           NULL);
   tt_assert(cert3);
   sign_skey_1 = crypto_pk_new();
   sign_skey_2 = crypto_pk_new();
@@ -2969,7 +3704,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_NS);
   tt_assert(consensus_text);
-  con = networkstatus_parse_vote_from_string(consensus_text, NULL,
+  con = networkstatus_parse_vote_from_string_(consensus_text, NULL,
                                              NS_TYPE_CONSENSUS);
   tt_assert(con);
   //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n",
@@ -2981,7 +3716,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_MICRODESC);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -3080,13 +3815,13 @@ test_a_networkstatus(
     tt_assert(consensus_text3);
     tt_assert(consensus_text_md2);
     tt_assert(consensus_text_md3);
-    con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL,
+    con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL,
+    con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL,
+    con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL,
+    con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL,
                                                 NS_TYPE_CONSENSUS);
     tt_assert(con2);
     tt_assert(con3);
@@ -3864,6 +4599,62 @@ mock_get_options(void)
   return mock_options;
 }
 
+/**
+ * Test dirauth_get_b64_digest_bw_file.
+ * This function should be near the other bwauth functions, but it needs
+ * mock_get_options, that is only defined here.
+ */
+
+static void
+test_dir_bwauth_bw_file_digest256(void *arg)
+{
+  (void)arg;
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  char *fname = tor_strdup(get_fname("V3BandwidthsFile"));
+  /* Initialize to a wrong digest. */
+  uint8_t digest[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+
+  /* Digest of an empty string. Initialize to a wrong digest. */
+  char digest_empty_str[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_empty_str, "", 0, DIGEST_SHA256);
+
+  /* Digest of the content. Initialize to a wrong digest. */
+  char digest_expected[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_expected, content, strlen(content), DIGEST_SHA256);
+
+  /* When the bandwidth file can not be found. */
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_empty_str, DIGEST256_LEN);
+
+  /* When there is a timestamp but it is too old. */
+  write_str_to_file(fname, content, 0);
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  /* The digest will be correct. */
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+  update_approx_time(1541171221);
+
+  /* When there is a bandwidth file and it can be read. */
+  tt_int_op(0, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+ done:
+  unlink(fname);
+  tor_free(fname);
+  update_approx_time(time(NULL));
+}
+
 static void
 reset_routerstatus(routerstatus_t *rs,
                    const char *hex_identity_digest,
@@ -6087,6 +6878,80 @@ test_dir_find_dl_min_delay(void* data)
 }
 
 static void
+test_dir_matching_flags(void *arg)
+{
+  (void) arg;
+  routerstatus_t *rs_noflags = NULL;
+  routerstatus_t *rs = NULL;
+  char *s = NULL;
+
+  smartlist_t *tokens = smartlist_new();
+  memarea_t *area = memarea_new();
+
+  int expected_val_when_unused = 0;
+
+  const char *ex_noflags =
+    "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 "
+       "192.168.0.1 9001 0\n"
+    "m thisoneislongerbecauseitisa256bitmddigest33\n"
+    "s\n";
+  const char *cp = ex_noflags;
+  rs_noflags = routerstatus_parse_entry_from_string(
+         area, &cp,
+         cp + strlen(cp),
+         tokens, NULL, NULL,
+         MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);
+  tt_assert(rs_noflags);
+
+#define FLAG(string, field) STMT_BEGIN {        \
+    tor_asprintf(&s,\
+                 "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \
+                 "192.168.0.1 9001 0\n"                                 \
+                 "m thisoneislongerbecauseitisa256bitmddigest33\n"      \
+                 "s %s\n", string);                                     \
+    cp = s;                                                             \
+    rs =  routerstatus_parse_entry_from_string(                         \
+      area, &cp,                                                        \
+      cp + strlen(cp),                                                  \
+      tokens, NULL, NULL,                                               \
+      MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);                  \
+    /* the field should usually be 0 when no flags are listed */        \
+    tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused);      \
+    /* the field should be 1 when this flags islisted */                \
+    tt_int_op(rs->field, OP_EQ, 1);                                     \
+    tor_free(s);                                                        \
+    routerstatus_free(rs);                                              \
+} STMT_END
+
+  FLAG("Authority", is_authority);
+  FLAG("BadExit", is_bad_exit);
+  FLAG("Exit", is_exit);
+  FLAG("Fast", is_fast);
+  FLAG("Guard", is_possible_guard);
+  FLAG("HSDir", is_hs_dir);
+  FLAG("Stable", is_stable);
+  FLAG("StaleDesc", is_staledesc);
+  FLAG("V2Dir", is_v2_dir);
+
+  // These flags are assumed to be set whether they're declared or not.
+  expected_val_when_unused = 1;
+  FLAG("Running", is_flagged_running);
+  FLAG("Valid", is_valid);
+  expected_val_when_unused = 0;
+
+  // These flags are no longer used, but still parsed.
+  FLAG("Named", is_named);
+  FLAG("Unnamed", is_unnamed);
+
+ done:
+  tor_free(s);
+  routerstatus_free(rs);
+  routerstatus_free(rs_noflags);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+}
+
+static void
 test_dir_assumed_flags(void *arg)
 {
   (void)arg;
@@ -6101,9 +6966,10 @@ test_dir_assumed_flags(void *arg)
        "192.168.0.1 9001 0\n"
     "m thisoneislongerbecauseitisa256bitmddigest33\n"
     "s Fast Guard Stable\n";
+  const char *eos = str1 + strlen(str1);
 
   const char *cp = str1;
-  rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
+  rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL,
                                             24, FLAV_MICRODESC);
   tt_assert(rs);
   tt_assert(rs->is_flagged_running);
@@ -6356,7 +7222,22 @@ test_dir_format_versions_list(void *arg)
 
 struct testcase_t dir_tests[] = {
   DIR_LEGACY(nicknames),
-  DIR_LEGACY(formats),
+  /* extrainfo without any stats */
+  DIR_ARG(formats_rsa, TT_FORK, ""),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, ""),
+  /* on a bridge */
+  DIR_ARG(formats_rsa, TT_FORK, "b"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "b"),
+  /* extrainfo with basic stats */
+  DIR_ARG(formats_rsa, TT_FORK, "e"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "e"),
+  DIR_ARG(formats_rsa, TT_FORK, "be"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "be"),
+  /* extrainfo with all stats */
+  DIR_ARG(formats_rsa, TT_FORK, "es"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "es"),
+  DIR_ARG(formats_rsa, TT_FORK, "bes"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "bes"),
   DIR(routerinfo_parsing, 0),
   DIR(extrainfo_parsing, 0),
   DIR(parse_router_list, TT_FORK),
@@ -6370,6 +7251,7 @@ struct testcase_t dir_tests[] = {
   DIR_LEGACY(measured_bw_kb_line_is_after_headers),
   DIR_LEGACY(measured_bw_kb_cache),
   DIR_LEGACY(dirserv_read_measured_bandwidths),
+  DIR(bwauth_bw_file_digest256, 0),
   DIR_LEGACY(param_voting),
   DIR(param_voting_lookup, 0),
   DIR_LEGACY(v3_networkstatus),
@@ -6410,6 +7292,7 @@ struct testcase_t dir_tests[] = {
   DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"),
   DIR_ARG(find_dl_min_delay, TT_FORK, "car"),
   DIR(assumed_flags, 0),
+  DIR(matching_flags, 0),
   DIR(networkstatus_compute_bw_weights_v10, 0),
   DIR(platform_str, 0),
   DIR(networkstatus_consensus_has_ipv6, TT_FORK),
diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c
index 3723d6c31b..0b87e29873 100644
--- a/src/test/test_dir_common.c
+++ b/src/test/test_dir_common.c
@@ -42,14 +42,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1,
 {
   /* Parse certificates and keys. */
   authority_cert_t *cert;
-  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                          strlen(AUTHORITY_CERT_1),
+                                          NULL);
   tt_assert(cert);
   tt_assert(cert->identity_key);
   *cert1 = cert;
   tt_assert(*cert1);
-  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                            strlen(AUTHORITY_CERT_2),
+                                            NULL);
   tt_assert(*cert2);
-  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                            strlen(AUTHORITY_CERT_3),
+                                            NULL);
   tt_assert(*cert3);
   *sign_skey_1 = crypto_pk_new();
   *sign_skey_2 = crypto_pk_new();
@@ -266,7 +272,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out,
   /* dump the vote and try to parse it. */
   v_text = format_networkstatus_vote(sign_skey, vote);
   tt_assert(v_text);
-  *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE);
+  *vote_out = networkstatus_parse_vote_from_string(v_text,
+                                                   strlen(v_text),
+                                                   NULL, NS_TYPE_VOTE);
 
  done:
   if (v_text)
@@ -424,4 +432,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert,
 
   return 0;
 }
-
diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h
index d6c5241b14..619dc83eb9 100644
--- a/src/test/test_dir_common.h
+++ b/src/test/test_dir_common.h
@@ -3,6 +3,9 @@
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_DIR_COMMON_H
+#define TOR_TEST_DIR_COMMON_H
+
 #include "core/or/or.h"
 #include "feature/nodelist/networkstatus.h"
 
@@ -49,3 +52,4 @@ int dir_common_construct_vote_3(networkstatus_t **vote,
                         networkstatus_t **vote_out, int *n_vrs, time_t now,
                         int clear_rl);
 
+#endif /* !defined(TOR_TEST_DIR_COMMON_H) */
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index 90691fff94..edfd0c74e1 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -72,6 +72,8 @@ ENABLE_GCC_WARNING(overlength-strings)
 #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \
   "Consensus not signed by sufficient number of requested authorities\r\n\r\n"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
 static dir_connection_t *
 new_dir_conn(void)
 {
@@ -477,8 +479,7 @@ static or_options_t *mock_options = NULL;
 static void
 init_mock_options(void)
 {
-  mock_options = tor_malloc(sizeof(or_options_t));
-  memset(mock_options, 0, sizeof(or_options_t));
+  mock_options = options_new();
   mock_options->TestingTorNetwork = 1;
   mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp"));
   mock_options->CacheDirectory = tor_strdup(mock_options->DataDirectory);
@@ -1275,7 +1276,9 @@ test_dir_handle_get_server_keys_authority(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
@@ -1425,7 +1428,9 @@ test_dir_handle_get_server_keys_sk(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
@@ -2217,6 +2222,31 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data)
     tor_free(header);
 }
 
+static void
+test_dir_handle_get_status_vote_next_bandwidth_not_found(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL;
+  (void) data;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwdith"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(NOT_FOUND, OP_EQ, header);
+
+  done:
+    UNMOCK(connection_write_to_buf_impl_);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+}
+
 NS_DECL(const char*,
 dirvote_get_pending_consensus, (consensus_flavor_t flav));
 
@@ -2393,7 +2423,9 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2455,6 +2487,85 @@ test_dir_handle_get_status_vote_next_authority(void* data)
 }
 
 static void
+test_dir_handle_get_status_vote_next_bandwidth(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL, *body = NULL;
+  size_t body_used = 0;
+  (void) data;
+
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  init_mock_options();
+  MOCK(get_options, mock_get_options);
+  mock_options->V3BandwidthsFile = tor_strdup(
+    get_fname_rnd("V3BandwidthsFile")
+  );
+
+  write_str_to_file(mock_options->V3BandwidthsFile, content, 0);
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Type: text/plain\r\n"));
+  tt_assert(strstr(header, "Content-Encoding: identity\r\n"));
+  tt_assert(strstr(header, "Content-Length: 167\r\n"));
+
+  /* Check cache lifetime */
+  char expbuf[RFC1123_TIME_LEN+1];
+  time_t now = approx_time();
+  /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */
+  format_rfc1123_time(expbuf, (time_t)(now + 30*60));
+  char *expires = NULL;
+  /* Change to 'Cache-control: max-age=%d' if using http/1.1. */
+  tor_asprintf(&expires, "Expires: %s\r\n", expbuf);
+  tt_assert(strstr(header, expires));
+
+  tt_int_op(body_used, OP_EQ, strlen(body));
+  tt_str_op(content, OP_EQ, body);
+
+  tor_free(header);
+  tor_free(body);
+
+  /* Request the file using compression, the result should be the same. */
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth.z"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Encoding: deflate\r\n"));
+
+  /* Since using connection_write_to_buf_mock instead of mocking
+   * connection_buf_add_compress, the content is not actually compressed.
+   * If it would, the size and content would be different than the original.
+  */
+
+ done:
+  UNMOCK(get_options);
+  UNMOCK(connection_write_to_buf_impl_);
+  connection_free_minimal(TO_CONN(conn));
+  tor_free(header);
+  tor_free(body);
+  tor_free(expires);
+  or_options_free(mock_options);
+}
+
+static void
 test_dir_handle_get_status_vote_current_authority(void* data)
 {
   dir_connection_t *conn = NULL;
@@ -2471,7 +2582,9 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2627,6 +2740,8 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_current_authority, 0),
   DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_authority, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth_not_found, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth, 0),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK),
diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c
new file mode 100644
index 0000000000..a62c18e0c9
--- /dev/null
+++ b/src/test/test_dispatch.c
@@ -0,0 +1,278 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static dispatch_t *dispatcher_in_use=NULL;
+
+static void
+test_dispatch_max_in_u16_sl(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  uint16_t nums[] = { 10, 20, 30 };
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[1]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[0]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[2]);
+  tt_int_op(30, OP_EQ, max_in_u16_sl(sl, -1));
+
+ done:
+  smartlist_free(sl);
+}
+
+/* Construct an empty dispatch_t. */
+static void
+test_dispatch_empty(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+
+  cfg = dcfg_new();
+  d = dispatch_new(cfg);
+  tt_assert(d);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+static int total_recv1_simple = 0;
+static int total_recv2_simple = 0;
+
+static void
+simple_recv1(const msg_t *m)
+{
+  total_recv1_simple += m->aux_data__.u64;
+}
+
+static char *recv2_received = NULL;
+
+static void
+simple_recv2(const msg_t *m)
+{
+  tor_free(recv2_received);
+  recv2_received = dispatch_fmt_msg_data(dispatcher_in_use, m);
+
+  total_recv2_simple += m->aux_data__.u64*10;
+}
+
+/* Construct a dispatch_t with two messages, make sure that they both get
+ * delivered. */
+static void
+test_dispatch_simple(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  r += dcfg_add_recv(cfg,0,1,simple_recv1);
+  r += dcfg_msg_set_type(cfg,1,0);
+  r += dcfg_msg_set_chan(cfg,1,0);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2); /* second copy */
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = {.u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 7);
+  tt_int_op(total_recv2_simple, OP_EQ, 0);
+
+  total_recv1_simple = 0;
+  r = dispatch_send(d, 99, 0, 1, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+  tt_int_op(total_recv2_simple, OP_EQ, 140);
+
+  tt_str_op(recv2_received, OP_EQ, "<>"); // no format function was set.
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(recv2_received);
+}
+
+/* Construct a dispatch_t with a message and no reciever; make sure that it
+ * gets dropped properly. */
+static void
+test_dispatch_no_recipient(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = { .u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+struct coord { int x; int y; };
+static void
+free_coord(msg_aux_data_t d)
+{
+  tor_free(d.ptr);
+}
+static char *
+fmt_coord(msg_aux_data_t d)
+{
+  char *v;
+  struct coord *c = d.ptr;
+  tor_asprintf(&v, "[%d, %d]", c->x, c->y);
+  return v;
+}
+static dispatch_typefns_t coord_fns = {
+  .fmt_fn = fmt_coord,
+  .free_fn = free_coord,
+};
+static void
+alert_run_immediate(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)arg;
+  dispatch_flush(d, ch, INT_MAX);
+}
+
+static char *received_data=NULL;
+
+static void
+recv_typed_data(const msg_t *m)
+{
+  tor_free(received_data);
+  received_data = dispatch_fmt_msg_data(dispatcher_in_use, m);
+}
+
+static void
+test_dispatch_with_types(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,5,3);
+  r += dcfg_msg_set_chan(cfg,5,2);
+  r += dcfg_add_recv(cfg,5,0,recv_typed_data);
+  r += dcfg_type_set_fns(cfg,3,&coord_fns);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  /* Make this message get run immediately. */
+  r = dispatch_set_alert_fn(d, 2, alert_run_immediate, NULL);
+  tt_int_op(r, OP_EQ, 0);
+
+  struct coord *xy = tor_malloc(sizeof(*xy));
+  xy->x = 13;
+  xy->y = 37;
+  msg_aux_data_t data = {.ptr = xy};
+  r = dispatch_send(d, 99/*sender*/, 2/*channel*/, 5/*msg*/, 3/*type*/, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_str_op(received_data, OP_EQ, "[13, 37]");
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(received_data);
+  dispatcher_in_use = NULL;
+}
+
+static void
+test_dispatch_bad_type_setup(void *arg)
+{
+  (void)arg;
+  static dispatch_typefns_t fns;
+  dispatch_cfg_t *cfg = dcfg_new();
+
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &coord_fns));
+
+  fns = coord_fns;
+  fns.fmt_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  fns.free_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+ done:
+  dcfg_free(cfg);
+}
+
+#define T(name)                                                 \
+  { #name, test_dispatch_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t dispatch_tests[] = {
+  T(max_in_u16_sl),
+  T(empty),
+  T(simple),
+  T(no_recipient),
+  T(with_types),
+  T(bad_type_setup),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_dns.c b/src/test/test_dns.c
index 41a56f65d8..51ff8729d0 100644
--- a/src/test/test_dns.c
+++ b/src/test/test_dns.c
@@ -1,6 +1,7 @@
 /* Copyright (c) 2015-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#include "orconfig.h"
 #include "core/or/or.h"
 #include "test/test.h"
 
@@ -13,9 +14,71 @@
 
 #include "core/or/edge_connection_st.h"
 #include "core/or/or_circuit_st.h"
+#include "app/config/or_options_st.h"
+#include "app/config/config.h"
+
+#include <event2/event.h>
+#include <event2/dns.h>
 
 #define NS_MODULE dns
 
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+#define NS_SUBMODULE configure_nameservers_fallback
+
+static or_options_t options = {
+  .ORPort_set = 1,
+};
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &options;
+}
+
+static void
+NS(test_main)(void *arg)
+{
+  (void)arg;
+  tor_addr_t *nameserver_addr = NULL;
+
+  MOCK(get_options, mock_get_options);
+
+  options.ServerDNSResolvConfFile = (char *)"no_such_file!!!";
+
+  dns_init(); // calls configure_nameservers()
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+
+#ifndef _WIN32
+  tor_free(nameserver_addr);
+
+  options.ServerDNSResolvConfFile = (char *)"/dev/null";
+
+  dns_init();
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+#endif /* !defined(_WIN32) */
+
+  UNMOCK(get_options);
+
+ done:
+  tor_free(nameserver_addr);
+  return;
+}
+
+#undef NS_SUBMODULE
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
 #define NS_SUBMODULE clip_ttl
 
 static void
@@ -736,6 +799,9 @@ NS(test_main)(void *arg)
 #undef NS_SUBMODULE
 
 struct testcase_t dns_tests[] = {
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+   TEST_CASE(configure_nameservers_fallback),
+#endif
    TEST_CASE(clip_ttl),
    TEST_CASE(resolve),
    TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve),
diff --git a/src/test/test_dos.c b/src/test/test_dos.c
index 01d7cd006e..c17cfedbf6 100644
--- a/src/test/test_dos.c
+++ b/src/test/test_dos.c
@@ -411,7 +411,7 @@ test_dos_bucket_refill(void *arg)
   }
   tt_uint_op(current_circ_count, OP_EQ, 0);
   tt_uint_op(dos_stats->cc_stats.circuit_bucket, OP_EQ, current_circ_count);
-#endif
+#endif /* SIZEOF_TIME_T == 8 */
 
  done:
   tor_free(chan);
diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c
index fc7c5d5800..8f2d507743 100644
--- a/src/test/test_entryconn.c
+++ b/src/test/test_entryconn.c
@@ -11,7 +11,7 @@
 
 #include "feature/client/addressmap.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "feature/nodelist/nodelist.h"
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index a486b13ae1..d59b1c7153 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -5,6 +5,7 @@
 
 #define CIRCUITLIST_PRIVATE
 #define CIRCUITBUILD_PRIVATE
+#define CONFIG_PRIVATE
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
@@ -17,7 +18,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dirclient/dirclient.h"
@@ -67,7 +68,7 @@ static networkstatus_t *dummy_consensus = NULL;
 
 static smartlist_t *big_fake_net_nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 bfn_mock_nodelist_get_list(void)
 {
   return big_fake_net_nodes;
@@ -127,6 +128,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
   return 1; /* NOP */
 }
 
+#define REASONABLY_FUTURE " reasonably-future"
+#define REASONABLY_PAST " reasonably-past"
+
 /* Unittest setup function: Setup a fake network. */
 static void *
 big_fake_network_setup(const struct testcase_t *testcase)
@@ -138,9 +142,10 @@ big_fake_network_setup(const struct testcase_t *testcase)
   const int N_NODES = 271;
 
   const char *argument = testcase->setup_data;
-  int reasonably_live_consensus = 0;
+  int reasonably_future_consensus = 0, reasonably_past_consensus = 0;
   if (argument) {
-    reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL;
+    reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL;
+    reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL;
   }
 
   big_fake_net_nodes = smartlist_new();
@@ -193,16 +198,21 @@ big_fake_network_setup(const struct testcase_t *testcase)
       n->md->exit_policy = parse_short_policy("accept 443");
     }
 
+    n->nodelist_idx = smartlist_len(big_fake_net_nodes);
     smartlist_add(big_fake_net_nodes, n);
   }
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
   dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
-  if (reasonably_live_consensus) {
-    /* Make the dummy consensus valid from 4 hours ago, but expired an hour
+  if (reasonably_future_consensus) {
+    /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */
+    dummy_consensus->valid_after = approx_time() + 6*3600;
+    dummy_consensus->valid_until = approx_time() + 7*3600;
+  } else if (reasonably_past_consensus) {
+    /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours
      * ago. */
-    dummy_consensus->valid_after = approx_time() - 4*3600;
-    dummy_consensus->valid_until = approx_time() - 3600;
+    dummy_consensus->valid_after = approx_time() - 16*3600;
+    dummy_consensus->valid_until = approx_time() - 12*3600;
   } else {
     /* Make the dummy consensus valid for an hour either side of now. */
     dummy_consensus->valid_after = approx_time() - 3600;
@@ -226,12 +236,12 @@ mock_randomize_time_no_randomization(time_t a, time_t b)
   return a;
 }
 
-static or_options_t mocked_options;
+static or_options_t *mocked_options;
 
 static const or_options_t *
 mock_get_options(void)
 {
-  return &mocked_options;
+  return mocked_options;
 }
 
 #define TEST_IPV4_ADDR "123.45.67.89"
@@ -250,7 +260,7 @@ test_node_preferred_orport(void *arg)
   tor_addr_port_t ap;
 
   /* Setup options */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  mocked_options = options_new();
   /* We don't test ClientPreferIPv6ORPort here, because it's used in
    * nodelist_set_consensus to setup node.ipv6_preferred, which we set
    * directly. */
@@ -273,8 +283,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're only using IPv4, regardless
    * of whether we prefer it or not */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 0;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 0;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -287,8 +297,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
    * don't prefer the IPv6 address */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -296,28 +306,29 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv6 if we prefer it and
    * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 1;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
-  mocked_options.ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv4 = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
   /* Check the preferred address is IPv6 if we don't prefer it, but
    * ClientUseIPv4 is 0 */
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
+  mocked_options->ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv6 = 1;
+  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(mocked_options);
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
  done:
+  or_options_free(mocked_options);
   UNMOCK(get_options);
 }
 
@@ -3039,13 +3050,17 @@ static const struct testcase_setup_t upgrade_circuits = {
 
 #define BFN_TEST(name) \
   EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &big_fake_network, (void*)("reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_PAST) }
 
 #define UPGRADE_TEST(name, arg) \
   EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &upgrade_circuits, (void*)(arg " reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_PAST) }
 
 struct testcase_t entrynodes_tests[] = {
   NO_PREFIX_TEST(node_preferred_orport),
diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c
index 0c34d37a71..cb53a4e662 100644
--- a/src/test/test_extorport.c
+++ b/src/test/test_extorport.c
@@ -5,11 +5,11 @@
 #define EXT_ORPORT_PRIVATE
 #define MAINLOOP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/relay/ext_orport.h"
 #include "core/mainloop/mainloop.h"
@@ -18,6 +18,7 @@
 
 #include "test/test.h"
 #include "test/test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -176,7 +177,7 @@ test_ext_or_init_auth(void *arg)
   /* Shouldn't be initialized already, or our tests will be a bit
    * meaningless */
   ext_or_auth_cookie = tor_malloc_zero(32);
-  tt_assert(tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Now make sure we use a temporary file */
   fn = get_fname("ext_cookie_file");
@@ -201,7 +202,7 @@ test_ext_or_init_auth(void *arg)
   tt_mem_op(cp,OP_EQ, "! Extended ORPort Auth Cookie !\x0a", 32);
   tt_mem_op(cp+32,OP_EQ, ext_or_auth_cookie, 32);
   memcpy(cookie0, ext_or_auth_cookie, 32);
-  tt_assert(!tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(!fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Operation should be idempotent. */
   tt_int_op(0, OP_EQ, init_ext_or_cookie_authentication(1));
@@ -303,16 +304,6 @@ test_ext_or_cookie_auth(void *arg)
 }
 
 static void
-crypto_rand_return_tse_str(char *to, size_t n)
-{
-  if (n != 32) {
-    TT_FAIL(("Asked for %d bytes, not 32", (int)n));
-    return;
-  }
-  memcpy(to, "te road There is always another ", 32);
-}
-
-static void
 test_ext_or_cookie_auth_testvec(void *arg)
 {
   char *reply=NULL, *client_hash=NULL;
@@ -326,7 +317,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
   memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32);
   ext_or_auth_cookie_is_set = 1;
 
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
 
   tt_int_op(0, OP_EQ,
             handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply,
@@ -351,7 +342,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
                  "33b3cd77ff79bd80c2074bbf438119a2");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(reply);
   tor_free(client_hash);
   tor_free(mem_op_hex_tmp);
@@ -414,9 +405,9 @@ do_ext_or_handshake(or_connection_t *conn)
   CONTAINS("\x01\x00", 2);
   WRITE("\x01", 1);
   WRITE("But when I look ahead up the whi", 32);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tt_int_op(TO_CONN(conn)->state, OP_EQ,
             EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH);
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
@@ -481,9 +472,9 @@ test_ext_or_handshake(void *arg)
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
   /* send the rest of the nonce. */
   WRITE("ahead up the whi", 16);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   /* We should get the right reply from the server. */
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
            "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21"
@@ -582,7 +573,7 @@ test_ext_or_handshake(void *arg)
 
  done:
   UNMOCK(connection_write_to_buf_impl_);
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   if (conn)
     connection_free_minimal(TO_CONN(conn));
 #undef CONTAINS
@@ -596,6 +587,6 @@ struct testcase_t extorport_tests[] = {
   { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL },
   { "cookie_auth_testvec", test_ext_or_cookie_auth_testvec, TT_FORK,
     NULL, NULL },
-  { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL },
+  { "handshake", test_ext_or_handshake, TT_FORK, &helper_pubsub_setup, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c
index 802d0a9ebe..8eb3c2c928 100644
--- a/src/test/test_helpers.c
+++ b/src/test/test_helpers.c
@@ -14,15 +14,20 @@
 #include "orconfig.h"
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/nodelist.h"
 #include "core/or/relay.h"
 #include "feature/nodelist/routerlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
 
@@ -78,7 +83,7 @@ helper_setup_fake_routerlist(void)
 {
   int retval;
   routerlist_t *our_routerlist = NULL;
-  smartlist_t *our_nodelist = NULL;
+  const smartlist_t *our_nodelist = NULL;
 
   /* Read the file that contains our test descriptors. */
 
@@ -290,7 +295,7 @@ helper_parse_options(const char *conf)
   if (ret != 0) {
     goto done;
   }
-  ret = config_assign(&options_format, opt, line, 0, &msg);
+  ret = config_assign(get_options_mgr(), opt, line, 0, &msg);
   if (ret != 0) {
     goto done;
   }
@@ -303,3 +308,54 @@ helper_parse_options(const char *conf)
   }
   return opt;
 }
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Setup helper for tests that need pubsub active
+ *
+ * Does not hook up mainloop events.  Does set immediate delivery for
+ * all channels.
+ */
+void *
+helper_setup_pubsub(const struct testcase_t *testcase)
+{
+  dispatch_t *dispatcher = NULL;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  channel_id_t chan = get_channel_id("orconn");
+
+  (void)testcase;
+  (void)subsystems_add_pubsub(builder);
+  dispatcher = pubsub_builder_finalize(builder, NULL);
+  tor_assert(dispatcher);
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  chan = get_channel_id("ocirc");
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  return dispatcher;
+}
+
+/**
+ * Cleanup helper for tests that need pubsub active
+ */
+int
+helper_cleanup_pubsub(const struct testcase_t *testcase, void *dispatcher_)
+{
+  dispatch_t *dispatcher = dispatcher_;
+
+  (void)testcase;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+const struct testcase_setup_t helper_pubsub_setup = {
+  helper_setup_pubsub, helper_cleanup_pubsub
+};
diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h
index 9e376a563d..d82072bb34 100644
--- a/src/test/test_helpers.h
+++ b/src/test/test_helpers.h
@@ -7,6 +7,7 @@
 #define BUFFERS_PRIVATE
 
 #include "core/or/or.h"
+#include "tinytest.h"
 
 const char *get_yesterday_date_str(void);
 
@@ -31,5 +32,10 @@ or_options_t *helper_parse_options(const char *conf);
 
 extern const char TEST_DESCRIPTORS[];
 
+void *helper_setup_pubsub(const struct testcase_t *);
+int helper_cleanup_pubsub(const struct testcase_t *, void *);
+
+extern const struct testcase_setup_t helper_pubsub_setup;
+
 #endif /* !defined(TOR_TEST_HELPERS_H) */
 
diff --git a/src/test/test_hs.c b/src/test/test_hs.c
index a611b46ca6..2b69aae547 100644
--- a/src/test/test_hs.c
+++ b/src/test/test_hs.c
@@ -6,7 +6,7 @@
  * \brief Unit tests for hidden service.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define RENDCOMMON_PRIVATE
 #define RENDSERVICE_PRIVATE
@@ -15,6 +15,8 @@
 #include "core/or/or.h"
 #include "test/test.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
 #include "feature/rend/rendcommon.h"
@@ -321,6 +323,16 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg,OP_EQ, expected_msg);
   tor_free(received_msg);
 
+  /* test HSDir rate limited */
+  rend_query.auth_type = REND_NO_AUTH;
+  control_event_hsv2_descriptor_failed(&rend_query.base_, NULL,
+                                     "QUERY_RATE_LIMITED");
+  expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \
+                 "UNKNOWN REASON=QUERY_RATE_LIMITED\r\n";
+  tt_assert(received_msg);
+  tt_str_op(received_msg,OP_EQ, expected_msg);
+  tor_free(received_msg);
+
   /* Test invalid content with no HSDir fingerprint. */
   char *exp_msg;
   control_event_hs_descriptor_content(rend_query.onion_address,
@@ -436,7 +448,7 @@ test_hs_rend_data(void *arg)
   tt_int_op(client_v2->auth_type, OP_EQ, REND_BASIC_AUTH);
   tt_int_op(strlen(client_v2->onion_address), OP_EQ, 0);
   tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie,
+  tt_int_op(fast_mem_is_zero(client_v2->descriptor_cookie,
                             sizeof(client_v2->descriptor_cookie)), OP_EQ, 1);
   tt_assert(client->hsdirs_fp);
   tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
index 9182829116..86ac7e7fb1 100644
--- a/src/test/test_hs_cache.c
+++ b/src/test/test_hs_cache.c
@@ -10,6 +10,7 @@
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
 #define HS_CACHE_PRIVATE
+#define TOR_CHANNEL_INTERNAL_
 
 #include "trunnel/ed25519_cert.h"
 #include "feature/hs/hs_cache.h"
@@ -20,7 +21,12 @@
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_http.h"
 #include "lib/crypt_ops/crypto_format.h"
+#include "core/or/circuitlist.h"
+#include "core/or/channel.h"
 
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 
@@ -232,22 +238,31 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
   /* The dir conn we are going to simulate */
   dir_connection_t *conn = NULL;
+  edge_connection_t *edge_conn = NULL;
+  or_circuit_t *or_circ = NULL;
 
   /* First extract the blinded public key that we are going to use in our
      query, and then build the actual query string. */
   {
     char hsdir_cache_key[ED25519_BASE64_LEN+1];
 
-    retval = ed25519_public_to_base64(hsdir_cache_key,
-                                      blinded_key);
-    tt_int_op(retval, OP_EQ, 0);
+    ed25519_public_to_base64(hsdir_cache_key, blinded_key);
     tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key);
   }
 
   /* Simulate an HTTP GET request to the HSDir */
   conn = dir_connection_new(AF_INET);
+  tt_assert(conn);
+  TO_CONN(conn)->linked = 1; /* Signal that it is encrypted. */
   tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
-  TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */
+
+  /* Pretend this conn is anonymous. */
+  edge_conn = edge_connection_new(CONN_TYPE_EXIT, AF_INET);
+  TO_CONN(conn)->linked_conn = TO_CONN(edge_conn);
+  or_circ = or_circuit_new(0, NULL);
+  or_circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+  edge_conn->on_circuit = TO_CIRCUIT(or_circ);
+
   retval = directory_handle_command_get(conn, hsdir_query_str,
                                         NULL, 0);
   tt_int_op(retval, OP_EQ, 0);
@@ -264,8 +279,11 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
  done:
   tor_free(hsdir_query_str);
-  if (conn)
+  if (conn) {
+    tor_free(or_circ->p_chan);
+    connection_free_minimal(TO_CONN(conn)->linked_conn);
     connection_free_minimal(TO_CONN(conn));
+  }
 
   return received_desc;
 }
@@ -487,7 +505,7 @@ test_client_cache(void *arg)
                                        NULL, &published_desc_str);
     tt_int_op(retval, OP_EQ, 0);
     memcpy(wanted_subcredential, published_desc->subcredential, DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
+    tt_assert(!fast_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
   }
 
   /* Test handle_response_fetch_hsdesc_v3() */
diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c
index f8af631c8b..403509fbc8 100644
--- a/src/test/test_hs_cell.c
+++ b/src/test/test_hs_cell.c
@@ -20,6 +20,7 @@
 #include "feature/hs/hs_service.h"
 
 /* Trunnel. */
+#include "trunnel/hs/cell_common.h"
 #include "trunnel/hs/cell_establish_intro.h"
 
 /** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we
@@ -38,11 +39,13 @@ test_gen_establish_intro_cell(void *arg)
   /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
      attempt to parse it. */
   {
+    hs_service_config_t config;
+    memset(&config, 0, sizeof(config));
     /* We only need the auth key pair here. */
-    hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+    hs_service_intro_point_t *ip = service_intro_point_new(NULL);
     /* Auth key pair is generated in the constructor so we are all set for
      * using this IP object. */
-    ret = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+    ret = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
     service_intro_point_free(ip);
     tt_u64_op(ret, OP_GT, 0);
   }
@@ -97,6 +100,9 @@ test_gen_establish_intro_cell_bad(void *arg)
   trn_cell_establish_intro_t *cell = NULL;
   char circ_nonce[DIGEST_LEN] = {0};
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed);
 
@@ -107,8 +113,8 @@ test_gen_establish_intro_cell_bad(void *arg)
      ed25519_sign_prefixed() function and make it fail. */
   cell = trn_cell_establish_intro_new();
   tt_assert(cell);
-  ip = service_intro_point_new(NULL, 0, 0);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL);
+  ip = service_intro_point_new(NULL);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, NULL);
   service_intro_point_free(ip);
   expect_log_msg_containing("Unable to make signature for "
                             "ESTABLISH_INTRO cell.");
@@ -120,11 +126,97 @@ test_gen_establish_intro_cell_bad(void *arg)
   UNMOCK(ed25519_sign_prefixed);
 }
 
+static void
+test_gen_establish_intro_dos_ext(void *arg)
+{
+  ssize_t ret;
+  hs_service_config_t config;
+  hs_service_intro_point_t *ip = NULL;
+  trn_cell_extension_t *extensions = NULL;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  (void) arg;
+
+  memset(&config, 0, sizeof(config));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+
+  /* Case 1: No DoS parameters so no extension to be built. */
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 0);
+  trn_cell_extension_free(extensions);
+  extensions = NULL;
+
+  /* Case 2: Enable the DoS extension. Parameter set to 0 should indicate to
+   * disable the defense on the intro point but there should be an extension
+   * nonetheless in the cell. */
+  config.has_dos_defense_enabled = 1;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  const trn_cell_extension_field_t *field =
+    trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  const trn_cell_extension_dos_param_t *param =
+    trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+  /* Case 3: Enable the DoS extension. Parameter set to some normal values. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 42;
+  config.intro_dos_burst_per_sec = 250;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  field = trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 42);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 250);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+ done:
+  service_intro_point_free(ip);
+  trn_cell_extension_dos_free(dos);
+  trn_cell_extension_free(extensions);
+}
+
 struct testcase_t hs_cell_tests[] = {
   { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK,
     NULL, NULL },
   { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK,
     NULL, NULL },
+  { "gen_establish_intro_dos_ext", test_gen_establish_intro_dos_ext, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c
index 2f2bb45581..b777dafdfb 100644
--- a/src/test/test_hs_client.c
+++ b/src/test/test_hs_client.c
@@ -14,6 +14,7 @@
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -36,6 +37,7 @@
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_cache.h"
+#include "feature/rend/rendcache.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "core/mainloop/connection.h"
@@ -44,6 +46,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "core/or/entry_connection_st.h"
 #include "core/or/extend_info_st.h"
@@ -157,8 +160,7 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out,
     or_circ->rend_data = rend_data_dup(conn_rend_data);
   } else {
     /* prop224: Setup hs ident on the circuit */
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
   }
 
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -241,12 +243,14 @@ test_e2e_rend_circuit_setup_legacy(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -311,12 +315,14 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check that the crypt path has prop224 algorithm parameters */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -395,7 +401,7 @@ test_client_pick_intro(void *arg)
     tt_assert(fetched_desc);
     tt_mem_op(fetched_desc->subcredential, OP_EQ, desc->subcredential,
               DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)fetched_desc->subcredential,
+    tt_assert(!fast_mem_is_zero((char*)fetched_desc->subcredential,
                                DIGEST256_LEN));
     tor_free(encoded);
   }
@@ -403,6 +409,9 @@ test_client_pick_intro(void *arg)
   /* 2) Mark all intro points except _the chosen one_ as failed. Then query the
    *   desc and get a random intro: check that we got _the chosen one_. */
   {
+    /* Tell hs_get_extend_info_from_lspecs() to skip the private address check.
+     */
+    get_options_mutable()->ExtendAllowPrivateAddresses = 1;
     /* Pick the chosen intro point and get its ei */
     hs_desc_intro_point_t *chosen_intro_point =
       smartlist_get(desc->encrypted_data.intro_points, 0);
@@ -430,7 +439,7 @@ test_client_pick_intro(void *arg)
     for (int i = 0; i < 64; ++i) {
       extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
       tor_assert(ip);
-      tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
+      tt_assert(!fast_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
       tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest,
                 DIGEST_LEN);
       extend_info_free(ip);
@@ -476,6 +485,18 @@ test_client_pick_intro(void *arg)
     SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                             hs_desc_intro_point_t *, ip) {
       extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip);
+      /* desc_intro_point_to_extend_info() doesn't return IPv6 intro points
+       * yet, because we can't extend to them. See #24404, #24451, and #24181.
+       */
+      if (intro_ei == NULL) {
+        /* Pretend we're making a direct connection, and that we can use IPv6
+         */
+        get_options_mutable()->ClientUseIPv6 = 1;
+        intro_ei = hs_get_extend_info_from_lspecs(ip->link_specifiers,
+                                                  &ip->onion_key, 1);
+        tt_assert(tor_addr_family(&intro_ei->addr) == AF_INET6);
+      }
+      tt_assert(intro_ei);
       if (intro_ei) {
         const char *ptr;
         char ip_addr[TOR_ADDR_BUF_LEN];
@@ -942,8 +963,7 @@ test_close_intro_circuits_new_desc(void *arg)
     const hs_desc_intro_point_t *ip =
       smartlist_get(desc1->encrypted_data.intro_points, 0);
     tt_assert(ip);
-    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey,
-                                           HS_IDENT_CIRCUIT_INTRO);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
     ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
                         &ip->auth_key_cert->signed_key);
   }
@@ -986,6 +1006,91 @@ test_close_intro_circuits_new_desc(void *arg)
   UNMOCK(networkstatus_get_live_consensus);
 }
 
+static void
+test_close_intro_circuits_cache_clean(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  circuit_t *circ = NULL;
+  origin_circuit_t *ocirc = NULL;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  hs_init();
+  rend_cache_init();
+
+  /* This is needed because of the client cache expiration timestamp is based
+   * on having a consensus. See cached_client_descriptor_has_expired(). */
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Create and add to the global list a dummy client introduction circuits.
+   * We'll then make sure the hs_ident is attached to a dummy descriptor. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+  /* Build the first descriptor and cache it. */
+  {
+    char *encoded;
+    desc1 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc1);
+    ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* Store it */
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, 0);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* We'll pick one introduction point and associate it with the circuit. */
+  {
+    const hs_desc_intro_point_t *ip =
+      smartlist_get(desc1->encrypted_data.intro_points, 0);
+    tt_assert(ip);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+    ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+  }
+
+  /* Before we are about to clean up the intro circuits, make sure it is
+   * actually there. */
+  tt_assert(circuit_get_next_intro_circ(NULL, true));
+
+  /* Cleanup the client cache. The ns valid after time is what decides if the
+   * descriptor has expired so put it in the future enough (72h) so we are
+   * sure to always expire. */
+  mock_ns.valid_after = approx_time() + (72 * 24 * 60 * 60);
+  hs_cache_clean_as_client(0);
+
+  /* Once stored, our intro circuit should be closed because it is related to
+   * an old introduction point that doesn't exists anymore. */
+  tt_assert(!circuit_get_next_intro_circ(NULL, true));
+
+ done:
+  circuit_free(circ);
+  hs_descriptor_free(desc1);
+  hs_free_all();
+  rend_cache_free_all();
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
 struct testcase_t hs_client_tests[] = {
   { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy,
     TT_FORK, NULL, NULL },
@@ -1005,6 +1110,8 @@ struct testcase_t hs_client_tests[] = {
     TT_FORK, NULL, NULL },
   { "close_intro_circuits_new_desc", test_close_intro_circuits_new_desc,
     TT_FORK, NULL, NULL },
+  { "close_intro_circuits_cache_clean", test_close_intro_circuits_cache_clean,
+    TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c
index 2aff179687..de3f7e04f7 100644
--- a/src/test/test_hs_common.c
+++ b/src/test/test_hs_common.c
@@ -275,7 +275,7 @@ test_start_time_of_next_time_period(void *arg)
 static void
 cleanup_nodelist(void)
 {
-  smartlist_t *nodelist = nodelist_get_list();
+  const smartlist_t *nodelist = nodelist_get_list();
   SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
     tor_free(node->md);
     node->md = NULL;
@@ -502,6 +502,7 @@ test_desc_reupload_logic(void *arg)
                 pubkey_hex, strlen(pubkey_hex));
   hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
   service = tor_malloc_zero(sizeof(hs_service_t));
+  tt_assert(service);
   memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
   ed25519_secret_key_generate(&service->keys.identity_sk, 0);
   ed25519_public_key_generate(&service->keys.identity_pk,
@@ -603,6 +604,10 @@ test_desc_reupload_logic(void *arg)
   SMARTLIST_FOREACH(ns->routerstatus_list,
                     routerstatus_t *, rs, routerstatus_free(rs));
   smartlist_clear(ns->routerstatus_list);
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   networkstatus_vote_free(ns);
   cleanup_nodelist();
   hs_free_all();
@@ -630,7 +635,7 @@ test_disaster_srv(void *arg)
   get_disaster_srv(1, srv_one);
   get_disaster_srv(2, srv_two);
 
-  /* Check that the cached ones where updated */
+  /* Check that the cached ones were updated */
   tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
   tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
 
diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c
index c2c556307d..71e1529216 100644
--- a/src/test/test_hs_config.c
+++ b/src/test/test_hs_config.c
@@ -12,6 +12,7 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
@@ -272,6 +273,7 @@ test_valid_service_v2(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. Basic client authorization. */
   {
@@ -314,7 +316,7 @@ test_valid_service_v2(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -392,6 +394,7 @@ test_valid_service_v3(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. */
   {
@@ -448,7 +451,7 @@ test_valid_service_v3(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -489,6 +492,111 @@ test_staging_service_v3(void *arg)
   hs_free_all();
 }
 
+static void
+test_dos_parameters(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  hs_init();
+
+  /* Valid configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_INFO);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    expect_log_msg_containing("Service INTRO2 DoS defenses rate set to: 42");
+    expect_log_msg_containing("Service INTRO2 DoS defenses burst set to: 87");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid rate. Value of 2^37. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 137438953472\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceEnableIntroDoSRatePerSec must "
+                              "be between 0 and 2147483647, "
+                              "not 137438953472");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid burst. Value of 2^38. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 274877906944\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceEnableIntroDoSBurstPerSec must "
+                              "be between 0 and 2147483647, "
+                              "not 274877906944");
+    teardown_capture_of_logs();
+  }
+
+  /* Burst is smaller than rate. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 27\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Hidden service DoS defenses burst (27) can "
+                              "not be smaller than the rate value (42).");
+    teardown_capture_of_logs();
+  }
+
+  /* Negative value. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec -1\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 42\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("HiddenServiceEnableIntroDoSRatePerSec must be "
+                              "between 0 and 2147483647, not -1");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  hs_free_all();
+}
+
 struct testcase_t hs_config_tests[] = {
   /* Invalid service not specific to any version. */
   { "invalid_service", test_invalid_service, TT_FORK,
@@ -512,6 +620,9 @@ struct testcase_t hs_config_tests[] = {
   { "staging_service_v3", test_staging_service_v3, TT_FORK,
     NULL, NULL },
 
+  /* Test HS DoS parameters. */
+  { "dos_parameters", test_dos_parameters, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c
index ba67712f1b..7cedc987bb 100644
--- a/src/test/test_hs_control.c
+++ b/src/test/test_hs_control.c
@@ -6,11 +6,13 @@
  * \brief Unit tests for hidden service control port event and command.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
 
 #include "core/or/or.h"
 #include "test/test.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_control.h"
@@ -105,8 +107,7 @@ test_hs_desc_event(void *arg)
   memset(&blinded_pk, 'B', sizeof(blinded_pk));
   memset(&hsdir_rs, 0, sizeof(hsdir_rs));
   memcpy(hsdir_rs.identity_digest, HSDIR_EXIST_ID, DIGEST_LEN);
-  ret = ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   memcpy(&ident.identity_pk, &identity_kp.pubkey,
          sizeof(ed25519_public_key_t));
   memcpy(&ident.blinded_pk, &blinded_pk, sizeof(blinded_pk));
diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c
index de584ed47a..6fe5573c0f 100644
--- a/src/test/test_hs_descriptor.c
+++ b/src/test/test_hs_descriptor.c
@@ -21,6 +21,7 @@
 #include "test/hs_test_helpers.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 DISABLE_GCC_WARNING(overlength-strings)
@@ -30,13 +31,6 @@ DISABLE_GCC_WARNING(overlength-strings)
 #include "test_hs_descriptor.inc"
 ENABLE_GCC_WARNING(overlength-strings)
 
-/* Mock function to fill all bytes with 1 */
-static void
-mock_crypto_strongest_rand(uint8_t *out, size_t out_len)
-{
-  memset(out, 1, out_len);
-}
-
 /* Test certificate encoding put in a descriptor. */
 static void
 test_cert_encoding(void *arg)
@@ -132,7 +126,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -149,7 +143,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -166,7 +160,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -179,115 +173,6 @@ test_descriptor_padding(void *arg)
 }
 
 static void
-test_link_specifier(void *arg)
-{
-  ssize_t ret;
-  hs_desc_link_specifier_t spec;
-  smartlist_t *link_specifiers = smartlist_new();
-  char buf[256];
-  char *b64 = NULL;
-  link_specifier_t *ls = NULL;
-
-  (void) arg;
-
-  /* Always this port. */
-  spec.u.ap.port = 42;
-  smartlist_add(link_specifiers, &spec);
-
-  /* Test IPv4 for starter. */
-  {
-    uint32_t ipv4;
-
-    spec.type = LS_IPV4;
-    ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4");
-    tt_int_op(ret, OP_EQ, AF_INET);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 8);
-    /* Should be 2 bytes for port and 4 bytes for IPv4. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 6);
-    ipv4 = link_specifier_get_un_ipv4_addr(ls);
-    tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), OP_EQ, ipv4);
-    tt_int_op(link_specifier_get_un_ipv4_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test IPv6. */
-  {
-    uint8_t ipv6[16];
-
-    spec.type = LS_IPV6;
-    ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]");
-    tt_int_op(ret, OP_EQ, AF_INET6);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 20);
-    /* Should be 2 bytes for port and 16 bytes for IPv6. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 18);
-    for (unsigned int i = 0; i < sizeof(ipv6); i++) {
-      ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i);
-    }
-    tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), OP_EQ, ipv6,
-              sizeof(ipv6));
-    tt_int_op(link_specifier_get_un_ipv6_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test legacy. */
-  {
-    uint8_t *id;
-
-    spec.type = LS_LEGACY_ID;
-    memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id));
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    /* 20 bytes digest + 1 byte type + 1 byte len. */
-    tt_int_op(ret, OP_EQ, 22);
-    tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN);
-    /* Digest length is 20 bytes. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN);
-    id = link_specifier_getarray_un_legacy_id(ls);
-    tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
- done:
-  link_specifier_free(ls);
-  tor_free(b64);
-  smartlist_free(link_specifiers);
-}
-
-static void
 test_encode_descriptor(void *arg)
 {
   int ret;
@@ -848,8 +733,7 @@ test_desc_signature(void *arg)
   ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data),
                               "Tor onion service descriptor sig v3", &kp);
   tt_int_op(ret, OP_EQ, 0);
-  ret = ed25519_signature_to_base64(sig_b64, &sig);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_signature_to_base64(sig_b64, &sig);
   /* Build the descriptor that should be valid. */
   tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64);
   ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc));
@@ -909,7 +793,7 @@ test_build_authorized_client(void *arg)
                 client_pubkey_b16,
                 strlen(client_pubkey_b16));
 
-  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+  testing_enable_prefilled_rng("\x01", 1);
 
   hs_desc_build_authorized_client(subcredential,
                                   &client_auth_pk, &auth_ephemeral_sk,
@@ -925,15 +809,13 @@ test_build_authorized_client(void *arg)
  done:
   tor_free(desc_client);
   tor_free(mem_op_hex_tmp);
-  UNMOCK(crypto_strongest_rand_);
+  testing_disable_prefilled_rng();
 }
 
 struct testcase_t hs_descriptor[] = {
   /* Encoding tests. */
   { "cert_encoding", test_cert_encoding, TT_FORK,
     NULL, NULL },
-  { "link_specifier", test_link_specifier, TT_FORK,
-    NULL, NULL },
   { "encode_descriptor", test_encode_descriptor, TT_FORK,
     NULL, NULL },
   { "descriptor_padding", test_descriptor_padding, TT_FORK,
diff --git a/src/test/test_hs_dos.c b/src/test/test_hs_dos.c
new file mode 100644
index 0000000000..f68639e24a
--- /dev/null
+++ b/src/test/test_hs_dos.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cell.c
+ * \brief Test hidden service cell functionality.
+ */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define HS_DOS_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+
+#include "feature/hs/hs_dos.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/nodelist/networkstatus.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_ns_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  current_ns_consensus->net_params = smartlist_new();
+  smartlist_add(current_ns_consensus->net_params,
+                (void *) "HiddenServiceEnableIntroDoSDefense=1");
+  hs_dos_consensus_has_changed(current_ns_consensus);
+}
+
+static void
+free_mock_consensus(void)
+{
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_can_send_intro2(void *arg)
+{
+  uint32_t now = (uint32_t) approx_time();
+  or_circuit_t *or_circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  hs_dos_init();
+
+  get_options_mutable()->ORPort_set = 1;
+  setup_mock_consensus();
+
+  or_circ =  or_circuit_new(1, NULL);
+
+  /* Make that circuit a service intro point. */
+  circuit_change_purpose(TO_CIRCUIT(or_circ), CIRCUIT_PURPOSE_INTRO_POINT);
+  hs_dos_setup_default_intro2_defenses(or_circ);
+  or_circ->introduce2_dos_defense_enabled = 1;
+
+  /* Brand new circuit, we should be able to send INTRODUCE2 cells. */
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+
+  /* Simulate that 10 cells have arrived in 1 second. There should be no
+   * refill since the bucket is already at maximum on the first cell. */
+  update_approx_time(++now);
+  for (int i = 0; i < 10; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 10);
+
+  /* Fully refill the bucket minus 1 cell. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Receive an INTRODUCE2 at each second. We should have the bucket full
+   * since at every second it gets refilled. */
+  for (int i = 0; i < 10; i++) {
+    update_approx_time(++now);
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  /* Last check if we can send the cell decrements the bucket so minus 1. */
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Manually reset bucket for next test. */
+  token_bucket_ctr_reset(&or_circ->introduce2_bucket, now);
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL));
+
+  /* Do a full burst in the current second which should empty the bucket and
+   * we shouldn't be allowed to send one more cell after that. We go minus 1
+   * cell else the very last check if we can send the INTRO2 cell returns
+   * false because the bucket goes down to 0. */
+  for (uint32_t i = 0; i < get_intro2_burst_consensus_param(NULL) - 1; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 1);
+  /* Get the last remaining cell, we shouldn't be allowed to send it. */
+  tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+
+  /* Make sure the next 100 cells aren't allowed and bucket stays at 0. */
+  for (int i = 0; i < 100; i++) {
+    tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+    tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+  }
+
+  /* One second has passed, we should have the rate minus 1 cell added. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_rate_consensus_param(NULL) - 1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(or_circ));
+
+  hs_free_all();
+  free_mock_consensus();
+}
+
+static void
+test_validate_dos_extension_params(void *arg)
+{
+  bool ret;
+
+  (void) arg;
+
+  /* Validate the default values. */
+  ret = cell_dos_extension_parameters_are_valid(
+                                      get_intro2_rate_consensus_param(NULL),
+                                      get_intro2_burst_consensus_param(NULL));
+  tt_assert(ret);
+
+  /* Valid custom rate/burst. */
+  ret = cell_dos_extension_parameters_are_valid(17, 42);
+  tt_assert(ret);
+  ret = cell_dos_extension_parameters_are_valid(INT32_MAX, INT32_MAX);
+  tt_assert(ret);
+
+  /* Invalid rate. */
+  ret = cell_dos_extension_parameters_are_valid(UINT64_MAX, 42);
+  tt_assert(!ret);
+
+  /* Invalid burst. */
+  ret = cell_dos_extension_parameters_are_valid(42, UINT64_MAX);
+  tt_assert(!ret);
+
+  /* Value of 0 is valid (but should disable defenses) */
+  ret = cell_dos_extension_parameters_are_valid(0, 0);
+  tt_assert(ret);
+
+  /* Can't have burst smaller than rate. */
+  ret = cell_dos_extension_parameters_are_valid(42, 40);
+  tt_assert(!ret);
+
+ done:
+  return;
+}
+
+struct testcase_t hs_dos_tests[] = {
+  { "can_send_intro2", test_can_send_intro2, TT_FORK,
+    NULL, NULL },
+  { "validate_dos_extension_params", test_validate_dos_extension_params,
+    TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c
index 558fc32c54..feb934d93c 100644
--- a/src/test/test_hs_intropoint.c
+++ b/src/test/test_hs_intropoint.c
@@ -16,6 +16,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 
 #include "core/or/or.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "ht.h"
@@ -25,6 +26,8 @@
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 
@@ -43,6 +46,9 @@ new_establish_intro_cell(const char *circ_nonce,
   uint8_t buf[RELAY_PAYLOAD_SIZE] = {0};
   trn_cell_establish_intro_t *cell = NULL;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Ensure that *cell_out is NULL such that we can use to check if we need to
    * free `cell` in case of an error. */
@@ -50,9 +56,9 @@ new_establish_intro_cell(const char *circ_nonce,
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
   tt_i64_op(cell_len, OP_GT, 0);
 
   cell_len = trn_cell_establish_intro_parse(&cell, buf, sizeof(buf));
@@ -73,12 +79,15 @@ new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out)
 {
   ssize_t cell_len = 0;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell_out);
   tt_i64_op(cell_len, OP_GT, 0);
 
  done:
@@ -118,6 +127,8 @@ helper_create_intro_circuit(void)
   or_circuit_t *circ = or_circuit_new(0, NULL);
   tt_assert(circ);
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+  token_bucket_ctr_init(&circ->introduce2_bucket, 100, 100,
+                        (uint32_t) approx_time());
  done:
   return circ;
 }
@@ -693,6 +704,17 @@ test_introduce1_suitable_circuit(void *arg)
     tt_int_op(ret, OP_EQ, 0);
   }
 
+  /* Single hop circuit should not be allowed. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+    circ->p_chan->is_client = 1;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    tor_free(circ->p_chan);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
  done:
   ;
 }
@@ -888,43 +910,213 @@ test_received_introduce1_handling(void *arg)
   UNMOCK(relay_send_command_from_edge_);
 }
 
+static void
+test_received_establish_intro_dos_ext(void *arg)
+{
+  int ret;
+  ssize_t cell_len = 0;
+  uint8_t cell[RELAY_PAYLOAD_SIZE] = {0};
+  char circ_nonce[DIGEST_LEN] = {0};
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  hs_circuitmap_init();
+
+  /* Setup. */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+  memset(&config, 0, sizeof(config));
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 13;
+  config.intro_dos_burst_per_sec = 42;
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+  /* The INTRO2 bucket should be 0 at this point. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 0);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 0);
+
+  /* Case 1: Build encoded cell. Usable DoS parameters. */
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be set to the burst value. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 42);
+  /* Validate the config of the intro2 bucket. */
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 13);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 42);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 1);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 2: Build encoded cell. Bad DoS parameters. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = UINT_MAX;
+  config.intro_dos_burst_per_sec = 13;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 3: Build encoded cell. Burst is smaller than rate. Not allowed. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 87;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 4: Build encoded cell. Rate is 0 but burst is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 0;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 5: Build encoded cell. Burst is 0 but rate is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 45;
+  config.intro_dos_burst_per_sec = 0;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  service_intro_point_free(ip);
+  hs_circuitmap_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+static void *
+hs_subsystem_setup_fn(const struct testcase_t *tc)
+{
+  (void) tc;
+
+  return NULL;
+}
+
+static int
+hs_subsystem_cleanup_fn(const struct testcase_t *tc, void *arg)
+{
+  (void) tc;
+  (void) arg;
+
+  return 1;
+}
+
+static struct testcase_setup_t test_setup = {
+  hs_subsystem_setup_fn, hs_subsystem_cleanup_fn
+};
+
 struct testcase_t hs_intropoint_tests[] = {
   { "intro_point_registration",
-    test_intro_point_registration, TT_FORK, NULL, NULL },
+    test_intro_point_registration, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype",
-    test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype2",
-    test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype2, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_purpose",
-    test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_purpose, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig",
-    test_establish_intro_wrong_sig, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig_len",
-    test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_auth_key_len",
-    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_mac",
-    test_establish_intro_wrong_mac, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_mac, TT_FORK, NULL, &test_setup},
 
   { "introduce1_suitable_circuit",
-    test_introduce1_suitable_circuit, TT_FORK, NULL, NULL },
+    test_introduce1_suitable_circuit, TT_FORK, NULL, &test_setup},
 
   { "introduce1_is_legacy",
-    test_introduce1_is_legacy, TT_FORK, NULL, NULL },
+    test_introduce1_is_legacy, TT_FORK, NULL, &test_setup},
 
   { "introduce1_validation",
-    test_introduce1_validation, TT_FORK, NULL, NULL },
+    test_introduce1_validation, TT_FORK, NULL, &test_setup},
 
   { "received_introduce1_handling",
-    test_received_introduce1_handling, TT_FORK, NULL, NULL },
+    test_received_introduce1_handling, TT_FORK, NULL, &test_setup},
+
+  { "received_establish_intro_dos_ext",
+    test_received_establish_intro_dos_ext, TT_FORK, NULL, &test_setup},
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c
index 32b08ecf37..66194cee3d 100644
--- a/src/test/test_hs_service.c
+++ b/src/test/test_hs_service.c
@@ -21,6 +21,7 @@
 #define STATEFILE_PRIVATE
 #define TOR_CHANNEL_INTERNAL_
 #define HS_CLIENT_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -60,6 +61,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -169,8 +171,7 @@ test_e2e_rend_circuit_setup(void *arg)
     tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
     tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
 
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
 
     TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
   }
@@ -193,12 +194,14 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
@@ -267,7 +270,7 @@ helper_clone_authorized_client(const hs_service_authorized_client_t *client)
 
 /* Helper: Return a newly allocated service object with the identity keypair
  * sets and the current descriptor. Then register it to the global map.
- * Caller should us hs_free_all() to free this service or remove it from the
+ * Caller should use hs_free_all() to free this service or remove it from the
  * global map before freeing. */
 static hs_service_t *
 helper_create_service(void)
@@ -289,6 +292,20 @@ helper_create_service(void)
   return service;
 }
 
+/* Helper: Deallocate a given service object, its child objects and
+ * remove it from onion service map.
+ * */
+static void
+helper_destroy_service(hs_service_t *service)
+{
+  if (!service)
+    return;
+
+  remove_service(get_hs_service_map(), service);
+
+  hs_service_free(service);
+}
+
 /* Helper: Return a newly allocated service object with clients. */
 static hs_service_t *
 helper_create_service_with_clients(int num_clients)
@@ -314,17 +331,18 @@ helper_create_service_with_clients(int num_clients)
 static hs_service_intro_point_t *
 helper_create_service_ip(void)
 {
-  hs_desc_link_specifier_t *ls;
-  hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+  link_specifier_t *ls;
+  hs_service_intro_point_t *ip = service_intro_point_new(NULL);
   tor_assert(ip);
   /* Add a first unused link specifier. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_IPV4;
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
   smartlist_add(ip->base.link_specifiers, ls);
   /* Add a second link specifier used by a test. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_LEGACY_ID;
-  memset(ls->u.legacy_id, 'A', sizeof(ls->u.legacy_id));
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memset(link_specifier_getarray_un_legacy_id(ls), 'A',
+         link_specifier_getlen_un_legacy_id(ls));
   smartlist_add(ip->base.link_specifiers, ls);
 
   return ip;
@@ -378,11 +396,11 @@ test_load_keys(void *arg)
   tt_assert(s);
 
   /* Ok we have the service object. Validate few things. */
-  tt_assert(!tor_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
+  tt_assert(!fast_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
   tt_int_op(hs_address_is_valid(s->onion_address), OP_EQ, 1);
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_sk.seckey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_sk.seckey,
                              ED25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_pk.pubkey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_pk.pubkey,
                              ED25519_PUBKEY_LEN));
   /* Check onion address from identity key. */
   hs_build_address(&s->keys.identity_pk, s->config.version, addr);
@@ -656,13 +674,15 @@ test_service_intro_point(void *arg)
 
   (void) arg;
 
+  update_approx_time(1481621834);
+
   /* Test simple creation of an object. */
   {
-    time_t now = time(NULL);
+    time_t now = approx_time();
     ip = helper_create_service_ip();
     tt_assert(ip);
     /* Make sure the authentication keypair is not zeroes. */
-    tt_int_op(tor_mem_is_zero((const char *) &ip->auth_key_kp,
+    tt_int_op(fast_mem_is_zero((const char *) &ip->auth_key_kp,
                               sizeof(ed25519_keypair_t)), OP_EQ, 0);
     /* The introduce2_max MUST be in that range. */
     tt_u64_op(ip->introduce2_max, OP_GE,
@@ -797,10 +817,11 @@ test_helper_functions(void *arg)
     const node_t *node = get_node_from_intro_point(ip);
     tt_ptr_op(node, OP_EQ, &mock_node);
     SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                            hs_desc_link_specifier_t *, ls) {
-      if (ls->type == LS_LEGACY_ID) {
+                            link_specifier_t *, ls) {
+      if (link_specifier_get_ls_type(ls) == LS_LEGACY_ID) {
         /* Change legacy id in link specifier which is not the mock node. */
-        memset(ls->u.legacy_id, 'B', sizeof(ls->u.legacy_id));
+        memset(link_specifier_getarray_un_legacy_id(ls), 'B',
+               link_specifier_getlen_un_legacy_id(ls));
       }
     } SMARTLIST_FOREACH_END(ls);
     node = get_node_from_intro_point(ip);
@@ -858,6 +879,10 @@ test_helper_functions(void *arg)
 
  done:
   /* This will free the service and all objects associated to it. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_service_free_all();
   UNMOCK(node_get_by_id);
 }
@@ -867,7 +892,7 @@ static void
 test_intro_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   origin_circuit_t *circ = NULL;
 
   (void) arg;
@@ -915,6 +940,10 @@ test_intro_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -929,7 +958,7 @@ test_intro_established(void *arg)
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -990,6 +1019,10 @@ test_intro_established(void *arg)
  done:
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1001,7 +1034,7 @@ test_rdv_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
 
   (void) arg;
 
@@ -1032,6 +1065,10 @@ test_rdv_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -1069,8 +1106,7 @@ test_closing_intro_circs(void *arg)
 
   /* Initialize intro circuit */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Register circuit in the circuitmap . */
   hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
                                                     &ip->auth_key_kp.pubkey);
@@ -1096,8 +1132,7 @@ test_closing_intro_circs(void *arg)
   /* Now pretend that a new intro point circ was launched and opened. Check
    * that the intro point will be established correctly. */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&intro_circ->hs_ident->intro_auth_pk,
                       &ip->auth_key_kp.pubkey);
   /* Register circuit in the circuitmap . */
@@ -1119,6 +1154,10 @@ test_closing_intro_circs(void *arg)
     circuit_free_(TO_CIRCUIT(intro_circ));
   }
   /* Frees the service object. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(assert_circuit_ok);
 }
@@ -1131,7 +1170,7 @@ test_introduce2(void *arg)
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -1141,7 +1180,7 @@ test_introduce2(void *arg)
   MOCK(get_or_state,
        get_or_state_replacement);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
   tt_assert(circ);
@@ -1198,6 +1237,10 @@ test_introduce2(void *arg)
   dummy_state = NULL;
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1221,6 +1264,7 @@ test_service_event(void *arg)
 
   /* Set a service for this circuit. */
   service = helper_create_service();
+  tt_assert(service);
   ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
                       &service->keys.identity_pk);
 
@@ -1256,8 +1300,14 @@ test_service_event(void *arg)
     run_housekeeping_event(now);
     tt_int_op(digest256map_size(service->desc_current->intro_points.map),
               OP_EQ, 1);
+    /* No removal if we have an established circuit after retries. */
+    ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    run_housekeeping_event(now);
+    tt_int_op(digest256map_size(service->desc_current->intro_points.map),
+              OP_EQ, 1);
     /* Remove the IP object at once for the next test. */
     ip->circuit_retries = MAX_INTRO_POINT_CIRCUIT_RETRIES + 1;
+    ip->circuit_established = 0;
     run_housekeeping_event(now);
     tt_int_op(digest256map_size(service->desc_current->intro_points.map),
               OP_EQ, 0);
@@ -1282,6 +1332,10 @@ test_service_event(void *arg)
  done:
   hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1292,12 +1346,12 @@ test_rotate_descriptors(void *arg)
 {
   int ret;
   time_t next_rotation_time, now;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_descriptor_t *desc_next;
 
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   hs_init();
   MOCK(get_or_state, get_or_state_replacement);
@@ -1384,6 +1438,10 @@ test_rotate_descriptors(void *arg)
   tt_assert(service->desc_next);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(get_or_state);
   UNMOCK(circuit_mark_for_close_);
@@ -1396,9 +1454,8 @@ static void
 test_build_update_descriptors(void *arg)
 {
   int ret;
-  time_t now = time(NULL);
   node_t *node;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip_cur, *ip_next;
   routerinfo_t ri;
 
@@ -1411,7 +1468,7 @@ test_build_update_descriptors(void *arg)
   MOCK(networkstatus_get_live_consensus,
        mock_networkstatus_get_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1422,7 +1479,8 @@ test_build_update_descriptors(void *arg)
   voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
-  now = mock_ns.valid_after+1;
+
+  time_t now = mock_ns.valid_after+1;
 
   /* Create a service without a current descriptor to trigger a build. */
   service = helper_create_service();
@@ -1546,9 +1604,9 @@ test_build_update_descriptors(void *arg)
   tt_int_op(smartlist_len(ip_cur->base.link_specifiers), OP_EQ, 3);
   /* Make sure we have a valid encryption keypair generated when we pick an
    * intro point in the update process. */
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
                              CURVE25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
                              CURVE25519_PUBKEY_LEN));
   tt_u64_op(ip_cur->time_to_expire, OP_GE, now +
             INTRO_POINT_LIFETIME_MIN_SECONDS);
@@ -1614,6 +1672,10 @@ test_build_update_descriptors(void *arg)
   tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   nodelist_free_all();
 }
@@ -1626,6 +1688,7 @@ test_build_descriptors(void *arg)
 {
   int ret;
   time_t now = time(NULL);
+  hs_service_t *last_service = NULL;
 
   (void) arg;
 
@@ -1636,7 +1699,7 @@ test_build_descriptors(void *arg)
   MOCK(networkstatus_get_live_consensus,
        mock_networkstatus_get_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1650,19 +1713,27 @@ test_build_descriptors(void *arg)
    * is disabled. */
   {
     hs_service_t *service = helper_create_service();
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
     build_all_descriptors(now);
+    tt_assert(service->desc_current);
+    tt_assert(service->desc_current->desc);
+
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(0);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1670,12 +1741,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is not a multiple of 16. */
   {
     hs_service_t *service = helper_create_service_with_clients(20);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1683,12 +1758,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Do not generate any fake desc client when the number of clients is
    * a multiple of 16 but not zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(32);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1696,9 +1775,13 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
  done:
+  helper_destroy_service(last_service);
   hs_free_all();
 }
 
@@ -1717,7 +1800,7 @@ test_upload_descriptors(void *arg)
   MOCK(networkstatus_get_live_consensus,
        mock_networkstatus_get_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1847,9 +1930,9 @@ test_rendezvous1_parsing(void *arg)
   }
 
   /* Send out the RENDEZVOUS1 and make sure that our mock func worked */
-  tt_assert(tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(fast_mem_is_zero(rend1_payload, 32));
   hs_circ_service_rp_has_opened(service, service_circ);
-  tt_assert(!tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(!fast_mem_is_zero(rend1_payload, 32));
   tt_int_op(rend1_payload_len, OP_EQ, HS_LEGACY_RENDEZVOUS_CELL_SIZE);
 
   /******************************/
diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c
index 4a6d90d97e..104e973b1f 100644
--- a/src/test/test_introduce.c
+++ b/src/test/test_introduce.c
@@ -383,8 +383,10 @@ make_intro_from_plaintext(
 
   /* Output the cell */
   *cell_out = cell;
+  cell = NULL;
 
  done:
+  tor_free(cell);
   return cell_len;
 }
 
@@ -535,4 +537,3 @@ struct testcase_t introduce_tests[] = {
   INTRODUCE_LEGACY(late_parse_v3),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh
index 3474210607..54abb4a2fa 100755
--- a/src/test/test_key_expiration.sh
+++ b/src/test/test_key_expiration.sh
@@ -6,14 +6,14 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -47,11 +47,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_key_expiration_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -60,10 +60,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh
index 7afff271cb..cbdfd1909c 100755
--- a/src/test/test_keygen.sh
+++ b/src/test/test_keygen.sh
@@ -6,14 +6,14 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -64,11 +64,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_keygen_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_keygen_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -77,10 +77,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
@@ -144,7 +144,9 @@ ME="${DATA_DIR}/case2a"
 SRC="${DATA_DIR}/orig"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: `cat ${ME}/stdout`" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Somehow succeeded when missing secret key, certs: $(cat "${ME}/stdout")"
+fi
 check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key"
 
 grep "We needed to load a secret key.*but couldn't find it" "${ME}/stdout" >/dev/null || die "Tor didn't declare that it was missing a secret key"
@@ -281,7 +283,9 @@ SRC="${DATA_DIR}/encrypted"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_secret_key_encrypted" "${ME}/keys/"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Tor started with encrypted secret key and no certs" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Tor started with encrypted secret key and no certs"
+fi
 check_no_file "${ME}/keys/ed25519_signing_cert"
 check_no_file "${ME}/keys/ed25519_signing_secret_key"
 
@@ -370,7 +374,9 @@ mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
 cp "${OTHER}/keys/ed25519_master_id_secret_key" "${ME}/keys/"
 
-${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout" && die "Successfully started with mismatched keys!?" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout"; then
+  die "Successfully started with mismatched keys!?"
+fi
 
 grep "public_key does not match.*secret_key" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a key mismatch"
 
@@ -386,7 +392,9 @@ ME="${DATA_DIR}/case11a"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout" && die "Successfully started with passphrase-fd but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout"; then
+  die "Successfully started with passphrase-fd but no keygen?"
+fi
 
 grep "passphrase-fd specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -402,7 +410,9 @@ ME="${DATA_DIR}/case11b"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout" && die "Successfully started with no-passphrase but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with no-passphrase but no keygen?"
+fi
 
 grep "no-passphrase specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -418,7 +428,9 @@ ME="${DATA_DIR}/case11C"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout" && die "Successfully started with newpass but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout"; then
+  die "Successfully started with newpass but no keygen?"
+fi
 
 grep "newpass specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -456,7 +468,9 @@ ME="${DATA_DIR}/case11E"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd?"
+fi
 
 grep "Invalid --passphrase-fd value" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -473,7 +487,9 @@ ME="${DATA_DIR}/case11F"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd combination?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd combination?"
+fi
 
 grep "no-passphrase specified with --passphrase-fd" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c
index 34f59f26cd..5e78e1ce4d 100644
--- a/src/test/test_link_handshake.c
+++ b/src/test/test_link_handshake.c
@@ -263,7 +263,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_assert(c1->handshake_state->authenticated_rsa);
     tt_assert(! c1->handshake_state->authenticated_ed25519);
   }
-  tt_assert(! tor_mem_is_zero(
+  tt_assert(! fast_mem_is_zero(
                 (char*)c1->handshake_state->authenticated_rsa_peer_id, 20));
 
   chan2 = tor_malloc_zero(sizeof(*chan2));
@@ -290,7 +290,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_ptr_op(c2->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
   }
   tt_assert(c2->handshake_state->certs->id_cert);
-  tt_assert(tor_mem_is_zero(
+  tt_assert(fast_mem_is_zero(
               (char*)c2->handshake_state->authenticated_rsa_peer_id, 20));
   /* no authentication has happened yet, since we haen't gotten an AUTH cell.
    */
@@ -948,7 +948,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_int_op(36, OP_EQ, cell1->payload_len);
   tt_int_op(36, OP_EQ, cell2->payload_len);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_int_op(0, OP_EQ, cell1->circ_id);
   tt_int_op(0, OP_EQ, cell2->circ_id);
   tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell1->command);
@@ -960,7 +960,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 4);
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 4);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_mem_op(cell1->payload, OP_NE, cell2->payload, 32);
 
  done:
diff --git a/src/test/test_logging.c b/src/test/test_logging.c
index 95a2fce757..203ce64e32 100644
--- a/src/test/test_logging.c
+++ b/src/test/test_logging.c
@@ -9,14 +9,13 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 
 static void
-dummy_cb_fn(int severity, uint32_t domain, const char *msg)
+dummy_cb_fn(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)severity; (void)domain; (void)msg;
 }
@@ -36,7 +35,7 @@ test_get_sigsafe_err_fds(void *arg)
 
   set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug);
   set_log_severity_config(LOG_WARN, LOG_ERR, &no_bug);
-  no_bug.masks[0] &= ~(LD_BUG|LD_GENERAL);
+  no_bug.masks[SEVERITY_MASK_IDX(LOG_ERR)] &= ~(LD_BUG|LD_GENERAL);
   set_log_severity_config(LOG_INFO, LOG_NOTICE, &no_bug2);
 
   /* Add some logs; make sure the output is as expected. */
@@ -117,22 +116,27 @@ test_sigsafe_err(void *arg)
   content = read_file_to_str(fn, 0, NULL);
 
   tt_ptr_op(content, OP_NE, NULL);
-  tor_split_lines(lines, content, (int)strlen(content));
+  smartlist_split_string(lines, content, "\n", 0, 0);
   tt_int_op(smartlist_len(lines), OP_GE, 5);
 
-  if (strstr(smartlist_get(lines, 0), "opening new log file"))
+  if (strstr(smartlist_get(lines, 0), "opening new log file")) {
+    void *item = smartlist_get(lines, 0);
     smartlist_del_keeporder(lines, 0);
+    tor_free(item);
+  }
+
   tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool"));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 1), "=============="));
-  tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal."));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 3), "=============="));
-  tt_str_op(smartlist_get(lines, 4), OP_EQ,
+  tt_str_op(smartlist_get(lines, 1), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 2), "=============="));
+  tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal."));
+  tt_str_op(smartlist_get(lines, 4), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 5), "=============="));
+  tt_str_op(smartlist_get(lines, 6), OP_EQ,
             "Testing any attempt to manually log from a signal.");
 
  done:
   tor_free(content);
+  SMARTLIST_FOREACH(lines, char *, x, tor_free(x));
   smartlist_free(lines);
 }
 
diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c
index 089ea812cf..ed6b8a9b66 100644
--- a/src/test/test_mainloop.c
+++ b/src/test/test_mainloop.c
@@ -6,11 +6,23 @@
  * \brief Tests for functions closely related to the Tor main loop
  */
 
+#define CONFIG_PRIVATE
+#define MAINLOOP_PRIVATE
+#define STATEFILE_PRIVATE
+
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
 #include "core/or/or.h"
+#include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+
+#include "feature/hs/hs_service.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/config/or_state_st.h"
 
 static const uint64_t BILLION = 1000000000;
 
@@ -131,12 +143,227 @@ test_mainloop_update_time_jumps(void *arg)
   monotime_disable_test_mocking();
 }
 
+static int schedule_rescan_called = 0;
+static void
+mock_schedule_rescan_periodic_events(void)
+{
+  ++schedule_rescan_called;
+}
+
+static void
+test_mainloop_user_activity(void *arg)
+{
+  (void)arg;
+  const time_t start = 1542658829;
+  update_approx_time(start);
+
+  MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events);
+
+  reset_user_activity(start);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  set_network_participation(false);
+
+  // reset can move backwards and forwards, but does not change network
+  // participation.
+  reset_user_activity(start-10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10);
+  reset_user_activity(start+10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  tt_int_op(schedule_rescan_called, OP_EQ, 0);
+  tt_int_op(false, OP_EQ, is_participating_on_network());
+
+  // "note" can only move forward.  Calling it from a non-participating
+  // state makes us rescan the periodic callbacks and set participation.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // Calling it again will move us forward, but not call rescan again.
+  note_user_activity(start+25);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We won't move backwards.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We _will_ adjust if the clock jumps though.
+  netstatus_note_clock_jumped(500);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525);
+
+  netstatus_note_clock_jumped(-400);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125);
+
+ done:
+  UNMOCK(schedule_rescan_periodic_events);
+}
+
+static unsigned int
+mock_get_num_services(void)
+{
+  return 1;
+}
+
+static connection_t *
+mock_connection_gbtu(int type)
+{
+  (void) type;
+  return (void *)"hello fellow connections";
+}
+
+static void
+test_mainloop_check_participation(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  const time_t start = 1542658829;
+  const time_t ONE_DAY = 24*60*60;
+
+  // Suppose we've been idle for a day or two
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  check_network_participation_callback(start, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+  // suppose we've been idle for 2 days... but we are a server.
+  reset_user_activity(start - 2*ONE_DAY);
+  options->ORPort_set = 1;
+  set_network_participation(true);
+  check_network_participation_callback(start+2, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2);
+  options->ORPort_set = 0;
+
+  // idle for 2 days, but we have a hidden service.
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  MOCK(hs_service_get_num_services, mock_get_num_services);
+  check_network_participation_callback(start+3, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3);
+  UNMOCK(hs_service_get_num_services);
+
+  // idle for 2 days but we have at least one user connection
+  MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu);
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+  check_network_participation_callback(start+10, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  // as above, but DormantTimeoutDisabledByIdleStreams is not set
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 0;
+  check_network_participation_callback(start+13, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+  UNMOCK(connection_get_by_type_nonlinked);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+
+  // idle for 2 days but DormantClientTimeout is 3 days
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantClientTimeout = ONE_DAY * 3;
+  check_network_participation_callback(start+30, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+ done:
+  or_options_free(options);
+  UNMOCK(hs_service_get_num_services);
+  UNMOCK(connection_get_by_type_nonlinked);
+}
+
+static void
+test_mainloop_dormant_load_state(void *arg)
+{
+  (void)arg;
+  or_state_t *state = or_state_new();
+  const time_t start = 1543956575;
+
+  reset_user_activity(0);
+  set_network_participation(false);
+
+  // When we construct a new state, it starts out in "auto" mode.
+  tt_int_op(state->Dormant, OP_EQ, -1);
+
+  // Initializing from "auto" makes us start out (by default) non-Dormant,
+  // with activity right now.
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  // Initializing from dormant clears the last user activity time, and
+  // makes us dormant.
+  state->Dormant = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(! is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, 0);
+
+  // Initializing from non-dormant sets the last user activity time, and
+  // makes us non-dormant.
+  state->Dormant = 0;
+  state->MinutesSinceUserActivity = 123;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60);
+
+  // If we would start dormant, but DormantCanceledByStartup is set, then
+  // we start up non-dormant.
+  state->Dormant = 1;
+  get_options_mutable()->DormantCanceledByStartup = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+ done:
+  or_state_free(state);
+}
+
+static void
+test_mainloop_dormant_save_state(void *arg)
+{
+  (void)arg;
+  or_state_t *state = or_state_new();
+  const time_t start = 1543956575;
+
+  // Can we save a non-dormant state correctly?
+  reset_user_activity(start - 1000);
+  set_network_participation(true);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 0);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60);
+
+  // Can we save a dormant state correctly?
+  set_network_participation(false);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 1);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0);
+
+ done:
+  or_state_free(state);
+}
+
 #define MAINLOOP_TEST(name) \
   { #name, test_mainloop_## name , TT_FORK, NULL, NULL }
 
 struct testcase_t mainloop_tests[] = {
   MAINLOOP_TEST(update_time_normal),
   MAINLOOP_TEST(update_time_jumps),
+  MAINLOOP_TEST(user_activity),
+  MAINLOOP_TEST(check_participation),
+  MAINLOOP_TEST(dormant_load_state),
+  MAINLOOP_TEST(dormant_save_state),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c
index 4c4317d81a..804e6c546a 100644
--- a/src/test/test_microdesc.c
+++ b/src/test/test_microdesc.c
@@ -11,6 +11,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
 
@@ -20,6 +21,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -70,6 +72,7 @@ test_md_cache(void *data)
   const char *test_md3_noannotation = strchr(test_md3, '\n')+1;
   time_t time1, time2, time3;
   char *fn = NULL, *s = NULL;
+  char *encoded_family = NULL;
   (void)data;
 
   options = get_options_mutable();
@@ -172,8 +175,9 @@ test_md_cache(void *data)
 
   tt_ptr_op(md1->family, OP_EQ, NULL);
   tt_ptr_op(md3->family, OP_NE, NULL);
-  tt_int_op(smartlist_len(md3->family), OP_EQ, 3);
-  tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX");
+
+  encoded_family = nodefamily_format(md3->family);
+  tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez");
 
   /* Now rebuild the cache! */
   tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0);
@@ -254,6 +258,7 @@ test_md_cache(void *data)
   smartlist_free(wanted);
   tor_free(s);
   tor_free(fn);
+  tor_free(encoded_family);
 }
 
 static const char truncated_md[] =
@@ -417,6 +422,28 @@ static const char test_md2_21[] =
   "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
   "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
 
+static const char test_md2_withfamily_28[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family OtherNode !Strange\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
+static const char test_md2_withfamily_29[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
 static void
 test_md_generate(void *arg)
 {
@@ -447,6 +474,17 @@ test_md_generate(void *arg)
   tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey,
                               &ri->cache_info.signing_key_cert->signing_key));
 
+  // Try family encoding.
+  microdesc_free(md);
+  ri->declared_family = smartlist_new();
+  smartlist_add_strdup(ri->declared_family, "OtherNode !Strange");
+  md = dirvote_create_microdescriptor(ri, 28);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_28);
+
+  microdesc_free(md);
+  md = dirvote_create_microdescriptor(ri, 29);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_29);
+
  done:
   microdesc_free(md);
   routerinfo_free(ri);
@@ -611,6 +649,41 @@ static const char MD_PARSE_TEST_DATA[] =
   "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
   "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   "p6 allow 80\n"
+  /* Good 11: Normal, non-exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 12: Normal, exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 13: Normal, exit relay with only ipv6 exit policy */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p6 accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   ;
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
 ENABLE_GCC_WARNING(overlength-strings)
@@ -628,7 +701,7 @@ test_md_parse(void *arg)
   smartlist_t *mds = microdescs_parse_from_string(MD_PARSE_TEST_DATA,
                                                   NULL, 1, SAVED_NOWHERE,
                                                   invalid);
-  tt_int_op(smartlist_len(mds), OP_EQ, 11);
+  tt_int_op(smartlist_len(mds), OP_EQ, 14);
   tt_int_op(smartlist_len(invalid), OP_EQ, 4);
 
   test_memeq_hex(smartlist_get(invalid,0),
@@ -675,6 +748,21 @@ test_md_parse(void *arg)
   tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
   tt_int_op(md->ipv6_orport, OP_EQ, 9090);
 
+  md = smartlist_get(mds, 11);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 1);
+
+  md = smartlist_get(mds, 12);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
+  md = smartlist_get(mds, 13);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
  done:
   SMARTLIST_FOREACH(mds, microdesc_t *, mdsc, microdesc_free(mdsc));
   smartlist_free(mds);
@@ -683,6 +771,80 @@ test_md_parse(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+static void
+test_md_parse_id_ed25519(void *arg)
+{
+  (void)arg;
+
+  /* A correct MD with an ed25519 ID ... and an unspecified ID type,
+   * which is permitted. */
+  const char GOOD_MD[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id wumpus dodecahedron\n";
+
+  smartlist_t *mds = NULL;
+  const microdesc_t *md;
+
+  mds = microdescs_parse_from_string(GOOD_MD,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 1);
+  md = smartlist_get(mds, 0);
+  tt_mem_op(md->ed25519_identity_pkey, OP_EQ,
+            "This isn't actually a public key", ED25519_PUBKEY_LEN);
+  SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key appears twice. */
+  const char DUPLICATE_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n";
+
+  setup_capture_of_logs(LOG_WARN);
+  mds = microdescs_parse_from_string(DUPLICATE_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Extra ed25519 key");
+  mock_clean_saved_logs();
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key is invalid. */
+  const char BOGUS_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyZZZZZZZZZZZ\n";
+
+  mds = microdescs_parse_from_string(BOGUS_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Bogus ed25519 key");
+
+ done:
+  if (mds) {
+    SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+    smartlist_free(mds);
+  }
+  teardown_capture_of_logs();
+}
+
 static int mock_rgsbd_called = 0;
 static routerstatus_t *mock_rgsbd_val_a = NULL;
 static routerstatus_t *mock_rgsbd_val_b = NULL;
@@ -816,6 +978,7 @@ struct testcase_t microdesc_tests[] = {
   { "broken_cache", test_md_cache_broken, TT_FORK, NULL, NULL },
   { "generate", test_md_generate, 0, NULL, NULL },
   { "parse", test_md_parse, 0, NULL, NULL },
+  { "parse_id_ed25519", test_md_parse_id_ed25519, 0, NULL, NULL },
   { "reject_cache", test_md_reject_cache, TT_FORK, NULL, NULL },
   { "corrupt_desc", test_md_corrupt_desc, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
diff --git a/src/test/test_namemap.c b/src/test/test_namemap.c
new file mode 100644
index 0000000000..df77d4e2de
--- /dev/null
+++ b/src/test/test_namemap.c
@@ -0,0 +1,174 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/malloc/malloc.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static void
+test_namemap_empty(void *arg)
+{
+  (void)arg;
+
+  namemap_t m;
+  namemap_init(&m);
+  namemap_t m2 = NAMEMAP_INIT();
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m2));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+ done:
+  namemap_clear(&m);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_toolong(void *arg)
+{
+  (void)arg;
+  namemap_t m;
+  char *ok = NULL;
+  char *toolong = NULL;
+  namemap_init(&m);
+
+  ok = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+1);
+  memset(ok, 'x', MAX_NAMEMAP_NAME_LEN);
+
+  toolong = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+2);
+  memset(toolong, 'x', MAX_NAMEMAP_NAME_LEN+1);
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+  unsigned u1 = namemap_get_or_create_id(&m, toolong);
+  unsigned u2 = namemap_get_or_create_id(&m, ok);
+  tt_uint_op(u1, OP_EQ, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+
+  tt_str_op(ok, OP_EQ, namemap_get_name(&m, u2));
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m, u1));
+
+ done:
+  tor_free(ok);
+  tor_free(toolong);
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_blackbox(void *arg)
+{
+  (void)arg;
+
+  namemap_t m1, m2;
+  namemap_init(&m1);
+  namemap_init(&m2);
+
+  unsigned u1 = namemap_get_or_create_id(&m1, "hello");
+  unsigned u2 = namemap_get_or_create_id(&m1, "world");
+  tt_uint_op(u1, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u1, OP_NE, u2);
+
+  tt_uint_op(u1, OP_EQ, namemap_get_id(&m1, "hello"));
+  tt_uint_op(u1, OP_EQ, namemap_get_or_create_id(&m1, "hello"));
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m1, "world"));
+  tt_uint_op(u2, OP_EQ, namemap_get_or_create_id(&m1, "world"));
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "HELLO"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+
+  unsigned u3 = namemap_get_or_create_id(&m2, "hola");
+  tt_uint_op(u3, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_or_create_id(&m2, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  unsigned int u4 = namemap_get_or_create_id(&m1, "hola");
+  tt_uint_op(u4, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u4, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  tt_str_op("hello", OP_EQ, namemap_get_name(&m1, u1));
+  tt_str_op("world", OP_EQ, namemap_get_name(&m1, u2));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m2, u3));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m1, u4));
+
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m2, u3 + 10));
+
+ done:
+  namemap_clear(&m1);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_internals(void *arg)
+{
+  (void)arg;
+  // This test actually assumes know something about the identity layout.
+  namemap_t m;
+  namemap_init(&m);
+
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+
+ done:
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_fmt(void *arg)
+{
+  (void)arg;
+  namemap_t m = NAMEMAP_INIT();
+
+  unsigned a = namemap_get_or_create_id(&m, "greetings");
+  unsigned b = namemap_get_or_create_id(&m, "earthlings");
+
+  tt_str_op(namemap_fmt_name(&m, a), OP_EQ, "greetings");
+  tt_str_op(namemap_fmt_name(&m, b), OP_EQ, "earthlings");
+  tt_int_op(a, OP_NE, 100);
+  tt_int_op(b, OP_NE, 100);
+  tt_str_op(namemap_fmt_name(&m, 100), OP_EQ, "{100}");
+
+ done:
+  namemap_clear(&m);
+}
+
+#define T(name) \
+  { #name, test_namemap_ ## name , 0, NULL, NULL }
+
+struct testcase_t namemap_tests[] = {
+  T(empty),
+  T(toolong),
+  T(blackbox),
+  T(internals),
+  T(fmt),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c
new file mode 100644
index 0000000000..27d276d42f
--- /dev/null
+++ b/src/test/test_netinfo.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "trunnel/netinfo.h"
+#include "test/test.h"
+
+static void
+test_netinfo_unsupported_addr(void *arg)
+{
+  const uint8_t wire_data[] =
+    { // TIME
+      0x00, 0x00, 0x00, 0x01,
+      // OTHERADDR
+      0x04, // ATYPE
+      0x04, // ALEN
+      0x08, 0x08, 0x08, 0x08, // AVAL
+      0x01, // NMYADDR
+      0x03, // ATYPE (unsupported)
+      0x05, // ALEN
+      'a', 'd', 'r', 'r', '!' // AVAL (unsupported)
+    };
+
+  (void)arg;
+
+  netinfo_cell_t *parsed_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data,
+                  sizeof(wire_data));
+
+  tt_assert(parsed == sizeof(wire_data));
+
+  netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0);
+  tt_assert(addr);
+
+  tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr));
+  tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr));
+
+ done:
+  netinfo_cell_free(parsed_cell);
+}
+
+struct testcase_t netinfo_tests[] = {
+  { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c
index 53eb0413e5..b8ca56bb81 100644
--- a/src/test/test_nodelist.c
+++ b/src/test/test_nodelist.c
@@ -6,15 +6,21 @@
  * \brief Unit tests for nodelist related functions.
  **/
 
+#define NODELIST_PRIVATE
+
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/torcert.h"
 
+#include "core/or/extend_info_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodefamily_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
@@ -72,7 +78,7 @@ test_nodelist_node_get_verbose_nickname_not_named(void *arg)
 }
 
 /** A node should be considered a directory server if it has an open dirport
- * of it accepts tunnelled directory requests.
+ * or it accepts tunnelled directory requests.
  */
 static void
 test_nodelist_node_is_dir(void *arg)
@@ -231,6 +237,1015 @@ test_nodelist_ed_id(void *arg)
 #undef N_NODES
 }
 
+static void
+test_nodelist_nodefamily(void *arg)
+{
+  (void)arg;
+  /* hex ID digests */
+  const char h1[] = "5B435D6869206861206C65207363617270652070";
+  const char h2[] = "75C3B220616E6461726520696E206769726F2061";
+  const char h3[] = "2074726F766172206461206D616E67696172652C";
+  const char h4[] = "206D656E747265206E6F6E2076616C65206C2769";
+  const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669";
+
+  /* binary ID digests */
+  uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN],
+    d5[DIGEST_LEN];
+  base16_decode((char*)d1, sizeof(d1), h1, strlen(h1));
+  base16_decode((char*)d2, sizeof(d2), h2, strlen(h2));
+  base16_decode((char*)d3, sizeof(d3), h3, strlen(h3));
+  base16_decode((char*)d4, sizeof(d4), h4, strlen(h4));
+  base16_decode((char*)d5, sizeof(d5), h5, strlen(h5));
+
+  char *enc=NULL, *enc2=NULL;
+
+  nodefamily_t *nf1 = NULL;
+  nodefamily_t *nf2 = NULL;
+  nodefamily_t *nf3 = NULL;
+
+  enc = nodefamily_format(NULL);
+  tt_str_op(enc, OP_EQ, "");
+  tor_free(enc);
+
+  /* Make sure that sorting and de-duplication work. */
+  tor_asprintf(&enc, "$%s hello", h1);
+  nf1 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf1);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "hello hello $%s hello", h1);
+  nf2 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf2);
+  tt_ptr_op(nf1, OP_EQ, nf2);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "%s $%s hello", h1, h1);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(! nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_nickname(nf1, "hello"));
+  tt_assert(nodefamily_contains_nickname(nf1, "HELLO"));
+  tt_assert(! nodefamily_contains_nickname(nf1, "goodbye"));
+
+  tt_int_op(nf1->refcnt, OP_EQ, 3);
+  nodefamily_free(nf3);
+  tt_int_op(nf1->refcnt, OP_EQ, 2);
+
+  /* Try parsing with a provided self RSA digest. */
+  nf3 = nodefamily_parse("hello ", d1, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+
+  /* Do we get the expected result when we re-encode? */
+  tor_asprintf(&enc, "$%s hello", h1);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc2, OP_EQ, enc);
+  tor_free(enc2);
+  tor_free(enc);
+
+  /* Make sure that we get a different result if we give a different digest. */
+  nodefamily_free(nf3);
+  tor_asprintf(&enc, "hello $%s hello", h3);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_NE, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf3, d3));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d2));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d1));
+  tt_assert(nodefamily_contains_nickname(nf3, "hello"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "goodbye"));
+
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+
+  /* Try one with several digests, all with nicknames appended, in different
+     formats. */
+  tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4);
+  nf1 = nodefamily_parse(enc, d5, 0);
+  tt_assert(nf1);
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d3));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d5));
+  /* Nicknames aren't preserved when ids are present, since node naming is
+   * deprecated */
+  tt_assert(! nodefamily_contains_nickname(nf3, "res"));
+  tor_free(enc);
+  tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc, OP_EQ, enc2);
+  tor_free(enc);
+  tor_free(enc2);
+
+  /* Try ones where we parse the empty string. */
+  nf2 = nodefamily_parse("", NULL, 0);
+  nf3 = nodefamily_parse("", d4, 0);
+  tt_assert(nf2);
+  tt_assert(nf3);
+  tt_ptr_op(nf2, OP_NE, nf3);
+
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf3, d4));
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d5));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d5));
+  tt_assert(! nodefamily_contains_nickname(nf2, "fred"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "bosco"));
+
+  /* The NULL family should contain nothing. */
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d4));
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d5));
+
+ done:
+  tor_free(enc);
+  tor_free(enc2);
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+  nodefamily_free_all();
+}
+
+static void
+test_nodelist_nodefamily_parse_err(void *arg)
+{
+  (void)arg;
+  nodefamily_t *nf1 = NULL;
+  char *enc = NULL;
+  const char *semibogus =
+    "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname
+    "$jkASDFLkjsadfjhkl " // not hex
+    "$7468696e67732d696e2d7468656d73656c766573 " // ok
+    "reticulatogranulate "// ok
+    "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex
+    "$616273656e746d696e6465646e6573736573" // too short for hex
+    ;
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // We only get two items when we parse this.
+  for (int reject = 0; reject <= 1; ++reject) {
+    for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) {
+      unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0;
+      flags |= reject ? NF_REJECT_MALFORMED : 0;
+      nf1 = nodefamily_parse(semibogus, NULL, flags);
+      if (reject) {
+        tt_assert(nf1 == NULL);
+      } else {
+        tt_assert(nf1);
+        enc = nodefamily_format(nf1);
+        tt_str_op(enc, OP_EQ,
+                  "$7468696E67732D696E2D7468656D73656C766573 "
+                  "reticulatogranulate");
+        tor_free(enc);
+      }
+
+      if (log_at_warn) {
+        expect_log_msg_containing("$616273656e746d696e6465646e6573736573");
+        expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf");
+      } else {
+        tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0);
+      }
+      mock_clean_saved_logs();
+    }
+  }
+
+ done:
+  tor_free(enc);
+  nodefamily_free(nf1);
+  teardown_capture_of_logs();
+}
+
+static const node_t *
+mock_node_get_by_id(const char *id)
+{
+  if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  memcpy(fake_node->identity, id, DIGEST_LEN);
+  return fake_node;
+}
+
+static const node_t *
+mock_node_get_by_nickname(const char *nn, unsigned flags)
+{
+  (void)flags;
+  if (!strcmp(nn, "nonesuch"))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  strlcpy(fake_node->identity, nn, DIGEST_LEN);
+  return fake_node;
+}
+
+static void
+test_nodelist_nodefamily_lookup(void *arg)
+{
+  (void)arg;
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  smartlist_t *sl = smartlist_new();
+  nodefamily_t *nf1 = NULL;
+  char *mem_op_hex_tmp = NULL;
+
+  // 'null' is allowed.
+  nodefamily_add_nodes_to_smartlist(NULL, sl);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
+
+  // Try a real family
+  nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE "
+                         "$2121212121212121212121212121212121212121 "
+                         "$3333333333333333333333333333333333333333 "
+                         "erewhon nonesuch", NULL, 0);
+  tt_assert(nf1);
+  nodefamily_add_nodes_to_smartlist(nf1, sl);
+  // There were 5 elements; 2 were dropped because the mocked lookup failed.
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+
+  const node_t *n = smartlist_get(sl, 0);
+  test_memeq_hex(n->identity, "3333333333333333333333333333333333333333");
+  n = smartlist_get(sl, 1);
+  test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE");
+  n = smartlist_get(sl, 2);
+  tt_str_op(n->identity, OP_EQ, "erewhon");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node));
+  smartlist_free(sl);
+  nodefamily_free(nf1);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_nodelist_nickname_matches(void *arg)
+{
+  (void)arg;
+  node_t mock_node;
+  routerstatus_t mock_rs;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+
+  strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname));
+  mock_node.rs = &mock_rs;
+  memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN);
+
+#define match(x) tt_assert(node_nickname_matches(&mock_node, (x)))
+#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x)))
+
+  match("evilgeniuses");
+  match("EvilGeniuses");
+  match("EvilGeniuses");
+  match("2e666f7261626574746572746f6d6f72726f772e");
+  match("2E666F7261626574746572746F6D6F72726F772E");
+  match("$2e666f7261626574746572746f6d6f72726f772e");
+  match("$2E666F7261626574746572746F6D6F72726F772E");
+  match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses");
+  match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES");
+
+  no_match("evilgenius");
+  no_match("evilgeniuseses");
+  no_match("evil.genius");
+  no_match("$2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E~fred");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES");
+ done:
+  ;
+}
+
+static void
+test_nodelist_node_nodefamily(void *arg)
+{
+  (void)arg;
+  node_t mock_node1;
+  routerstatus_t mock_rs;
+  microdesc_t mock_md;
+
+  node_t mock_node2;
+  routerinfo_t mock_ri;
+
+  smartlist_t *nodes=smartlist_new();
+
+  memset(&mock_node1, 0, sizeof(mock_node1));
+  memset(&mock_node2, 0, sizeof(mock_node2));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+  memset(&mock_md, 0, sizeof(mock_md));
+  memset(&mock_ri, 0, sizeof(mock_ri));
+
+  mock_node1.rs = &mock_rs;
+  mock_node1.md = &mock_md;
+
+  mock_node2.ri = &mock_ri;
+
+  strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname));
+  mock_ri.nickname = tor_strdup("nodetwo");
+
+  memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN);
+  memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN);
+
+  // empty families.
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain nodes, but not these nodes
+  mock_ri.declared_family = smartlist_new();
+  smartlist_add(mock_ri.declared_family, (char*)"NodeThree");
+  mock_md.family = nodefamily_parse("NodeFour", NULL, 0);
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain one another.
+  smartlist_add(mock_ri.declared_family, (char*)
+                "4e6f64654f6e654e6f6465314e6f64654f6e6531");
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  nodefamily_free(mock_md.family);
+  mock_md.family = nodefamily_parse(
+            "NodeFour "
+            "5365636f6e644e6f64655765277265546573746e", NULL, 0);
+  tt_assert(node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  // Try looking up families now.
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  node_lookup_declared_family(nodes, &mock_node1);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  const node_t *n = smartlist_get(nodes, 0);
+  tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN);
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "nodefour");
+
+  // free, try the other one.
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_clear(nodes);
+
+  node_lookup_declared_family(nodes, &mock_node2);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  n = smartlist_get(nodes, 0);
+  // This gets a truncated hex hex ID since it was looked up by name
+  tt_str_op(n->identity, OP_EQ, "NodeThree");
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  smartlist_free(mock_ri.declared_family);
+  nodefamily_free(mock_md.family);
+  tor_free(mock_ri.nickname);
+  // use tor_free, these aren't real nodes
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_free(nodes);
+}
+
+static void
+test_nodelist_nodefamily_canonicalize(void *arg)
+{
+  (void)arg;
+  char *c = NULL;
+
+  c = nodefamily_canonicalize("", NULL, 0);
+  tt_str_op(c, OP_EQ, "");
+  tor_free(c);
+
+  uint8_t own_id[20];
+  memset(own_id, 0, sizeof(own_id));
+  c = nodefamily_canonicalize(
+           "alice BOB caroL %potrzebie !!!@#@# "
+           "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred "
+           "ffffffffffffffffffffffffffffffffffffffff "
+           "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0);
+  tt_str_op(c, OP_EQ,
+           "!!!@#@# "
+           "$0000000000000000000000000000000000000000 "
+           "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB "
+           "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC "
+           "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF "
+           "%potrzebie "
+           "alice bob carol");
+
+ done:
+  tor_free(c);
+}
+
+/** format_node_description() should return
+ * "Fingerprint~Nickname at IPv4 and [IPv6]".
+ * The nickname and addresses are optional.
+ */
+static void
+test_nodelist_format_node_description(void *arg)
+{
+  char mock_digest[DIGEST_LEN];
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_null_ip;
+  tor_addr_t mock_ipv4;
+  tor_addr_t mock_ipv6;
+
+  char ndesc[NODE_DESC_BUF_LEN];
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(ndesc, 0, sizeof(ndesc));
+  memset(mock_digest, 0, sizeof(mock_digest));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_null_ip, 0, sizeof(mock_null_ip));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ipv6, 0, sizeof(mock_ipv6));
+
+  /* Set variables */
+  memcpy(mock_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  tor_addr_parse(&mock_ipv6, "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Test function with variables */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  /* format node description should use ~ because named is deprecated */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~""TestOR7890123456789");
+
+  /* Try a null IP address, rather than NULL */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_null_ip,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789");
+
+  /* Try some real IP addresses */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               &mock_ipv4,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA at 111.222.233.244");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               tor_addr_to_ipv4h(&mock_ipv4));
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = format_node_description(NULL, NULL, NULL, NULL, 0);
+  tt_str_op(rv, OP_EQ, "<NULL BUFFER>");
+
+  rv = format_node_description(ndesc, NULL, NULL, NULL, 0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(rv, OP_EQ, "<NULL ID DIGEST>");
+
+ done:
+  return;
+}
+
+/** router_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_router_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+  routerinfo_t mock_ri_ipv4;
+  routerinfo_t mock_ri_ipv6;
+  routerinfo_t mock_ri_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_ipv4, 0, sizeof(mock_ri_ipv4));
+  memset(&mock_ri_ipv6, 0, sizeof(mock_ri_ipv6));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerinfos.
+   * mock_nickname is referenced from all 3 routerinfos.
+   * That's ok, all their memory is static. */
+  memcpy(&mock_ri_ipv4, &mock_ri_dual, sizeof(mock_ri_ipv4));
+  memcpy(&mock_ri_ipv6, &mock_ri_dual, sizeof(mock_ri_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_ri_ipv4.ipv6_addr, 0, sizeof(mock_ri_ipv4.ipv6_addr));
+  mock_ri_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = router_describe(&mock_ri_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = router_describe(&mock_ri_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = router_describe(&mock_ri_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = router_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerinfos */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.ri = &mock_ri_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.ri = &mock_ri_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = &mock_ri_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** node_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only and routerstatus-only node_describe() tests are in
+ * test_nodelist_router_describe() and test_nodelist_routerstatus_describe(),
+ * so we can re-use their mocked variables.
+ */
+static void
+test_nodelist_node_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Routerinfos */
+  routerinfo_t mock_ri_dual;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Routerstatuses */
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_dual;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB"
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "Bbb",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "2.2.2.2");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[bbbb::bbbb]");
+
+  /* Create and modify the other routerstatus. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  /* Clear the unnecessary IPv6 address */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+
+  /* Microdescs */
+  microdesc_t mock_md_null;
+  microdesc_t mock_md_ipv6;
+
+  /* Clear variables */
+  memset(&mock_md_null, 0, sizeof(mock_md_null));
+  memset(&mock_md_ipv6, 0, sizeof(mock_md_ipv6));
+
+  /* Set up the microdesc */
+  tor_addr_parse(&mock_md_ipv6.ipv6_addr,
+                 "[eeee::6000:6000]");
+
+  /* Set up the node */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC"
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC",
+         sizeof(mock_node.identity));
+
+  /* Test that the routerinfo and routerstatus work separately, but the
+   * identity comes from the node */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerstatus overrides the routerinfo */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the microdesc IPv6 is used if the routerinfo doesn't have IPv6
+   */
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerinfo doesn't change the results above
+   */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* test NULL handling */
+  rv = node_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  mock_node.ri = NULL;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "<null rs and ri>");
+
+ done:
+  return;
+}
+
+/** routerstatus_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ *
+ * The routerstatus-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_routerstatus_describe(void *arg)
+{
+  tor_addr_t mock_ipv4;
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_ipv6;
+  routerstatus_t mock_rs_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_ipv6, 0, sizeof(mock_rs_ipv6));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "TestOR7890123456789",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerstatuses. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  memcpy(&mock_rs_ipv6, &mock_rs_dual, sizeof(mock_rs_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+  mock_rs_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = routerstatus_describe(&mock_rs_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = routerstatus_describe(&mock_rs_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = routerstatus_describe(&mock_rs_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = routerstatus_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerstatuses */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.rs = &mock_rs_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.rs = &mock_rs_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.rs = &mock_rs_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** extend_info_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ */
+static void
+test_nodelist_extend_info_describe(void *arg)
+{
+  extend_info_t mock_ei_ipv4;
+  extend_info_t mock_ei_ipv6;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ei_ipv4, 0, sizeof(mock_ei_ipv4));
+  memset(&mock_ei_ipv6, 0, sizeof(mock_ei_ipv6));
+
+  /* Set up the IPv4 extend info */
+  memcpy(mock_ei_ipv4.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ei_ipv4.identity_digest));
+  strlcpy(mock_ei_ipv4.nickname, "TestOR7890123456789",
+          sizeof(mock_ei_ipv4.nickname));
+  tor_addr_parse(&mock_ei_ipv4.addr, "111.222.233.244");
+
+  /* Create and modify the other extend info. */
+  memcpy(&mock_ei_ipv6, &mock_ei_ipv4, sizeof(mock_ei_ipv6));
+  tor_addr_parse(&mock_ei_ipv6.addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = extend_info_describe(&mock_ei_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = extend_info_describe(&mock_ei_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Extend infos only have one IP address, so there is no dual case */
+
+  /* test NULL handling */
+  rv = extend_info_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+ done:
+  return;
+}
+
+/** router_get_verbose_nickname() should return "Fingerprint~Nickname"
+ */
+static void
+test_nodelist_router_get_verbose_nickname(void *arg)
+{
+  routerinfo_t mock_ri;
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+
+  char vname[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  (void) arg;
+
+  memset(&mock_ri, 0, sizeof(routerinfo_t));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  mock_ri.nickname = mock_nickname;
+
+  /* verbose nickname should use ~ because named is deprecated */
+  strlcpy(mock_nickname, "TestOR", sizeof(mock_nickname));
+  memcpy(mock_ri.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         DIGEST_LEN);
+  router_get_verbose_nickname(vname, &mock_ri);
+  tt_str_op(vname, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR");
+
+  /* test NULL router handling */
+  router_get_verbose_nickname(vname, NULL);
+  tt_str_op(vname, OP_EQ, "<null>");
+
+  router_get_verbose_nickname(NULL, &mock_ri);
+  router_get_verbose_nickname(NULL, NULL);
+
+ done:
+  return;
+}
+
 #define NODE(name, flags) \
   { #name, test_nodelist_##name, (flags), NULL, NULL }
 
@@ -239,6 +1254,17 @@ struct testcase_t nodelist_tests[] = {
   NODE(node_get_verbose_nickname_not_named, TT_FORK),
   NODE(node_is_dir, TT_FORK),
   NODE(ed_id, TT_FORK),
+  NODE(nodefamily, TT_FORK),
+  NODE(nodefamily_parse_err, TT_FORK),
+  NODE(nodefamily_lookup, TT_FORK),
+  NODE(nickname_matches, 0),
+  NODE(node_nodefamily, TT_FORK),
+  NODE(nodefamily_canonicalize, 0),
+  NODE(format_node_description, 0),
+  NODE(router_describe, 0),
+  NODE(node_describe, 0),
+  NODE(routerstatus_describe, 0),
+  NODE(extend_info_describe, 0),
+  NODE(router_get_verbose_nickname, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_oom.c b/src/test/test_oom.c
index b813fa43a3..da6b2ee14d 100644
--- a/src/test/test_oom.c
+++ b/src/test/test_oom.c
@@ -8,7 +8,7 @@
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "lib/evloop/compat_libevent.h"
 #include "core/mainloop/connection.h"
diff --git a/src/test/test_options.c b/src/test/test_options.c
index 0e52967a23..a2641425fe 100644
--- a/src/test/test_options.c
+++ b/src/test/test_options.c
@@ -5,7 +5,7 @@
 
 #define CONFIG_PRIVATE
 #include "core/or/or.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "app/config/config.h"
 #include "test/test.h"
 #include "lib/geoip/geoip.h"
@@ -14,6 +14,7 @@
 #include "feature/nodelist/routerset.h"
 #include "core/mainloop/mainloop.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "lib/sandbox/sandbox.h"
 #include "lib/memarea/memarea.h"
@@ -31,14 +32,14 @@
 
 typedef struct {
   int severity;
-  uint32_t domain;
+  log_domain_mask_t domain;
   char *msg;
 } logmsg_t;
 
 static smartlist_t *messages = NULL;
 
 static void
-log_cback(int severity, uint32_t domain, const char *msg)
+log_cback(int severity, log_domain_mask_t domain, const char *msg)
 {
   logmsg_t *x = tor_malloc(sizeof(*x));
   x->severity = severity;
@@ -54,9 +55,9 @@ setup_log_callback(void)
 {
   log_severity_list_t lst;
   memset(&lst, 0, sizeof(lst));
-  lst.masks[LOG_ERR - LOG_ERR] = ~0;
-  lst.masks[LOG_WARN - LOG_ERR] = ~0;
-  lst.masks[LOG_NOTICE - LOG_ERR] = ~0;
+  lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = LD_ALL_DOMAINS;
   add_callback_log(&lst, log_cback);
   mark_logs_temp();
 }
@@ -96,7 +97,7 @@ clear_log_messages(void)
     opt->command = CMD_RUN_TOR;              \
     options_init(opt);                       \
                                              \
-    dflt = config_dup(&options_format, opt); \
+    dflt = config_dup(get_options_mgr(), opt); \
     clear_log_messages();                    \
   } while (0)
 
@@ -196,7 +197,7 @@ test_options_validate_impl(const char *configuration,
   if (r)
     goto done;
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   if (phase == PH_ASSIGN) {
     if (test_options_checkmsgs(configuration, expect_errmsg,
                                expect_log_severity,
@@ -241,6 +242,7 @@ test_options_validate(void *arg)
   (void)arg;
   setup_log_callback();
   sandbox_disable_getaddrinfo_cache();
+  mock_hostname_resolver();
 
   WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE);
 
@@ -258,13 +260,17 @@ test_options_validate(void *arg)
   WANT_ERR("BridgeRelay 1\nDirCache 0",
            "We're a bridge but DirCache is disabled.", PH_VALIDATE);
 
+  // XXXX We should replace this with a more full error message once #29211
+  // XXXX is done.  It is truncated for now because at the current stage
+  // XXXX of refactoring, we can't give a full error message like before.
   WANT_ERR_LOG("HeartbeatPeriod 21 snarks",
-               "Interval 'HeartbeatPeriod 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               "malformed or out of bounds", LOG_WARN,
+               "Unknown unit 'snarks'.",
                PH_ASSIGN);
+  // XXXX As above.
   WANT_ERR_LOG("LogTimeGranularity 21 snarks",
-               "Msec interval 'LogTimeGranularity 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
+               "malformed or out of bounds", LOG_WARN,
+               "Unknown unit 'snarks'.",
                PH_ASSIGN);
   OK("HeartbeatPeriod 1 hour", PH_VALIDATE);
   OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE);
@@ -278,6 +284,7 @@ test_options_validate(void *arg)
 
   close_temp_logs();
   clear_log_messages();
+  unmock_hostname_resolver();
   return;
 }
 
@@ -300,7 +307,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled */
@@ -323,7 +330,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled, Bridge */
@@ -346,7 +353,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 200 MB RAM available, DirCache disabled */
@@ -429,10 +436,12 @@ get_options_test_data(const char *conf)
   // with options_init(), but about a dozen tests break when I do that.
   // Being kinda lame and just fixing the immedate breakage for now..
   result->opt->ConnectionPadding = -1; // default must be "auto"
+  result->opt->DormantClientTimeout = 1800; // must be over 600.
+  result->opt->CircuitPadding = 1; // default must be "1"
 
   rv = config_get_lines(conf, &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->opt, cl, 0, &msg);
+  rv = config_assign(get_options_mgr(), result->opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
@@ -443,7 +452,7 @@ get_options_test_data(const char *conf)
   result->opt->TokenBucketRefillInterval = 1;
   rv = config_get_lines(TEST_OPTIONS_OLD_VALUES, &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->def_opt, cl, 0, &msg);
+  rv = config_assign(get_options_mgr(), result->def_opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
@@ -1171,7 +1180,7 @@ test_options_validate__transproxy(void *ignored)
   // Assert that a test has run for some TransProxyType
   tt_assert(tdata);
 
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   tdata = get_options_test_data("TransPort 127.0.0.1:555\n");
 
   ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
@@ -1346,29 +1355,6 @@ test_options_validate__token_bucket(void *ignored)
 }
 
 static void
-test_options_validate__recommended_packages(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_WARN);
-  options_test_data_t *tdata = get_options_test_data(
-            "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n"
-            "RecommendedPackages invalid-package-line\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg("Invalid RecommendedPackage line "
-            "invalid-package-line will be ignored\n");
-
- done:
-  escaped(NULL); // This will free the leaking memory from the previous escaped
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__fetch_dir(void *ignored)
 {
   (void)ignored;
@@ -4203,7 +4189,6 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(exclude_nodes),
   LOCAL_VALIDATE_TEST(node_families),
   LOCAL_VALIDATE_TEST(token_bucket),
-  LOCAL_VALIDATE_TEST(recommended_packages),
   LOCAL_VALIDATE_TEST(fetch_dir),
   LOCAL_VALIDATE_TEST(conn_limit),
   LOCAL_VALIDATE_TEST(paths_needed),
diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c
new file mode 100644
index 0000000000..0c8f467a45
--- /dev/null
+++ b/src/test/test_parsecommon.c
@@ -0,0 +1,594 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/memarea/memarea.h"
+#include "lib/encoding/binascii.h"
+#include "feature/dirparse/parsecommon.h"
+#include "test/log_test_helpers.h"
+
+static void
+test_parsecommon_tokenize_string_null(void *arg)
+{
+
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  const char *str_with_null = "a\0bccccccccc";
+
+  int retval =
+  tokenize_string(area, str_with_null,
+                  str_with_null + 3,
+                  tokens, NULL, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_multiple_lines(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, GE(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup(
+          "hibernating 0\nuptime 1024\n"
+          "published 2018-10-15 10:00:00\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(smartlist_len(tokens), OP_EQ, 3);
+  directory_token_t *token = smartlist_get(tokens, 0);
+
+  tt_int_op(token->tp, OP_EQ, K_HIBERNATING);
+
+  token = smartlist_get(tokens, 1);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+
+  token = smartlist_get(tokens, 2);
+
+  tt_int_op(token->tp, OP_EQ, K_PUBLISHED);
+
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_min_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(2), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // Missing "uptime"
+  char *str = tor_strdup("uptime 1024\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_max_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "uptime" expected once, but occurs twice in input.
+  char *str = tor_strdup(
+                  "uptime 1024\nuptime 2048\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_start(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the first line.
+  char *str = tor_strdup(
+                  "uptime 1024\nclient-name Alice\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_end(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the last line.
+  char *str = tor_strdup(
+                  "client-name Alice\nuptime 1024\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_no_annotations(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_success(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "1024");
+  tt_assert(!token->object_type);
+  tt_int_op(token->object_size, OP_EQ, 0);
+  tt_assert(!token->object_body);
+
+  tt_ptr_op(*s, OP_EQ, end);
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_concat_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "proto A=1 B=2";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_PROTO);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "A=1 B=2");
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_parse_keys(void *arg)
+{
+  (void)arg;
+
+  memarea_t *area = memarea_new();
+  const char *str =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n"
+    "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n"
+    "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = (const char **)&str;
+  directory_token_t *token = NULL;
+  directory_token_t *token2 = NULL;
+
+  token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024);
+
+  token = get_next_token(area, s, end, &rule);
+  tt_assert(token);
+
+  tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY);
+  tt_int_op(token->n_args, OP_EQ, 0);
+  tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY");
+  tt_int_op(token->object_size, OP_EQ, 140);
+  tt_assert(token->object_body);
+  tt_assert(token->key);
+  tt_assert(!token->error);
+
+  const char *str2 =
+    "client-key\n"
+    "-----BEGIN RSA PRIVATE KEY-----\n"
+    "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n"
+    "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n"
+    "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n"
+    "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n"
+    "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n"
+    "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n"
+    "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n"
+    "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n"
+    "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n"
+    "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n"
+    "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n"
+    "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n"
+    "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"
+    "-----END RSA PRIVATE KEY-----\n";
+
+  const char *end2 = str2 + strlen(str2);
+  const char **s2 = (const char **)&str2;
+
+  token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS,
+                           NEED_SKEY_1024);
+
+  token2 = get_next_token(area, s2, end2, &rule2);
+  tt_assert(token2);
+
+  tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY);
+  tt_int_op(token2->n_args, OP_EQ, 0);
+  tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY");
+  tt_int_op(token2->object_size, OP_EQ, 608);
+  tt_assert(token2->object_body);
+  tt_assert(token2->key);
+  tt_assert(!token->error);
+
+ done:
+  if (token) token_clear(token);
+  if (token2) token_clear(token2);
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_object(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE);
+  tt_int_op(token->n_args, OP_EQ, 2);
+  tt_str_op(token->args[0], OP_EQ,
+                  "0232AF901C31A04EE9848595AF9BB7620D4C5B2E");
+  tt_str_op(token->args[1], OP_EQ,
+                  "CD1FD971855430880D3C31E0331C5C55800C2F79");
+
+  tt_assert(!token->error);
+
+  char decoded[256];
+  const char *signature =
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+  tt_assert(signature);
+  size_t signature_len = strlen(signature);
+  base64_decode(decoded, sizeof(decoded), signature, signature_len);
+
+  tt_str_op(token->object_type, OP_EQ, "SIGNATURE");
+  tt_int_op(token->object_size, OP_EQ, 256);
+  tt_mem_op(token->object_body, OP_EQ, decoded, 256);
+
+  tt_assert(!token->key);
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_err_too_many_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024 1024 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too many arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_too_few_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too few arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_obj_missing_endline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_beginline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-Z---\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_tag_mismatch(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SOMETHINGELSE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ,
+            "Malformed object: mismatched end tag SIGNATURE");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_base64(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "%%@%%%%%%%!!!'\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+#define PARSECOMMON_TEST(name) \
+  { #name, test_parsecommon_ ## name, 0, NULL, NULL }
+
+struct testcase_t parsecommon_tests[] = {
+  PARSECOMMON_TEST(tokenize_string_null),
+  PARSECOMMON_TEST(tokenize_string_multiple_lines),
+  PARSECOMMON_TEST(tokenize_string_min_cnt),
+  PARSECOMMON_TEST(tokenize_string_max_cnt),
+  PARSECOMMON_TEST(tokenize_string_at_start),
+  PARSECOMMON_TEST(tokenize_string_at_end),
+  PARSECOMMON_TEST(tokenize_string_no_annotations),
+  PARSECOMMON_TEST(get_next_token_success),
+  PARSECOMMON_TEST(get_next_token_concat_args),
+  PARSECOMMON_TEST(get_next_token_parse_keys),
+  PARSECOMMON_TEST(get_next_token_object),
+  PARSECOMMON_TEST(get_next_token_err_too_many_args),
+  PARSECOMMON_TEST(get_next_token_err_too_few_args),
+  PARSECOMMON_TEST(get_next_token_err_obj_missing_endline),
+  PARSECOMMON_TEST(get_next_token_err_bad_beginline),
+  PARSECOMMON_TEST(get_next_token_err_tag_mismatch),
+  PARSECOMMON_TEST(get_next_token_err_bad_base64),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_parseconf.sh b/src/test/test_parseconf.sh
new file mode 100755
index 0000000000..eeb80cdfa7
--- /dev/null
+++ b/src/test/test_parseconf.sh
@@ -0,0 +1,204 @@
+#!/bin/sh
+# Copyright 2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+# Integration test script for verifying that Tor configurations are parsed as
+# we expect.
+#
+# Valid configurations are tested with --dump-config, which parses and
+# validates the configuration before writing it out.  We then make sure that
+# the result is what we expect, before parsing and dumping it again to make
+# sure that there is no change.
+#
+# Invalid configurations are tested with --verify-config, which parses
+# and validates the configuration. We capture its output and make sure that
+# it contains the error message we expect.
+
+# This script looks for its test cases as individual directories in
+# src/test/conf_examples/.  Each test may have these files:
+#
+# torrc -- Usually needed. This file is passed to Tor on the command line
+#      with the "-f" flag. (If you omit it, you'll test Tor's behavior when
+#      it receives a nonexistent configuration file.)
+#
+# torrc.defaults -- Optional. If present, it is passed to Tor on the command
+#      line with the --defaults-torrc option. If this file is absent, an empty
+#      file is passed instead to prevent Tor from reading the system defaults.
+#
+# cmdline -- Optional. If present, it contains command-line arguments that
+#      will be passed to Tor.
+#
+# expected -- If this file is present, then it should be the expected result
+#      of "--dump-config short" for this test case.  Exactly one of
+#      "expected" or "error" must be present, or the test will fail.
+#
+# error -- If this file is present, then it contains a regex that must be
+#      matched by some line in the output of "--verify-config", which must
+#      fail. Exactly one of "expected" or "error" must be present, or the
+#      test will fail.
+
+umask 077
+set -e
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+# make a safe space for temporary files
+DATA_DIR=$(mktemp -d -t tor_parseconf_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# This is where we look for examples
+EXAMPLEDIR="$(dirname "$0")"/conf_examples
+
+case "$(uname -s)" in
+    CYGWIN*) WINDOWS=1;;
+    MINGW*) WINDOWS=1;;
+    MSYS*) WINDOWS=1;;
+    *) WINDOWS=0;;
+esac
+
+####
+# BUG WORKAROUND FOR 31757:
+#  On Appveyor, it seems that Tor sometimes randomly fails to produce
+#  output with --dump-config.  Whil we are figuring this out, do not treat
+#  windows errors as hard failures.
+####
+if test "$WINDOWS" = 1; then
+    EXITCODE=0
+else
+    EXITCODE=1
+fi
+
+die() { echo "$1" >&2 ; exit "$EXITCODE"; }
+
+if test "$WINDOWS" = 1; then
+    FILTER="dos2unix"
+else
+    FILTER="cat"
+fi
+
+touch "${DATA_DIR}/EMPTY" || die "Couldn't create empty file."
+
+for dir in "${EXAMPLEDIR}"/*; do
+    if ! test -d "${dir}"; then
+       # Only count directories.
+       continue
+    fi
+
+    testname="$(basename "${dir}")"
+    # We use printf since "echo -n" is not standard
+    printf "%s: " "$testname"
+
+    PREV_DIR="$(pwd)"
+    cd "${dir}"
+
+    if test -f "./torrc.defaults"; then
+        DEFAULTS="./torrc.defaults"
+    else
+        DEFAULTS="${DATA_DIR}/EMPTY"
+    fi
+
+    if test -f "./cmdline"; then
+        CMDLINE="$(cat ./cmdline)"
+    else
+        CMDLINE=""
+    fi
+
+    if test -f "./expected"; then
+        if test -f "./error"; then
+            echo "FAIL: Found both ${dir}/expected and ${dir}/error."
+            echo "(Only one of these files should exist.)"
+            exit $EXITCODE
+        fi
+
+        # This case should succeed: run dump-config and see if it does.
+
+        "${TOR_BINARY}" -f "./torrc" \
+                        --defaults-torrc "${DEFAULTS}" \
+                        --dump-config short \
+                        ${CMDLINE} \
+                        | "${FILTER}" > "${DATA_DIR}/output.${testname}" \
+                        || die "Failure: Tor exited."
+
+        if cmp "./expected" "${DATA_DIR}/output.${testname}">/dev/null ; then
+            # Check round-trip.
+            "${TOR_BINARY}" -f "${DATA_DIR}/output.${testname}" \
+                            --defaults-torrc "${DATA_DIR}/empty" \
+                            --dump-config short \
+                            | "${FILTER}" \
+                            > "${DATA_DIR}/output_2.${testname}" \
+                        || die "Failure: Tor exited on round-trip."
+
+            if ! cmp "${DATA_DIR}/output.${testname}" \
+                 "${DATA_DIR}/output_2.${testname}"; then
+                echo "Failure: did not match on round-trip."
+                exit $EXITCODE
+            fi
+
+            echo "OK"
+        else
+            echo "FAIL"
+            if test "$(wc -c < "${DATA_DIR}/output.${testname}")" = 0; then
+                # There was no output -- probably we failed.
+                "${TOR_BINARY}" -f "./torrc" \
+                                --defaults-torrc "${DEFAULTS}" \
+                                --verify-config \
+                                ${CMDLINE} || true
+            fi
+            diff -u "./expected" "${DATA_DIR}/output.${testname}" || /bin/true
+            exit $EXITCODE
+        fi
+
+   elif test -f "./error"; then
+        # This case should fail: run verify-config and see if it does.
+
+        "${TOR_BINARY}" --verify-config \
+                        -f ./torrc \
+                        --defaults-torrc "${DEFAULTS}" \
+                        ${CMDLINE} \
+                        > "${DATA_DIR}/output.${testname}" \
+                        && die "Failure: Tor did not report an error."
+
+        expect_err="$(cat ./error)"
+        if grep "${expect_err}" "${DATA_DIR}/output.${testname}" >/dev/null; then
+            echo "OK"
+        else
+            echo "FAIL"
+            echo "Expected error: ${expect_err}"
+            echo "Tor said:"
+            cat "${DATA_DIR}/output.${testname}"
+            exit $EXITCODE
+        fi
+
+    else
+        # This case is not actually configured with a success or a failure.
+        # call that an error.
+
+        echo "FAIL: Did not find ${dir}/expected or ${dir}/error."
+        exit $EXITCODE
+    fi
+
+    cd "${PREV_DIR}"
+
+done
diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c
index 4a53639dad..267156a908 100644
--- a/src/test/test_periodic_event.c
+++ b/src/test/test_periodic_event.c
@@ -19,6 +19,7 @@
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_service.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/periodic.h"
 
 /** Helper function: This is replaced in some tests for the event callbacks so
@@ -50,16 +51,21 @@ test_pe_initialize(void *arg)
    * need to run the main loop and then wait for a second delaying the unit
    * tests. Instead, we'll test the callback work indepedently elsewhere. */
   initialize_periodic_events();
+  periodic_events_connect_all();
+  set_network_participation(false);
+  rescan_periodic_events(get_options());
 
   /* Validate that all events have been set up. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     tt_assert(item->ev);
     tt_assert(item->fn);
     tt_u64_op(item->last_action_time, OP_EQ, 0);
     /* Every event must have role(s) assign to it. This is done statically. */
     tt_u64_op(item->roles, OP_NE, 0);
-    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
  done:
@@ -79,17 +85,20 @@ test_pe_launch(void *arg)
    * network gets enabled. */
   consider_hibernation(time(NULL));
 
+  set_network_participation(true);
+
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
   options = get_options_mutable();
   options->SocksPort_set = 1;
   periodic_events_on_new_options(options);
+
 #if 0
   /* Lets make sure that before intialization, we can't scan the periodic
    * events list and launch them. Lets try by being a Client. */
@@ -99,20 +108,20 @@ test_pe_launch(void *arg)
     periodic_event_item_t *item = &periodic_events[i];
     tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
   }
-#endif
+#endif /* 0 */
 
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Now that we've initialized, rescan the list to launch. */
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
-    } else {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
-    }
+  int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL|
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = !!(item->roles & mask);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
     // enabled or not, the event has not yet been run.
     tt_u64_op(item->last_action_time, OP_EQ, 0);
   }
@@ -124,10 +133,11 @@ test_pe_launch(void *arg)
 
   unsigned roles = get_my_roles(options);
   tt_uint_op(roles, OP_EQ,
-             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER);
+             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER|
+             PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     /* Only Client role should be disabled. */
     if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
@@ -144,17 +154,23 @@ test_pe_launch(void *arg)
   /* Disable everything and we'll enable them ALL. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
+  options->DisableNetwork = 1;
+  set_network_participation(false);
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
   /* Enable everything. */
   options->SocksPort_set = 1; options->ORPort_set = 1;
   options->BridgeRelay = 1; options->AuthoritativeDir = 1;
   options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1;
+  options->DisableNetwork = 0;
+  set_network_participation(true);
   register_dummy_hidden_service(&service);
   periodic_events_on_new_options(options);
   /* Note down the reference because we need to remove this service from the
@@ -163,9 +179,10 @@ test_pe_launch(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ,
+              (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV));
   }
 
  done:
@@ -187,42 +204,49 @@ test_pe_get_roles(void *arg)
 
   or_options_t *options = get_options_mutable();
   tt_assert(options);
+  set_network_participation(true);
+
+  const int ALL = PERIODIC_EVENT_ROLE_ALL |
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
 
   /* Nothing configured, should be no roles. */
+  tt_assert(net_is_disabled());
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Indicate we have a SocksPort, roles should be come Client. */
   options->SocksPort_set = 1;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT);
+  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL);
 
   /* Now, we'll add a ORPort so should now be a Relay + Client. */
   options->ORPort_set = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_DIRSERVER | ALL));
 
   /* Now add a Bridge. */
   options->BridgeRelay = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER);
   /* Unset client so we can solely test Router role. */
   options->SocksPort_set = 0;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER |
+            ALL);
 
   /* Reset options so we can test authorities. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
   options->BridgeRelay = 0;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Now upgrade to Dirauth. */
   options->DirPort_set = 1;
@@ -230,7 +254,7 @@ test_pe_get_roles(void *arg)
   options->V3AuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Now Bridge Authority. */
@@ -238,7 +262,7 @@ test_pe_get_roles(void *arg)
   options->BridgeAuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Move that bridge auth to become a relay. */
@@ -246,7 +270,7 @@ test_pe_get_roles(void *arg)
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY
-             | PERIODIC_EVENT_ROLE_DIRSERVER));
+             | PERIODIC_EVENT_ROLE_DIRSERVER|ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* And now an Hidden service. */
@@ -257,7 +281,8 @@ test_pe_get_roles(void *arg)
   remove_service(get_hs_service_map(), &service);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
  done:
@@ -277,12 +302,13 @@ test_pe_hs_service(void *arg)
   consider_hibernation(time(NULL));
   /* Initialize the events so we can enable them */
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
@@ -295,8 +321,8 @@ test_pe_hs_service(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
     }
@@ -306,8 +332,8 @@ test_pe_hs_service(void *arg)
   /* Remove the service from the global map, it should trigger a rescan and
    * disable the HS service events. */
   remove_service(get_hs_service_map(), &service);
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
     }
diff --git a/src/test/test_policy.c b/src/test/test_policy.c
index 9c001c294a..e58bb3d174 100644
--- a/src/test/test_policy.c
+++ b/src/test/test_policy.c
@@ -6,13 +6,18 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/circuitbuild.h"
 #include "core/or/policies.h"
 #include "feature/dirparse/policy_parse.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_descriptor.h"
 #include "feature/relay/router.h"
 #include "lib/encoding/confline.h"
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #include "core/or/addr_policy_st.h"
+#include "core/or/extend_info_st.h"
 #include "core/or/port_cfg_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -2024,6 +2029,115 @@ test_policies_fascist_firewall_allows_address(void *arg)
                            expect_ap); \
   STMT_END
 
+/* Check that fascist_firewall_choose_address_ls() returns the expected
+ * results. */
+#define CHECK_CHOSEN_ADDR_NULL_LS() \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(NULL, 1, &chosen_ls_ap); \
+    expect_single_log_msg("Unknown or missing link specifiers"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_CHOSEN_ADDR_LS(fake_ls, pref_only, expect_rv, expect_ap) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, pref_only, &chosen_ls_ap); \
+    if (smartlist_len(fake_ls) == 0) { \
+      expect_single_log_msg("Link specifiers are empty"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_ls_ap.addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, chosen_ls_ap.port); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_LS_LEGACY_ONLY(fake_ls) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, 0, &chosen_ls_ap); \
+    expect_single_log_msg("None of our link specifiers have IPv4 or IPv6"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS(fake_ls, direct_conn, expect_ap) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    if (fake_ls == NULL) { \
+      tt_ptr_op(ei, OP_EQ, NULL); \
+      expect_single_log_msg("Specified link specifiers is null"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &ei->addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, ei->port); \
+      extend_info_free(ei); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(fake_ls) \
+  STMT_BEGIN \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, NULL, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg("Specified onion key is null"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(fake_ls, direct_conn) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(fake_ls, msg_level, msg) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(msg_level); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg(msg); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+/** Mock the preferred address function to return zero (prefer IPv4). */
+static int
+mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4(void)
+{
+  return 0;
+}
+
+/** Mock the preferred address function to return one (prefer IPv6). */
+static int
+mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6(void)
+{
+  return 1;
+}
+
 /** Run unit tests for fascist_firewall_choose_address */
 static void
 test_policies_fascist_firewall_choose_address(void *arg)
@@ -2422,6 +2536,177 @@ test_policies_fascist_firewall_choose_address(void *arg)
   CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1,
                        ipv4_dir_ap);
 
+  /* Test ClientAutoIPv6ORPort and pretend we prefer IPv4. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientAutoIPv6ORPort = 1;
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+  MOCK(fascist_firewall_rand_prefer_ipv6_addr,
+       mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv4);
+  /* Simulate the initialisation of fake_node.ipv6_preferred */
+  fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(
+                                                                &mock_options);
+
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1,
+                       ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1,
+                       ipv4_or_ap);
+
+  UNMOCK(fascist_firewall_rand_prefer_ipv6_addr);
+
+  /* Test ClientAutoIPv6ORPort and pretend we prefer IPv6. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientAutoIPv6ORPort = 1;
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+  MOCK(fascist_firewall_rand_prefer_ipv6_addr,
+       mock_fascist_firewall_rand_prefer_ipv6_addr_use_ipv6);
+  /* Simulate the initialisation of fake_node.ipv6_preferred */
+  fake_node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(
+                                                                &mock_options);
+
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 0, 1,
+                       ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_OR_CONNECTION, 1, 1,
+                       ipv6_or_ap);
+
+  UNMOCK(fascist_firewall_rand_prefer_ipv6_addr);
+
+  /* Test firewall_choose_address_ls(). To do this, we make a fake link
+   * specifier. */
+  smartlist_t *lspecs = smartlist_new(),
+              *lspecs_blank = smartlist_new(),
+              *lspecs_v4 = smartlist_new(),
+              *lspecs_v6 = smartlist_new(),
+              *lspecs_no_legacy = smartlist_new(),
+              *lspecs_legacy_only = smartlist_new();
+  link_specifier_t *fake_ls;
+
+  /* IPv4 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(fake_ls,
+                                  tor_addr_to_ipv4h(&ipv4_or_ap.addr));
+  link_specifier_set_un_ipv4_port(fake_ls, ipv4_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, sizeof(ipv4_or_ap.addr.addr.in_addr) +
+                            sizeof(ipv4_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_no_legacy, fake_ls);
+
+  /* IPv6 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV6);
+  size_t addr_len = link_specifier_getlen_un_ipv6_addr(fake_ls);
+  const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ipv6_or_ap.addr);
+  uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(fake_ls);
+  memcpy(ipv6_array, in6_addr, addr_len);
+  link_specifier_set_un_ipv6_port(fake_ls, ipv6_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, addr_len + sizeof(ipv6_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Legacy ID link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_LEGACY_ID);
+  uint8_t *legacy_id = link_specifier_getarray_un_legacy_id(fake_ls);
+  memset(legacy_id, 'A', sizeof(*legacy_id));
+  link_specifier_set_ls_len(fake_ls,
+                            link_specifier_getlen_un_legacy_id(fake_ls));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_legacy_only, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Check with bogus requests. */
+  tor_addr_port_t null_ap; \
+  tor_addr_make_null(&null_ap.addr, AF_UNSPEC); \
+  null_ap.port = 0; \
+
+  /* Check for a null link state. */
+  CHECK_CHOSEN_ADDR_NULL_LS();
+  CHECK_HS_EXTEND_INFO_ADDR_LS(NULL, 1, null_ap);
+
+  /* Check for a blank link state. */
+  CHECK_CHOSEN_ADDR_LS(lspecs_blank, 0, 0, null_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_blank, 0);
+
+  /* Check for a link state with only a Legacy ID. */
+  CHECK_LS_LEGACY_ONLY(lspecs_legacy_only);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_legacy_only, 0);
+  smartlist_free(lspecs_legacy_only);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(lspecs_blank);
+  smartlist_free(lspecs_blank);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(lspecs_no_legacy, LOG_WARN,
+                                          "Missing Legacy ID in link state");
+  smartlist_free(lspecs_no_legacy);
+
+  /* Enable both IPv4 and IPv6. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Prefer IPv4, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* Prefer IPv6, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* IPv4-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v6, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 1);
+
+  /* IPv6-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 0;
+  mock_options.ClientUseIPv6 = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v4, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v4, 1);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+
+  smartlist_free(lspecs_v4);
+  smartlist_free(lspecs_v6);
+
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, lspec, \
+                    link_specifier_free(lspec)); \
+  smartlist_free(lspecs);
+
  done:
   UNMOCK(get_options);
 }
diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c
new file mode 100644
index 0000000000..0ecbf65f41
--- /dev/null
+++ b/src/test/test_prob_distr.c
@@ -0,0 +1,1402 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_prob_distr.c
+ * \brief Test probability distributions.
+ * \detail
+ *
+ * For each probability distribution we do two kinds of tests:
+ *
+ * a) We do numerical deterministic testing of their cdf/icdf/sf/isf functions
+ *    and the various relationships between them for each distribution. We also
+ *    do deterministic tests on their sampling functions. Test vectors for
+ *    these tests were computed from alternative implementations and were
+ *    eyeballed to make sure they make sense
+ *    (e.g. src/test/prob_distr_mpfr_ref.c computes logit(p) using GNU mpfr
+ *    with 200-bit precision and is then tested in test_logit_logistic()).
+ *
+ * b) We do stochastic hypothesis testing (G-test) to ensure that sampling from
+ *    the given distributions is distributed properly. The stochastic tests are
+ *    slow and their false positive rate is not well suited for CI, so they are
+ *    currently disabled-by-default and put into 'tests-slow'.
+ */
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "test/test.h"
+
+#include "core/or/or.h"
+
+#include "lib/math/prob_distr.h"
+#include "lib/math/fp.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "test/rng_test_helpers.h"
+
+#include <float.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/**
+ * Return floor(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)floor(d).
+ */
+static size_t
+floor_to_size_t(double d)
+{
+  double integral_d = floor(d);
+  return (size_t)integral_d;
+}
+
+/**
+ * Return ceil(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)ceil(d).
+ */
+static size_t
+ceil_to_size_t(double d)
+{
+  double integral_d = ceil(d);
+  return (size_t)integral_d;
+}
+
+/*
+ * Geometric(p) distribution, supported on {1, 2, 3, ...}.
+ *
+ * Compute the probability mass function Geom(n; p) of the number of
+ * trials before the first success when success has probability p.
+ */
+static double
+logpmf_geometric(unsigned n, double p)
+{
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1) {
+    if (n == 1)
+      return 0;
+    else
+      return -HUGE_VAL;
+  }
+  return (n - 1)*log1p(-p) + log(p);
+}
+
+/**
+ * Compute the logistic function, translated in output by 1/2:
+ * logistichalf(x) = logistic(x) - 1/2.  Well-conditioned on the entire
+ * real plane, with maximum condition number 1 at 0.
+ *
+ * This implementation gives relative error bounded by 5 eps.
+ */
+static double
+logistichalf(double x)
+{
+  /*
+   * Rewrite this with the identity
+   *
+   *  1/(1 + e^{-x}) - 1/2
+   *  = (1 - 1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1 - e^{-x})/[2 (1 + e^{-x})]
+   *  = -(e^{-x} - 1)/[2 (1 + e^{-x})],
+   *
+   * which we can evaluate by -expm1(-x)/[2 (1 + exp(-x))].
+   *
+   * Suppose exp has error d0, + has error d1, expm1 has error
+   * d2, and / has error d3, so we evaluate
+   *
+   *  -(1 + d2) (1 + d3) (e^{-x} - 1)
+   *    / [2 (1 + d1) (1 + (1 + d0) e^{-x})].
+   *
+   * In the denominator,
+   *
+   *  1 + (1 + d0) e^{-x}
+   *  = 1 + e^{-x} + d0 e^{-x}
+   *  = (1 + e^{-x}) (1 + d0 e^{-x}/(1 + e^{-x})),
+   *
+   * so the relative error of the numerator is
+   *
+   *  d' = d2 + d3 + d2 d3,
+   * and of the denominator,
+   *  d'' = d1 + d0 e^{-x}/(1 + e^{-x}) + d0 d1 e^{-x}/(1 + e^{-x})
+   *      = d1 + d0 L(-x) + d0 d1 L(-x),
+   *
+   * where L(-x) is logistic(-x).  By Lemma 1 the relative error
+   * of the quotient is bounded by
+   *
+   *  2|d2 + d3 + d2 d3 - d1 - d0 L(x) + d0 d1 L(x)|,
+   *
+   * Since 0 < L(x) < 1, this is bounded by
+   *
+   *  2|d2| + 2|d3| + 2|d2 d3| + 2|d1| + 2|d0| + 2|d0 d1|
+   *  <= 4 eps + 2 eps^2.
+   */
+  if (x < log(DBL_EPSILON/8)) {
+    /*
+     * Avoid overflow in e^{-x}.  When x < log(eps/4), we
+     * we further have x < logit(eps/4), so that
+     * logistic(x) < eps/4.  Hence the relative error of
+     * logistic(x) - 1/2 from -1/2 is bounded by eps/2, and
+     * so the relative error of -1/2 from logistic(x) - 1/2
+     * is bounded by eps.
+     */
+    return -0.5;
+  } else {
+    return -expm1(-x)/(2*(1 + exp(-x)));
+  }
+}
+
+/**
+ * Compute the log of the sum of the exps.  Caller should arrange the
+ * array in descending order to minimize error because I don't want to
+ * deal with using temporary space and the one caller in this file
+ * arranges that anyway.
+ *
+ * Warning: This implementation does not handle infinite or NaN inputs
+ * sensibly, because I don't need that here at the moment.  (NaN, or
+ * -inf and +inf together, should yield NaN; +inf and finite should
+ * yield +inf; otherwise all -inf should be ignored because exp(-inf) =
+ * 0.)
+ */
+static double
+logsumexp(double *A, size_t n)
+{
+  double maximum, sum;
+  size_t i;
+
+  if (n == 0)
+    return log(0);
+
+  maximum = A[0];
+  for (i = 1; i < n; i++) {
+    if (A[i] > maximum)
+      maximum = A[i];
+  }
+
+  sum = 0;
+  for (i = n; i --> 0;)
+    sum += exp(A[i] - maximum);
+
+  return log(sum) + maximum;
+}
+
+/**
+ * Compute log(1 - e^x).  Defined only for negative x so that e^x < 1.
+ * This is the complement of a probability in log space.
+ */
+static double
+log1mexp(double x)
+{
+
+  /*
+   * We want to compute log on [0, 1/2) but log1p on [1/2, +inf),
+   * so partition x at -log(2) = log(1/2).
+   */
+  if (-log(2) < x)
+    return log(-expm1(x));
+  else
+    return log1p(-exp(x));
+}
+
+/*
+ * Tests of numerical errors in computing logit, logistic, and the
+ * various cdfs, sfs, icdfs, and isfs.
+ */
+
+#define arraycount(A) (sizeof(A)/sizeof(A[0]))
+
+/** Return relative error between <b>actual</b> and <b>expected</b>.
+ *  Special cases: If <b>expected</b> is zero or infinite, return 1 if
+ *  <b>actual</b> is equal to <b>expected</b> and 0 if not, since the
+ *  usual notion of relative error is undefined but we only use this
+ *  for testing relerr(e, a) <= bound.  If either is NaN, return NaN,
+ *  which has the property that NaN <= bound is false no matter what
+ *  bound is.
+ *
+ *  Beware: if you test !(relerr(e, a) > bound), then then the result
+ *  is true when a is NaN because NaN > bound is false too.  See
+ *  CHECK_RELERR for correct use to decide when to report failure.
+ */
+static double
+relerr(double expected, double actual)
+{
+  /*
+   * To silence -Wfloat-equal, we have to test for equality using
+   * inequalities: we have (fabs(expected) <= 0) iff (expected == 0),
+   * and (actual <= expected && actual >= expected) iff actual ==
+   * expected whether expected is zero or infinite.
+   */
+  if (fabs(expected) <= 0 || tor_isinf(expected)) {
+    if (actual <= expected && actual >= expected)
+      return 0;
+    else
+      return 1;
+  } else {
+    return fabs((expected - actual)/expected);
+  }
+}
+
+/** Check that relative error of <b>expected</b> and <b>actual</b> is within
+ *  <b>relerr_bound</b>.  Caller must arrange to have i and relerr_bound in
+ *  scope.  */
+#define CHECK_RELERR(expected, actual) do {                                   \
+  double check_expected = (expected);                                         \
+  double check_actual = (actual);                                             \
+  const char *str_expected = #expected;                                       \
+  const char *str_actual = #actual;                                           \
+  double check_relerr = relerr(expected, actual);                             \
+  if (!(relerr(check_expected, check_actual) <= relerr_bound)) {              \
+    log_warn(LD_GENERAL, "%s:%d: case %u: relerr(%s=%.17e, %s=%.17e)"        \
+             " = %.17e > %.17e\n",                                            \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_expected, check_expected,                                    \
+             str_actual, check_actual,                                        \
+             check_relerr, relerr_bound);                                     \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/* Check that a <= b.
+ * Caller must arrange to have i in scope.  */
+#define CHECK_LE(a, b) do {                                                   \
+  double check_a = (a);                                                       \
+  double check_b = (b);                                                       \
+  const char *str_a = #a;                                                     \
+  const char *str_b = #b;                                                     \
+  if (!(check_a <= check_b)) {                                                \
+    log_warn(LD_GENERAL, "%s:%d: case %u: %s=%.17e > %s=%.17e\n",             \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_a, check_a, str_b, check_b);                                 \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/**
+ * Test the logit and logistic functions.  Confirm that they agree with
+ * the cdf, sf, icdf, and isf of the standard Logistic distribution.
+ * Confirm that the sampler for the standard logistic distribution maps
+ * [0, 1] into the right subinterval for the inverse transform, for
+ * this implementation.
+ */
+static void
+test_logit_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    double x;                   /* x = logit(p) */
+    double p;                   /* p = logistic(x) */
+    double phalf;               /* p - 1/2 = logistic(x) - 1/2 */
+  } cases[] = {
+    { -HUGE_VAL, 0, -0.5 },
+    { -1000, 0, -0.5 },
+    { -710, 4.47628622567513e-309, -0.5 },
+    { -708, 3.307553003638408e-308, -0.5 },
+    { -2, .11920292202211755, -.3807970779778824 },
+    { -1.0000001, .2689414017088022, -.23105859829119776 },
+    { -1, .2689414213699951, -.23105857863000487 },
+    { -0.9999999, .26894144103118883, -.2310585589688111 },
+    /* see src/test/prob_distr_mpfr_ref.c for computation */
+    { -4.000000000537333e-5, .49999, -1.0000000000010001e-5 },
+    { -4.000000000533334e-5, .49999, -.00001 },
+    { -4.000000108916878e-9, .499999999, -1.0000000272292198e-9 },
+    { -4e-9, .499999999, -1e-9 },
+    { -4e-16, .5, -1e-16 },
+    { -4e-300, .5, -1e-300 },
+    { 0, .5, 0 },
+    { 4e-300, .5, 1e-300 },
+    { 4e-16, .5, 1e-16 },
+    { 3.999999886872274e-9, .500000001, 9.999999717180685e-10 },
+    { 4e-9, .500000001, 1e-9 },
+    { 4.0000000005333336e-5, .50001, .00001 },
+    { 8.000042667076272e-3, .502, .002 },
+    { 0.9999999, .7310585589688111, .2310585589688111 },
+    { 1, .7310585786300049, .23105857863000487 },
+    { 1.0000001, .7310585982911977, .23105859829119774 },
+    { 2, .8807970779778823, .3807970779778824 },
+    { 708, 1, .5 },
+    { 710, 1, .5 },
+    { 1000, 1, .5 },
+    { HUGE_VAL, 1, .5 },
+  };
+  double relerr_bound = 3e-15; /* >10eps */
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double phalf = cases[i].phalf;
+
+    /*
+     * cdf is logistic, icdf is logit, and symmetry for
+     * sf/isf.
+     */
+    CHECK_RELERR(logistic(x), cdf_logistic(x, 0, 1));
+    CHECK_RELERR(logistic(-x), sf_logistic(x, 0, 1));
+    CHECK_RELERR(logit(p), icdf_logistic(p, 0, 1));
+    CHECK_RELERR(-logit(p), isf_logistic(p, 0, 1));
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2, 0, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2, 0, 2));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0, 2)/2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, 2)/2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x/2, 0, .5));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x/2, 0, .5));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0,.5)*2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, .5)*2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2 + 1, 1, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2 + 1, 1, 2));
+
+    /*
+     * For p near 0 and p near 1/2, the arithmetic of
+     * translating by 1 loses precision.
+     */
+    if (fabs(p) > DBL_EPSILON && fabs(p) < 0.4) {
+      CHECK_RELERR(icdf_logistic(p, 0, 1),
+          (icdf_logistic(p, 1, 2) - 1)/2);
+      CHECK_RELERR(isf_logistic(p, 0, 1),
+          (isf_logistic(p, 1, 2) - 1)/2);
+    }
+
+    CHECK_RELERR(p, logistic(x));
+    CHECK_RELERR(phalf, logistichalf(x));
+
+    /*
+     * On the interior floating-point numbers, either logit or
+     * logithalf had better give the correct answer.
+     *
+     * For probabilities near 0, we can get much finer resolution with
+     * logit, and for probabilities near 1/2, we can get much finer
+     * resolution with logithalf by representing them using p - 1/2.
+     *
+     * E.g., we can write -.00001 for phalf, and .49999 for p, but the
+     * difference 1/2 - .00001 gives 1.0000000000010001e-5 in binary64
+     * arithmetic.  So test logit(.49999) which should give the same
+     * answer as logithalf(-1.0000000000010001e-5), namely
+     * -4.000000000537333e-5, and also test logithalf(-.00001) which
+     * gives -4.000000000533334e-5 instead -- but don't expect
+     * logit(.49999) to give -4.000000000533334e-5 even though it looks
+     * like 1/2 - .00001.
+     *
+     * A naive implementation of logit will just use log(p/(1 - p)) and
+     * give the answer -4.000000000551673e-05 for .49999, which is
+     * wrong in a lot of digits, which happens because log is
+     * ill-conditioned near 1 and thus amplifies whatever relative
+     * error we made in computing p/(1 - p).
+     */
+    if ((0 < p && p < 1) || tor_isinf(x)) {
+      if (phalf >= p - 0.5 && phalf <= p - 0.5)
+        CHECK_RELERR(x, logit(p));
+      if (p >= 0.5 + phalf && p <= 0.5 + phalf)
+        CHECK_RELERR(x, logithalf(phalf));
+    }
+
+    CHECK_RELERR(-phalf, logistichalf(-x));
+    if (fabs(phalf) < 0.5 || tor_isinf(x))
+      CHECK_RELERR(-x, logithalf(-phalf));
+    if (p < 1 || tor_isinf(x)) {
+      CHECK_RELERR(1 - p, logistic(-x));
+      if (p > .75 || tor_isinf(x))
+        CHECK_RELERR(-x, logit(1 - p));
+    } else {
+      CHECK_LE(logistic(-x), 1e-300);
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(logit(p0/(1 + M_E)), sample_logistic(0, 0, p0));
+    CHECK_RELERR(-logit(p0/(1 + M_E)), sample_logistic(1, 0, p0));
+    CHECK_RELERR(logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(0, 1, p0));
+    CHECK_RELERR(-logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(1, 1, p0));
+  }
+
+  if (!ok)
+    printf("fail logit/logistic / logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the LogLogistic distribution.
+ */
+static void
+test_log_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the LogLogistic distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given LogLogistic
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given LogLogistic
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { 1e-15, 1e-15, .999999999999999 },
+    { .1, .09090909090909091, .90909090909090909 },
+    { .25, .2, .8 },
+    { .5, .33333333333333333, .66666666666666667 },
+    { .75, .42857142857142855, .5714285714285714 },
+    { .9999, .49997499874993756, .5000250012500626 },
+    { .99999999, .49999999749999996, .5000000025 },
+    { .999999999999999, .49999999999999994, .5000000000000002 },
+    { 1, .5, .5 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_log_logistic(x, 1, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(p, cdf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, sf_log_logistic(x, 1, 1));
+    CHECK_RELERR(np, sf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(np, sf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(np, sf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, cdf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(np, cdf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(np, cdf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(np, cdf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(np, cdf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(np, cdf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(p, sf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(p, sf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(p, sf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(p, sf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(p, sf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(p, sf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(x/2, icdf_log_logistic(p, .5, 1));
+    CHECK_RELERR(x*2, icdf_log_logistic(p, 2, 1));
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(sqrt(x)/2, icdf_log_logistic(p, .5, 2));
+    CHECK_RELERR(sqrt(x)*2, icdf_log_logistic(p, 2, 2));
+    CHECK_RELERR(sqrt(x), icdf_log_logistic(p, 1, 2));
+    CHECK_RELERR(x*x/2, icdf_log_logistic(p, .5, .5));
+    CHECK_RELERR(x*x*2, icdf_log_logistic(p, 2, .5));
+
+    if (np < .9) {
+      CHECK_RELERR(x, isf_log_logistic(np, 1, 1));
+      CHECK_RELERR(x/2, isf_log_logistic(np, .5, 1));
+      CHECK_RELERR(x*2, isf_log_logistic(np, 2, 1));
+      CHECK_RELERR(sqrt(x), isf_log_logistic(np, 1, 2));
+      CHECK_RELERR(sqrt(x)/2, isf_log_logistic(np, .5, 2));
+      CHECK_RELERR(sqrt(x)*2, isf_log_logistic(np, 2, 2));
+      CHECK_RELERR(x*x, isf_log_logistic(np, 1, .5));
+      CHECK_RELERR(x*x/2, isf_log_logistic(np, .5, .5));
+      CHECK_RELERR(x*x*2, isf_log_logistic(np, 2, .5));
+
+      CHECK_RELERR(1/x, icdf_log_logistic(np, 1, 1));
+      CHECK_RELERR(1/(2*x), icdf_log_logistic(np, .5, 1));
+      CHECK_RELERR(2/x, icdf_log_logistic(np, 2, 1));
+      CHECK_RELERR(1/sqrt(x), icdf_log_logistic(np, 1, 2));
+      CHECK_RELERR(1/(2*sqrt(x)),
+          icdf_log_logistic(np, .5, 2));
+      CHECK_RELERR(2/sqrt(x), icdf_log_logistic(np, 2, 2));
+      CHECK_RELERR(1/(x*x), icdf_log_logistic(np, 1, .5));
+      CHECK_RELERR(1/(2*x*x), icdf_log_logistic(np, .5, .5));
+      CHECK_RELERR(2/(x*x), icdf_log_logistic(np, 2, .5));
+    }
+
+    CHECK_RELERR(1/x, isf_log_logistic(p, 1, 1));
+    CHECK_RELERR(1/(2*x), isf_log_logistic(p, .5, 1));
+    CHECK_RELERR(2/x, isf_log_logistic(p, 2, 1));
+    CHECK_RELERR(1/sqrt(x), isf_log_logistic(p, 1, 2));
+    CHECK_RELERR(1/(2*sqrt(x)), isf_log_logistic(p, .5, 2));
+    CHECK_RELERR(2/sqrt(x), isf_log_logistic(p, 2, 2));
+    CHECK_RELERR(1/(x*x), isf_log_logistic(p, 1, .5));
+    CHECK_RELERR(1/(2*x*x), isf_log_logistic(p, .5, .5));
+    CHECK_RELERR(2/(x*x), isf_log_logistic(p, 2, .5));
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(0.5*p0/(1 - 0.5*p0), sample_log_logistic(0, p0));
+    CHECK_RELERR((1 - 0.5*p0)/(0.5*p0),
+        sample_log_logistic(1, p0));
+  }
+
+  if (!ok)
+    printf("fail log logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, isf of the Weibull distribution.
+ */
+static void
+test_weibull(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the Weibull distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given Weibull
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given Weibull
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { .1, .09516258196404043, .9048374180359595 },
+    { .5, .3934693402873666, .6065306597126334 },
+    { .6931471805599453, .5, .5 },
+    { 1, .6321205588285577, .36787944117144233 },
+    { 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 36, .9999999999999998, 2.319522830243569e-16 },
+    { 37, .9999999999999999, 8.533047625744066e-17 },
+    { 38, 1, 3.1391327920480296e-17 },
+    { 100, 1, 3.720075976020836e-44 },
+    { 708, 1, 3.307553003638408e-308 },
+    { 710, 1, 4.47628622567513e-309 },
+    { 1000, 1, 0 },
+    { HUGE_VAL, 1, 0 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_weibull(x, 1, 1));
+    CHECK_RELERR(p, cdf_weibull(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_weibull(x*2, 2, 1));
+    /* For 0 < x < sqrt(DBL_MIN), x^2 loses lots of bits.  */
+    if (x <= 0 ||
+        sqrt(DBL_MIN) <= x) {
+      CHECK_RELERR(p, cdf_weibull(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x*2, 2, .5));
+    }
+    CHECK_RELERR(p, cdf_weibull(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)*2, 2, 2));
+    CHECK_RELERR(np, sf_weibull(x, 1, 1));
+    CHECK_RELERR(np, sf_weibull(x/2, .5, 1));
+    CHECK_RELERR(np, sf_weibull(x*2, 2, 1));
+    CHECK_RELERR(np, sf_weibull(x*x, 1, .5));
+    CHECK_RELERR(np, sf_weibull(x*x/2, .5, .5));
+    CHECK_RELERR(np, sf_weibull(x*x*2, 2, .5));
+    if (x >= 10) {
+      /*
+       * exp amplifies the error of sqrt(x)^2
+       * proportionally to exp(x); for large inputs
+       * this is significant.
+       */
+      double t = -expm1(-x*(2*DBL_EPSILON + DBL_EPSILON));
+      relerr_bound = t + DBL_EPSILON + t*DBL_EPSILON;
+      if (relerr_bound < 3e-15)
+        /*
+         * The tests are written only to 16
+         * decimal places anyway even if your
+         * `double' is, say, i387 binary80, for
+         * whatever reason.
+         */
+        relerr_bound = 3e-15;
+      CHECK_RELERR(np, sf_weibull(sqrt(x), 1, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)/2, .5, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)*2, 2, 2));
+    }
+
+    if (p <= 0.75) {
+      /*
+       * For p near 1, not enough precision near 1 to
+       * recover x.
+       */
+      CHECK_RELERR(x, icdf_weibull(p, 1, 1));
+      CHECK_RELERR(x/2, icdf_weibull(p, .5, 1));
+      CHECK_RELERR(x*2, icdf_weibull(p, 2, 1));
+    }
+    if (p >= 0.25 && !tor_isinf(x) && np > 0) {
+      /*
+       * For p near 0, not enough precision in np
+       * near 1 to recover x.  For 0, isf gives inf,
+       * even if p is precise enough for the icdf to
+       * work.
+       */
+      CHECK_RELERR(x, isf_weibull(np, 1, 1));
+      CHECK_RELERR(x/2, isf_weibull(np, .5, 1));
+      CHECK_RELERR(x*2, isf_weibull(np, 2, 1));
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(3*sqrt(-log(p0/2)), sample_weibull(0, p0, 3, 2));
+    CHECK_RELERR(3*sqrt(-log1p(-p0/2)),
+        sample_weibull(1, p0, 3, 2));
+  }
+
+  if (!ok)
+    printf("fail Weibull cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the generalized Pareto
+ * distribution.
+ */
+static void
+test_genpareto(void *arg)
+{
+  (void) arg;
+
+  struct {
+    /* xi is the 'xi' parameter of the generalized Pareto distribution, and the
+     * rest are the same as in the above tests */
+    double xi, x, p, np;
+  } cases[] = {
+    { 0, 0, 0, 1 },
+    { 1e-300, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-300, .1, .09516258196404043, .9048374180359595 },
+    { 1e-300, 1, .6321205588285577, .36787944117144233 },
+    { 1e-300, 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 1e-200, 1e-16, 9.999999999999999e-17, .9999999999999999 },
+    { 1e-16, 1e-200, 9.999999999999998e-201, 1 },
+    { 1e-16, 1e-16, 1e-16, 1 },
+    { 1e-16, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-16, .1, .09516258196404043, .9048374180359595 },
+    { 1e-16, 1, .6321205588285577, .36787944117144233 },
+    { 1e-16, 10, .9999546000702375, 4.539992976248509e-5 },
+    { 1e-10, 1e-6, 9.999995000001667e-7, .9999990000005 },
+    { 1e-8, 1e-8, 9.999999950000001e-9, .9999999900000001 },
+    { 1, 1e-300, 1e-300, 1 },
+    { 1, 1e-16, 1e-16, .9999999999999999 },
+    { 1, .1, .09090909090909091, .9090909090909091 },
+    { 1, 1, .5, .5 },
+    { 1, 10, .9090909090909091, .0909090909090909 },
+    { 1, 100, .9900990099009901, .0099009900990099 },
+    { 1, 1000, .999000999000999, 9.990009990009992e-4 },
+    { 10, 1e-300, 1e-300, 1 },
+    { 10, 1e-16, 9.999999999999995e-17, .9999999999999999 },
+    { 10, .1, .06696700846319258, .9330329915368074 },
+    { 10, 1, .21320655780322778, .7867934421967723 },
+    { 10, 10, .3696701667040189, .6303298332959811 },
+    { 10, 100, .49886285755007337, .5011371424499267 },
+    { 10, 1000, .6018968102992647, .3981031897007353 },
+  };
+  double xi_array[] = { -1.5, -1, -1e-30, 0, 1e-30, 1, 1.5 };
+  size_t i, j;
+  double relerr_bound = 3e-15;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double xi = cases[i].xi;
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(p, cdf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(p, cdf_genpareto(x/2, 0, .5, xi));
+    CHECK_RELERR(np, sf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(np, sf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(np, sf_genpareto(x/2, 0, .5, xi));
+
+    if (p < .5) {
+      CHECK_RELERR(x, icdf_genpareto(p, 0, 1, xi));
+      CHECK_RELERR(x*2, icdf_genpareto(p, 0, 2, xi));
+      CHECK_RELERR(x/2, icdf_genpareto(p, 0, .5, xi));
+    }
+    if (np < .5) {
+      CHECK_RELERR(x, isf_genpareto(np, 0, 1, xi));
+      CHECK_RELERR(x*2, isf_genpareto(np, 0, 2, xi));
+      CHECK_RELERR(x/2, isf_genpareto(np, 0, .5, xi));
+    }
+  }
+
+  for (i = 0; i < arraycount(xi_array); i++) {
+    for (j = 0; j <= 100; j++) {
+      double p0 = (j == 0 ? 2*DBL_MIN : (double)j/100);
+
+      /* This is actually a check against 0, but we do <= so that the compiler
+         does not raise a -Wfloat-equal */
+      if (fabs(xi_array[i]) <= 0) {
+        /*
+         * When xi == 0, the generalized Pareto
+         * distribution reduces to an
+         * exponential distribution.
+         */
+        CHECK_RELERR(-log(p0/2),
+            sample_genpareto(0, p0, 0));
+        CHECK_RELERR(-log1p(-p0/2),
+            sample_genpareto(1, p0, 0));
+      } else {
+        CHECK_RELERR(expm1(-xi_array[i]*log(p0/2))/xi_array[i],
+            sample_genpareto(0, p0, xi_array[i]));
+        CHECK_RELERR((j == 0 ? DBL_MIN :
+                expm1(-xi_array[i]*log1p(-p0/2))/xi_array[i]),
+            sample_genpareto(1, p0, xi_array[i]));
+      }
+
+      CHECK_RELERR(isf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(0, p0, xi_array[i]));
+      CHECK_RELERR(icdf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(1, p0, xi_array[i]));
+    }
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the deterministic sampler for uniform distribution on [a, b].
+ *
+ * This currently only tests whether the outcome lies within [a, b].
+ */
+static void
+test_uniform_interval(void *arg)
+{
+  (void) arg;
+  struct {
+    /* Sample from a uniform distribution with parameters 'a' and 'b', using
+     * 't' as the sampling index. */
+    double t, a, b;
+  } cases[] = {
+    { 0, 0, 0 },
+    { 0, 0, 1 },
+    { 0, 1.0000000000000007, 3.999999999999995 },
+    { 0, 4000, 4000 },
+    { 0.42475836677491291, 4000, 4000 },
+    { 0, -DBL_MAX, DBL_MAX },
+    { 0.25, -DBL_MAX, DBL_MAX },
+    { 0.5, -DBL_MAX, DBL_MAX },
+  };
+  size_t i = 0;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double t = cases[i].t;
+    double a = cases[i].a;
+    double b = cases[i].b;
+
+    CHECK_LE(a, sample_uniform_interval(t, a, b));
+    CHECK_LE(sample_uniform_interval(t, a, b), b);
+
+    CHECK_LE(a, sample_uniform_interval(1 - t, a, b));
+    CHECK_LE(sample_uniform_interval(1 - t, a, b), b);
+
+    CHECK_LE(sample_uniform_interval(t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(t, -b, -a));
+
+    CHECK_LE(sample_uniform_interval(1 - t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(1 - t, -b, -a));
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/********************** Stochastic tests ****************************/
+
+/*
+ * Psi test, sometimes also called G-test.  The psi test statistic,
+ * suitably scaled, has chi^2 distribution, but the psi test tends to
+ * have better statistical power in practice to detect deviations than
+ * the chi^2 test does.  (The chi^2 test statistic is the first term of
+ * the Taylor expansion of the psi test statistic.)  The psi test is
+ * generic, for any CDF; particular distributions might have higher-
+ * power tests to distinguish them from predictable deviations or bugs.
+ *
+ * We choose the psi critical value so that a single psi test has
+ * probability below alpha = 1% of spuriously failing even if all the
+ * code is correct.  But the false positive rate for a suite of n tests
+ * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n.  For n = 10,
+ * this is about 10%, and for n = 100 it is well over 50%.
+ *
+ * Given that these tests will run with every CI job, we want to drive down the
+ * false positive rate. We can drive it down by running each test four times,
+ * and accepting it if it passes at least once; in that case, it is as if we
+ * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test,
+ * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14
+ * jobs, then the chance of a CI build failing is 1.39999903326e-06, which
+ * means that a CI build will break with probability 50% after about 495106
+ * builds.
+ *
+ * The critical value for a chi^2 distribution with 100 degrees of
+ * freedom and false positive rate alpha = 1% was taken from:
+ *
+ *  NIST/SEMATECH e-Handbook of Statistical Methods, Section
+ *  1.3.6.7.4 `Critical Values of the Chi-Square Distribution',
+ *  <http://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm>,
+ *  retrieved 2018-10-28.
+ */
+
+static const size_t NSAMPLES = 100000;
+/* Number of chances we give to the test to succeed. */
+static const unsigned NTRIALS = 4;
+/* Number of times we want the test to pass per NTRIALS. */
+static const unsigned NPASSES_MIN = 1;
+
+#define PSI_DF 100                          /* degrees of freedom */
+static const double PSI_CRITICAL = 135.807; /* critical value, alpha = .01 */
+
+/**
+ * Perform a psi test on an array of sample counts, C, adding up to N
+ * samples, and an array of log expected probabilities, logP,
+ * representing the null hypothesis for the distribution of samples
+ * counted.  Return false if the psi test rejects the null hypothesis,
+ * true if otherwise.
+ */
+static bool
+psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N)
+{
+  double psi = 0;
+  double c = 0;                 /* Kahan compensation */
+  double t, u;
+  size_t i;
+
+  for (i = 0; i < PSI_DF; i++) {
+    /*
+     * c*log(c/(n*p)) = (1/n) * f*log(f/p) where f = c/n is
+     * the frequency, and f*log(f/p) ---> 0 as f ---> 0, so
+     * this is a reasonable choice.  Further, any mass that
+     * _fails_ to turn up in this bin will inflate another
+     * bin instead, so we don't really lose anything by
+     * ignoring empty bins even if they have high
+     * probability.
+     */
+    if (C[i] == 0)
+      continue;
+    t = C[i]*(log((double)C[i]/N) - logP[i]) - c;
+    u = psi + t;
+    c = (u - psi) - t;
+    psi = u;
+  }
+  psi *= 2;
+
+  return psi <= PSI_CRITICAL;
+}
+
+static bool
+test_stochastic_geometric_impl(double p)
+{
+  const struct geometric geometric = {
+    .base = GEOMETRIC(geometric),
+    .p = p,
+  };
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  unsigned i;
+  size_t j;
+
+  /* Compute logP[i] = Geom(i + 1; p).  */
+  for (i = 0; i < PSI_DF - 1; i++)
+    logP[i] = logpmf_geometric(i + 1, p);
+
+  /* Compute logP[n-1] = log (1 - (P[0] + P[1] + ... + P[n-2])).  */
+  logP[PSI_DF - 1] = log1mexp(logsumexp(logP, PSI_DF - 1));
+
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+
+    for (j = 0; j < NSAMPLES; j++) {
+      double n_tmp = dist_sample(&geometric.base);
+
+      /* Must be an integer.  (XXX -Wfloat-equal)  */
+      tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp);
+
+      /* Must be a positive integer.  */
+      tor_assert(n_tmp >= 1);
+
+      /* Probability of getting a value in the billions is negligible.  */
+      tor_assert(n_tmp <= (double)UINT_MAX);
+
+      unsigned n = (unsigned) n_tmp;
+
+      if (n > PSI_DF)
+        n = PSI_DF;
+      C[n - 1]++;
+    }
+
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", "geometric"); */
+    return true;
+  } else {
+    printf("fail %s sampler\n", "geometric");
+    return false;
+  }
+}
+
+/**
+ * Divide the support of <b>dist</b> into histogram bins in <b>logP</b>. Start
+ * at the 1st percentile and ending at the 99th percentile. Pick the bin
+ * boundaries using linear interpolation so that they are uniformly spaced.
+ *
+ * In each bin logP[i] we insert the expected log-probability that a sampled
+ * value will fall into that bin. We will use this as the null hypothesis of
+ * the psi test.
+ *
+ * Set logP[i] = log(CDF(x_i) - CDF(x_{i-1})), where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_cdfs(const struct dist *dist, double lo, double hi, double *logP, size_t n)
+{
+#define CDF(x)  dist_cdf(dist, x)
+#define SF(x)   dist_sf(dist, x)
+  const double w = (hi - lo)/(n - 2);
+  double halfway = dist_icdf(dist, 0.5);
+  double x_0, x_1;
+  size_t i;
+  size_t n2 = ceil_to_size_t((halfway - lo)/w);
+
+  tor_assert(lo <= halfway);
+  tor_assert(halfway <= hi);
+  tor_assert(n2 <= n);
+
+  x_1 = lo;
+  logP[0] = log(CDF(x_1) - 0); /* 0 = CDF(-inf) */
+  for (i = 1; i < n2; i++) {
+    x_0 = x_1;
+    /* do the linear interpolation */
+    x_1 = (i <= n/2 ? lo + i*w : hi - (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[i] = log(CDF(x_1) - CDF(x_0));
+  }
+  x_0 = hi;
+  logP[n - 1] = log(SF(x_0) - 0); /* 0 = SF(+inf) = 1 - CDF(+inf) */
+
+  /* In this loop we are filling out the high part of the array. We are using
+   * SF because in these cases the CDF is near 1 where precision is lower. So
+   * instead we are using SF near 0 where the precision is higher. We have
+   * SF(t) = 1 - CDF(t).  */
+  for (i = 1; i < n - n2; i++) {
+    x_1 = x_0;
+    /* do the linear interpolation */
+    x_0 = (i <= n/2 ? hi - i*w : lo + (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[n - i - 1] = log(SF(x_0) - SF(x_1));
+  }
+#undef SF
+#undef CDF
+}
+
+/**
+ * Draw NSAMPLES samples from dist, counting the number of samples x in
+ * the ith bin C[i] if x_{i-1} <= x < x_i, where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_samples(const struct dist *dist, double lo, double hi, size_t *C, size_t n)
+{
+  const double w = (hi - lo)/(n - 2);
+  size_t i;
+
+  for (i = 0; i < NSAMPLES; i++) {
+    double x = dist_sample(dist);
+    size_t bin;
+
+    if (x < lo)
+      bin = 0;
+    else if (x < hi)
+      bin = 1 + floor_to_size_t((x - lo)/w);
+    else
+      bin = n - 1;
+    tor_assert(bin < n);
+    C[bin]++;
+  }
+}
+
+/**
+ * Carry out a Psi test on <b>dist</b>.
+ *
+ * Sample NSAMPLES from dist, putting them in bins from -inf to lo to
+ * hi to +inf, and apply up to two psi tests.  True if at least one psi
+ * test passes; false if not.  False positive rate should be bounded by
+ * 0.01^2 = 0.0001.
+ */
+static bool
+test_psi_dist_sample(const struct dist *dist)
+{
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  double lo = dist_icdf(dist, 1/(double)(PSI_DF + 2));
+  double hi = dist_isf(dist, 1/(double)(PSI_DF + 2));
+
+  /* Create the null hypothesis in logP */
+  bin_cdfs(dist, lo, hi, logP, PSI_DF);
+
+  /* Now run the test */
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+    bin_samples(dist, lo, hi, C, PSI_DF);
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  /* Did we fail or succeed? */
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", dist_name(dist));*/
+    return true;
+  } else {
+    printf("fail %s sampler\n", dist_name(dist));
+    return false;
+  }
+}
+
+static void
+write_stochastic_warning(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    printf("\n"
+         "NOTE: This is a stochastic test, and we expect it to fail from\n"
+         "time to time, with some low probability. If you see it fail more\n"
+         "than one trial in 100, though, please tell us.\n\n");
+  }
+}
+
+static void
+test_stochastic_uniform(void *arg)
+{
+  (void) arg;
+
+  const struct uniform uniform01 = {
+    .base = UNIFORM(uniform01),
+    .a = 0,
+    .b = 1,
+  };
+  const struct uniform uniform_pos = {
+    .base = UNIFORM(uniform_pos),
+    .a = 1.23,
+    .b = 4.56,
+  };
+  const struct uniform uniform_neg = {
+    .base = UNIFORM(uniform_neg),
+    .a = -10,
+    .b = -1,
+  };
+  const struct uniform uniform_cross = {
+    .base = UNIFORM(uniform_cross),
+    .a = -1.23,
+    .b = 4.56,
+  };
+  const struct uniform uniform_subnormal = {
+    .base = UNIFORM(uniform_subnormal),
+    .a = 4e-324,
+    .b = 4e-310,
+  };
+  const struct uniform uniform_subnormal_cross = {
+    .base = UNIFORM(uniform_subnormal_cross),
+    .a = -4e-324,
+    .b = 4e-310,
+  };
+  bool ok = true, tests_failed = true;
+
+  testing_enable_reproducible_rng();
+
+  ok &= test_psi_dist_sample(&uniform01.base);
+  ok &= test_psi_dist_sample(&uniform_pos.base);
+  ok &= test_psi_dist_sample(&uniform_neg.base);
+  ok &= test_psi_dist_sample(&uniform_cross.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal_cross.base);
+
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static bool
+test_stochastic_logistic_impl(double mu, double sigma)
+{
+  const struct logistic dist = {
+    .base = LOGISTIC(dist),
+    .mu = mu,
+    .sigma = sigma,
+  };
+
+  /* XXX Consider some fancier logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_log_logistic_impl(double alpha, double beta)
+{
+  const struct log_logistic dist = {
+    .base = LOG_LOGISTIC(dist),
+    .alpha = alpha,
+    .beta = beta,
+  };
+
+  /* XXX Consider some fancier log logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_weibull_impl(double lambda, double k)
+{
+  const struct weibull dist = {
+    .base = WEIBULL(dist),
+    .lambda = lambda,
+    .k = k,
+  };
+
+/*
+ * XXX Consider applying a Tiku-Singh test:
+ *
+ *    M.L. Tiku and M. Singh, `Testing the two-parameter
+ *    Weibull distribution', Communications in Statistics --
+ *    Theory and Methods A10(9), 1981, 907--918.
+ *https://www.tandfonline.com/doi/pdf/10.1080/03610928108828082?needAccess=true
+ */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_genpareto_impl(double mu, double sigma, double xi)
+{
+  const struct genpareto dist = {
+    .base = GENPARETO(dist),
+    .mu = mu,
+    .sigma = sigma,
+    .xi = xi,
+  };
+
+  /* XXX Consider some fancier GPD test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static void
+test_stochastic_genpareto(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_genpareto_impl(0, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, -1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(-1, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(1, 2, 0.25);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_geometric(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_geometric_impl(0.1);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.5);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.9);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(1);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_logistic(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_logistic_impl(0, 1);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(0, 1e-16);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(-10, 100);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_log_logistic(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_log_logistic_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(M_E, 1e-1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_weibull(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_weibull_impl(1, 0.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 2);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(10, 1);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+  UNMOCK(crypto_rand);
+}
+
+struct testcase_t prob_distr_tests[] = {
+  { "logit_logistics", test_logit_logistic, TT_FORK, NULL, NULL },
+  { "log_logistic", test_log_logistic, TT_FORK, NULL, NULL },
+  { "weibull", test_weibull, TT_FORK, NULL, NULL },
+  { "genpareto", test_genpareto, TT_FORK, NULL, NULL },
+  { "uniform_interval", test_uniform_interval, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
+
+struct testcase_t slow_stochastic_prob_distr_tests[] = {
+  { "stochastic_genpareto", test_stochastic_genpareto, TT_FORK, NULL, NULL },
+  { "stochastic_geometric", test_stochastic_geometric, TT_FORK, NULL, NULL },
+  { "stochastic_uniform", test_stochastic_uniform, TT_FORK, NULL, NULL },
+  { "stochastic_logistic", test_stochastic_logistic, TT_FORK, NULL, NULL },
+  { "stochastic_log_logistic", test_stochastic_log_logistic, TT_FORK, NULL,
+    NULL },
+  { "stochastic_weibull", test_stochastic_weibull, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process.c b/src/test/test_process.c
new file mode 100644
index 0000000000..7836312761
--- /dev/null
+++ b/src/test/test_process.c
@@ -0,0 +1,669 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process.c
+ * \brief Test cases for the Process API.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/process/env.h"
+
+#define PROCESS_PRIVATE
+#include "lib/process/process.h"
+#define PROCESS_UNIX_PRIVATE
+#include "lib/process/process_unix.h"
+#define PROCESS_WIN32_PRIVATE
+#include "lib/process/process_win32.h"
+
+static const char *stdout_read_buffer;
+static const char *stderr_read_buffer;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static int
+process_mocked_read_stdout(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stdout_read_buffer != NULL) {
+    buf_add_string(buffer, stdout_read_buffer);
+    stdout_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static int
+process_mocked_read_stderr(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stderr_read_buffer != NULL) {
+    buf_add_string(buffer, stderr_read_buffer);
+    stderr_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static void
+process_mocked_write_stdin(process_t *process, buf_t *buffer)
+{
+  const size_t size = buf_datalen(buffer);
+
+  if (size == 0)
+    return;
+
+  char *data = tor_malloc_zero(size + 1);
+  process_data_t *process_data = process_get_data(process);
+
+  buf_get_bytes(buffer, data, size);
+  smartlist_add(process_data->stdin_data, data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+static void
+test_default_values(void *arg)
+{
+  (void)arg;
+  process_t *process = process_new("/path/to/nothing");
+
+  /* We are not running by default. */
+  tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process));
+
+  /* We use the line protocol by default. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  /* We don't set any custom data by default. */
+  tt_ptr_op(NULL, OP_EQ, process_get_data(process));
+
+  /* Our command was given to the process_t's constructor in process_new(). */
+  tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process));
+
+  /* Make sure we are listed in the list of proccesses. */
+  tt_assert(smartlist_contains(process_get_all_processes(),
+                               process));
+
+  /* Default PID is 0. */
+  tt_u64_op(0, OP_EQ, process_get_pid(process));
+
+  /* Our arguments should be empty. */
+  tt_int_op(0, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+ done:
+  process_free(process);
+}
+
+static void
+test_environment(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("");
+  process_environment_t *env = NULL;
+
+  process_set_environment(process, "E", "F");
+  process_set_environment(process, "C", "D");
+  process_set_environment(process, "A", "B");
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "A=B\0C=D\0E=F\0", 12);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "A=B");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "C=D");
+  tt_str_op(env->unixoid_environment_block[2], OP_EQ,
+            "E=F");
+  tt_ptr_op(env->unixoid_environment_block[3], OP_EQ,
+            NULL);
+  process_environment_free(env);
+
+  /* Reset our environment. */
+  smartlist_t *new_env = smartlist_new();
+  smartlist_add(new_env, (char *)"FOO=bar");
+  smartlist_add(new_env, (char *)"HELLO=world");
+
+  process_reset_environment(process, new_env);
+  smartlist_free(new_env);
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "FOO=bar\0HELLO=world\0", 20);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "FOO=bar");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "HELLO=world");
+  tt_ptr_op(env->unixoid_environment_block[2], OP_EQ,
+            NULL);
+
+ done:
+  process_environment_free(env);
+  process_free(process);
+}
+
+static void
+test_stringified_types(void *arg)
+{
+  (void)arg;
+
+  /* process_protocol_t values. */
+  tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW));
+  tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE));
+
+  /* process_status_t values. */
+  tt_str_op("not running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_NOT_RUNNING));
+  tt_str_op("running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_RUNNING));
+  tt_str_op("error", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_ERROR));
+
+ done:
+  return;
+}
+
+static void
+test_line_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\r\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_multi(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Onion Onion Onion");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "A B C D");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Foo bar baz");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Onion Onion Onion");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_partial(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout this is a partial line ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr this is a partial line ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should NOT be ready. */
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+                          "Hello stdout this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+                          "Hello stderr this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_raw_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_protocol(process, PROCESS_PROTOCOL_RAW);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the raw protocol. */
+  tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout\n");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Hello, again, stdout\nThis contains multiple lines");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr\n");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Hello, again, stderr\nThis contains multiple lines");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_write_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  MOCK(process_write_stdin, process_mocked_write_stdin);
+
+  process_write(process, (uint8_t *)"Hello world\n", 12);
+  process_notify_event_stdin(process);
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data));
+
+  process_printf(process, "Hello %s !\n", "moon");
+  process_notify_event_stdin(process);
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data));
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_write_stdin);
+}
+
+static void
+test_exit_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Our default is 0. */
+  tt_u64_op(0, OP_EQ, process_data->exit_code);
+
+  /* Fake that we are a running process. */
+  process_set_status(process, PROCESS_STATUS_RUNNING);
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Fake an exit. */
+  process_notify_event_exit(process, 1337);
+
+  /* Check if our state changed and if our callback fired. */
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+  tt_u64_op(1337, OP_EQ, process_data->exit_code);
+
+ done:
+  process_set_data(process, process_data);
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_argv_simple(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("/bin/cat");
+  char **argv = NULL;
+
+  /* Setup some arguments. */
+  process_append_argument(process, "foo");
+  process_append_argument(process, "bar");
+  process_append_argument(process, "baz");
+
+  /* Check the number of elements. */
+  tt_int_op(3, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+  /* Let's try to convert it into a Unix style char **argv. */
+  argv = process_get_argv(process);
+
+  /* Check our values. */
+  tt_str_op(argv[0], OP_EQ, "/bin/cat");
+  tt_str_op(argv[1], OP_EQ, "foo");
+  tt_str_op(argv[2], OP_EQ, "bar");
+  tt_str_op(argv[3], OP_EQ, "baz");
+  tt_ptr_op(argv[4], OP_EQ, NULL);
+
+ done:
+  tor_free(argv);
+  process_free(process);
+}
+
+static void
+test_unix(void *arg)
+{
+  (void)arg;
+#ifndef _WIN32
+  process_t *process = process_new("");
+
+  /* On Unix all processes should have a Unix process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_unix_process(process));
+
+ done:
+  process_free(process);
+#endif /* !defined(_WIN32) */
+}
+
+static void
+test_win32(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  process_t *process = process_new("");
+  char *joined_argv = NULL;
+
+  /* On Win32 all processes should have a Win32 process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_win32_process(process));
+
+  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
+   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
+   * will try to only generate simple cases for the child process to parse;
+   * i.e. we never embed quoted strings in arguments. */
+
+  const char *argvs[][4] = {
+    {"a", "bb", "CCC", NULL}, // Normal
+    {NULL, NULL, NULL, NULL}, // Empty argument list
+    {"", NULL, NULL, NULL}, // Empty argument
+    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
+    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
+    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
+    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
+    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
+    { NULL } // Terminator
+  };
+
+  const char *cmdlines[] = {
+    "a bb CCC",
+    "",
+    "\"\"",
+    "\\\"a b\\\"b CCC\\\"",
+    "\"a\tbc\" \"dd  dd\" E",
+    "a\\\\\\b \"de fg\" H",
+    "a\\\\\\\"b \\c D\\",
+    "\"a\\\\b c\" d E",
+    NULL // Terminator
+  };
+
+  int i;
+
+  for (i=0; cmdlines[i]!=NULL; i++) {
+    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
+    joined_argv = tor_join_win_cmdline(argvs[i]);
+    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
+    tor_free(joined_argv);
+  }
+
+ done:
+  tor_free(joined_argv);
+  process_free(process);
+#endif /* defined(_WIN32) */
+}
+
+struct testcase_t process_tests[] = {
+  { "default_values", test_default_values, TT_FORK, NULL, NULL },
+  { "environment", test_environment, TT_FORK, NULL, NULL },
+  { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL },
+  { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL },
+  { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL },
+  { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL },
+  { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL },
+  { "write_simple", test_write_simple, TT_FORK, NULL, NULL },
+  { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL },
+  { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL },
+  { "unix", test_unix, TT_FORK, NULL, NULL },
+  { "win32", test_win32, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c
new file mode 100644
index 0000000000..7dc9abde31
--- /dev/null
+++ b/src/test/test_process_descs.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "test/test.h"
+
+static void
+test_process_descs_versions(void *arg)
+{
+  (void)arg;
+  struct {
+    const char *version;
+    bool should_reject;
+  } cases[] = {
+    // a very old version: reject.
+    { "Tor 0.1.2.3-alpha", true },
+    // a non-tor program: don't reject.
+    { "Wombat 0.1.2.3-alpha", false },
+    // a slightly old version: reject
+    { "Tor 0.2.9.4-alpha", true },
+    // a slightly old version: just new enough to support.
+    { "Tor 0.2.9.5-alpha", false },
+    // a newer 0.2.9 version: supported.
+    { "Tor 0.2.9.100", false },
+    // some unsupported versions: reject.
+    { "Tor 0.3.0.0-alpha-dev", true },
+    { "Tor 0.3.0.2-alpha", true },
+    { "Tor 0.3.0.5", true },
+    { "Tor 0.3.1.4", true },
+    { "Tor 0.3.2.4", true },
+    { "Tor 0.3.3.4", true },
+    { "Tor 0.3.4.1-alpha", true },
+    { "Tor 0.3.4.100", true },
+    { "Tor 0.3.5.1-alpha", true },
+    { "Tor 0.3.5.6-rc", true},
+    // new enough to be supported
+    { "Tor 0.3.5.7", false },
+    { "Tor 0.3.5.8", false },
+    { "Tor 0.4.0.1-alpha", false },
+    { "Tor 0.4.1.5", false },
+    // Very far in the future
+    { "Tor 100.100.1.5", false },
+  };
+  size_t n_cases = ARRAY_LENGTH(cases);
+
+  for (unsigned i = 0; i < n_cases; ++i) {
+    const char *msg = NULL;
+    bool rejected = dirserv_rejects_tor_version(cases[i].version, &msg);
+    tt_int_op(rejected, OP_EQ, cases[i].should_reject);
+    tt_int_op(msg == NULL, OP_EQ, rejected == false);
+  }
+
+ done:
+  ;
+}
+
+#define T(name,flags)                                   \
+  { #name, test_process_descs_##name, (flags), NULL, NULL }
+
+struct testcase_t process_descs_tests[] = {
+  T(versions,0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c
new file mode 100644
index 0000000000..f311e8b293
--- /dev/null
+++ b/src/test/test_process_slow.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process_slow.c
+ * \brief Slow test cases for the Process API.
+ */
+
+#define MAINLOOP_PRIVATE
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/process/process.h"
+#include "lib/process/waitpid.h"
+#include "test/test.h"
+
+#ifndef BUILDDIR
+#define BUILDDIR "."
+#endif
+
+#ifdef _WIN32
+#define TEST_PROCESS "test-process.exe"
+#else
+#define TEST_PROCESS BUILDDIR "/src/test/test-process"
+#endif /* defined(_WIN32) */
+
+/** Timer that ticks once a second and stop the event loop after 5 ticks. */
+static periodic_timer_t *main_loop_timeout_timer;
+
+/** How many times have our timer ticked? */
+static int timer_tick_count;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+  bool did_exit;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  process_status_t status;
+
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+  process_data->did_exit = true;
+
+  /* Check if our process is still running? */
+  status = process_get_status(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+#ifdef _WIN32
+static const char *
+get_win32_test_binary_path(void)
+{
+  static char buffer[MAX_PATH];
+
+  /* Get the absolute path of our binary: \path\to\test-slow.exe. */
+  GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer));
+
+  /* Find our process name. */
+  char *offset = strstr(buffer, "test-slow.exe");
+  tt_ptr_op(offset, OP_NE, NULL);
+
+  /* Change test-slow.exe to test-process.exe. */
+  memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS));
+
+  return buffer;
+ done:
+  return NULL;
+}
+#endif /* defined(_WIN32) */
+
+static void
+main_loop_timeout_cb(periodic_timer_t *timer, void *data)
+{
+  /* Sanity check. */
+  tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer);
+  tt_ptr_op(data, OP_NE, NULL);
+
+  /* Our process data. */
+  process_data_t *process_data = data;
+
+  /* Our process did exit. */
+  if (process_data->did_exit)
+    tor_shutdown_event_loop_and_exit(0);
+
+  /* Have we been called 10 times we exit the main loop. */
+  timer_tick_count++;
+
+  tt_int_op(timer_tick_count, OP_LT, 10);
+
+#ifndef _WIN32
+  /* Call waitpid callbacks. */
+  notify_pending_waitpid_callbacks();
+#endif
+
+  return;
+ done:
+  /* Exit with an error. */
+  tor_shutdown_event_loop_and_exit(-1);
+}
+
+static void
+run_main_loop(process_data_t *process_data)
+{
+  int ret;
+
+  /* Wake up after 1 seconds. */
+  static const struct timeval interval = {1, 0};
+
+  timer_tick_count = 0;
+  main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(),
+                                               &interval,
+                                               main_loop_timeout_cb,
+                                               process_data);
+
+  /* Run our main loop. */
+  ret = run_main_loop_until_done();
+
+  /* Clean up our main loop timeout timer. */
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  periodic_timer_free(main_loop_timeout_timer);
+}
+
+static void
+test_callbacks(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Set environment variable. */
+  process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!");
+
+  /* Add some arguments. */
+  process_append_argument(process, "This is the first one");
+  process_append_argument(process, "Second one");
+  process_append_argument(process, "Third: Foo bar baz");
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Write some lines to stdin. */
+  process_printf(process, "Hi process!\r\n");
+  process_printf(process, "Can you read more than one line?\n");
+  process_printf(process, "Can you read partial ...");
+  process_printf(process, " lines?\r\n");
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* We returned. Let's see what our event loop said. */
+  tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12);
+  tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3);
+  tt_assert(process_data->did_exit);
+  tt_u64_op(process_data->exit_code, OP_EQ, 0);
+
+  /* Check stdout output. */
+  char argv0_expected[256];
+  tor_snprintf(argv0_expected, sizeof(argv0_expected),
+               "argv[0] = '%s'", filename);
+
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            argv0_expected);
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "argv[1] = 'This is the first one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "argv[2] = 'Second one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "argv[3] = 'Third: Foo bar baz'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ,
+            "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ,
+            "Output on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ,
+            "Partial line on stdout ...end of partial line on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ,
+            "Read line from stdin: 'Hi process!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ,
+            "Read line from stdin: 'Can you read more than one line?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ,
+            "Read line from stdin: 'Can you read partial ... lines?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ,
+            "We are done for here, thank you!");
+
+  /* Check stderr output. */
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Output on stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Partial line on stderr ...end of partial line on stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_callbacks_terminate(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Zap our process. */
+  bool success;
+
+  success = process_terminate(process);
+  tt_assert(success);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Check if we did exit. */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_nonexistent_executable(void *arg)
+{
+  (void)arg;
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new("binary-does-not-exist");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_exec(process);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Ensure that the exit callback was actually called even though the binary
+   * did not exist.
+   */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+struct testcase_t slow_process_tests[] = {
+  { "callbacks", test_callbacks, 0, NULL, NULL },
+  { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL },
+  { "nonexistent_executable", test_nonexistent_executable, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c
index 08990c0b6a..f9339e8dd3 100644
--- a/src/test/test_proto_http.c
+++ b/src/test/test_proto_http.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 #include "test/log_test_helpers.h"
 
diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c
index af9cf7eee2..18669a7772 100644
--- a/src/test/test_proto_misc.c
+++ b/src/test/test_proto_misc.c
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_cell.h"
diff --git a/src/test/test_protover.c b/src/test/test_protover.c
index 63c508bd13..0254501165 100644
--- a/src/test/test_protover.c
+++ b/src/test/test_protover.c
@@ -22,7 +22,7 @@ test_protover_parse(void *arg)
   tt_skip();
  done:
   ;
-#else
+#else /* !defined(HAVE_RUST) */
   char *re_encoded = NULL;
 
   const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16,900";
@@ -89,7 +89,7 @@ test_protover_parse(void *arg)
     SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(elts);
   tor_free(re_encoded);
-#endif
+#endif /* defined(HAVE_RUST) */
 }
 
 static void
@@ -133,7 +133,7 @@ test_protover_parse_fail(void *arg)
                            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
   tt_ptr_op(elts, OP_EQ, NULL);
 
-#endif
+#endif /* defined(HAVE_RUST) */
  done:
   ;
 }
@@ -335,7 +335,7 @@ test_protover_all_supported(void *arg)
                  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
                  "aaaaaaaaaaaa=1-65536", &msg));
   tor_end_capture_bugs_();
-#endif
+#endif /* !defined(HAVE_RUST) */
 
  done:
   tor_end_capture_bugs_();
@@ -459,7 +459,7 @@ test_protover_supported_protocols(void *arg)
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
                                             PROTOVER_LINKAUTH_V1));
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   /* Latest LinkAuth is not exposed in the headers. */
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
diff --git a/src/test/test_pt.c b/src/test/test_pt.c
index 1f9786648a..8f3ce03c42 100644
--- a/src/test/test_pt.c
+++ b/src/test/test_pt.c
@@ -7,22 +7,25 @@
 #define PT_PRIVATE
 #define UTIL_PRIVATE
 #define STATEFILE_PRIVATE
-#define CONTROL_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define PROCESS_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confparse.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/transports.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/statefile.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/process/process.h"
 
 #include "app/config/or_state_st.h"
 
+#include "test/log_test_helpers.h"
+
 static void
 reset_mp(managed_proxy_t *mp)
 {
@@ -288,41 +291,35 @@ test_pt_get_extrainfo_string(void *arg)
   tor_free(s);
 }
 
-#ifdef _WIN32
-#define STDIN_HANDLE HANDLE*
-#else
-#define STDIN_HANDLE int
-#endif
-
-static smartlist_t *
-tor_get_lines_from_handle_replacement(STDIN_HANDLE handle,
-                                      enum stream_status *stream_status_out)
+static int
+process_read_stdout_replacement(process_t *process, buf_t *buffer)
 {
+  (void)process;
   static int times_called = 0;
-  smartlist_t *retval_sl = smartlist_new();
-
-  (void) handle;
-  (void) stream_status_out;
 
   /* Generate some dummy CMETHOD lines the first 5 times. The 6th
      time, send 'CMETHODS DONE' to finish configuring the proxy. */
-  if (times_called++ != 5) {
-    smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d",
+  times_called++;
+
+  if (times_called <= 5) {
+    buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n",
                            times_called, times_called);
-  } else {
-    smartlist_add_strdup(retval_sl, "SMETHODS DONE");
+  } else if (times_called <= 6) {
+    buf_add_string(buffer, "SMETHODS DONE\n");
+  } else if (times_called <= 7) {
+    buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something "
+                           "bad happened. What do we do!?\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n");
+  } else if (times_called <= 8) {
+    buf_add_string(buffer, "STATUS TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\n");
   }
 
-  return retval_sl;
-}
-
-/* NOP mock */
-static void
-tor_process_handle_destroy_replacement(process_handle_t *process_handle,
-                                       int also_terminate_process)
-{
-  (void) process_handle;
-  (void) also_terminate_process;
+  return (int)buf_datalen(buffer);
 }
 
 static or_state_t *dummy_state = NULL;
@@ -355,12 +352,9 @@ test_pt_configure_proxy(void *arg)
   managed_proxy_t *mp = NULL;
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
-  MOCK(tor_get_lines_from_handle,
-       tor_get_lines_from_handle_replacement);
-  MOCK(tor_process_handle_destroy,
-       tor_process_handle_destroy_replacement);
+  MOCK(process_read_stdout, process_read_stdout_replacement);
   MOCK(get_or_state,
        get_or_state_replacement);
   MOCK(queue_control_event_string,
@@ -372,24 +366,34 @@ test_pt_configure_proxy(void *arg)
   mp->conf_state = PT_PROTO_ACCEPTING_METHODS;
   mp->transports = smartlist_new();
   mp->transports_to_launch = smartlist_new();
-  mp->process_handle = tor_malloc_zero(sizeof(process_handle_t));
   mp->argv = tor_malloc_zero(sizeof(char*)*2);
   mp->argv[0] = tor_strdup("<testcase>");
   mp->is_server = 1;
 
+  /* Configure the process. */
+  mp->process = process_new("");
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_data(mp->process, mp);
+
   /* Test the return value of configure_proxy() by calling it some
      times while it is uninitialized and then finally finalizing its
      configuration. */
   for (i = 0 ; i < 5 ; i++) {
+    /* force a read from our mocked stdout reader. */
+    process_notify_event_stdout(mp->process);
+    /* try to configure our proxy. */
     retval = configure_proxy(mp);
     /* retval should be zero because proxy hasn't finished configuring yet */
     tt_int_op(retval, OP_EQ, 0);
     /* check the number of registered transports */
-    tt_assert(smartlist_len(mp->transports) == i+1);
+    tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1);
     /* check that the mp is still waiting for transports */
     tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS);
   }
 
+  /* Get the SMETHOD DONE written to the process. */
+  process_notify_event_stdout(mp->process);
+
   /* this last configure_proxy() should finalize the proxy configuration. */
   retval = configure_proxy(mp);
   /* retval should be 1 since the proxy finished configuring */
@@ -412,6 +416,49 @@ test_pt_configure_proxy(void *arg)
   tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ,
             "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n");
 
+  /* Get the log message out. */
+  setup_full_capture_of_logs(LOG_ERR);
+  process_notify_event_stdout(mp->process);
+  expect_single_log_msg_containing("Oh noes, something bad happened");
+  teardown_capture_of_logs();
+
+  tt_int_op(controlevent_n, OP_EQ, 10);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10);
+  tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=error "
+            "MESSAGE=\"Oh noes, "
+            "something bad happened. What do we do!?\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=warning "
+            "MESSAGE=\"warning msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=notice "
+            "MESSAGE=\"notice msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=info "
+            "MESSAGE=\"info msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=debug "
+            "MESSAGE=\"debug msg\"\r\n");
+
+  /* Get the STATUS messages out. */
+  process_notify_event_stdout(mp->process);
+
+  tt_int_op(controlevent_n, OP_EQ, 13);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13);
+
+  tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\r\n");
+
   { /* check that the transport info were saved properly in the tor state */
     config_line_t *transport_in_state = NULL;
     smartlist_t *transport_info_sl = smartlist_new();
@@ -434,9 +481,9 @@ test_pt_configure_proxy(void *arg)
   }
 
  done:
+  teardown_capture_of_logs();
   or_state_free(dummy_state);
-  UNMOCK(tor_get_lines_from_handle);
-  UNMOCK(tor_process_handle_destroy);
+  UNMOCK(process_read_stdout);
   UNMOCK(get_or_state);
   UNMOCK(queue_control_event_string);
   if (controlevent_msgs) {
@@ -449,7 +496,7 @@ test_pt_configure_proxy(void *arg)
     smartlist_free(mp->transports);
   }
   smartlist_free(mp->transports_to_launch);
-  tor_free(mp->process_handle);
+  process_free(mp->process);
   tor_free(mp->argv[0]);
   tor_free(mp->argv);
   tor_free(mp);
diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c
new file mode 100644
index 0000000000..76bdbf1891
--- /dev/null
+++ b/src/test/test_ptr_slow.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/ptr_helpers.h"
+
+#include <stdint.h>
+#include <limits.h>
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_int_voidptr_roundtrip(int a)
+{
+  intptr_t ap = (intptr_t)a;
+  void *b = cast_intptr_to_voidstar(ap);
+  intptr_t c = cast_voidstar_to_intptr(b);
+  void *d = cast_intptr_to_voidstar(c);
+
+  tt_assert(ap == c);
+  tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_int_voidstar_interop(void *arg)
+{
+  int a;
+  (void)arg;
+
+  for (a = -1024; a <= 1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MIN; a <= INT_MIN+1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MAX-1024; a < INT_MAX; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_int_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_uint_voidptr_roundtrip(unsigned int a)
+{
+ uintptr_t ap = (uintptr_t)a;
+ void *b = cast_uintptr_to_voidstar(ap);
+ uintptr_t c = cast_voidstar_to_uintptr(b);
+ void *d = cast_uintptr_to_voidstar(c);
+
+ tt_assert(ap == c);
+ tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_uint_voidstar_interop(void *arg)
+{
+  unsigned int a;
+  (void)arg;
+
+  for (a = 0; a <= 1024; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  for (a = UINT_MAX-1024; a < UINT_MAX; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_uint_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+struct testcase_t slow_ptr_tests[] = {
+  { .name = "int_voidstar_interop",
+    .fn = test_int_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  { .name = "uint_voidstar_interop",
+    .fn = test_uint_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c
new file mode 100644
index 0000000000..021323fbf1
--- /dev/null
+++ b/src/test/test_pubsub_build.c
@@ -0,0 +1,578 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+#define PUBSUB_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "test/log_test_helpers.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_int_fmt(msg_aux_data_t aux)
+{
+  int val = (int) aux.u64;
+  char *r=NULL;
+  tor_asprintf(&r, "%d", val);
+  return r;
+}
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+
+static dispatch_typefns_t intfns = {
+  .fmt_fn = ex_int_fmt
+};
+
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+DECLARE_MESSAGE_INT(bunch_of_coconuts, int, int);
+DECLARE_PUBLISH(bunch_of_coconuts);
+DECLARE_SUBSCRIBE(bunch_of_coconuts, coconut_recipient_cb);
+
+DECLARE_MESSAGE(yes_we_have_no, string, char *);
+DECLARE_PUBLISH(yes_we_have_no);
+DECLARE_SUBSCRIBE(yes_we_have_no, absent_item_cb);
+
+static void
+coconut_recipient_cb(const msg_t *m, int n_coconuts)
+{
+  (void)m;
+  (void)n_coconuts;
+}
+
+static void
+absent_item_cb(const msg_t *m, const char *fruitname)
+{
+  (void)m;
+  (void)fruitname;
+}
+
+#define FLAG_SKIP 99999
+
+static void
+seed_dispatch_builder(pubsub_builder_t *b,
+                      unsigned fl1, unsigned fl2, unsigned fl3, unsigned fl4)
+{
+  pubsub_connector_t *c = NULL;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    DISPATCH_REGISTER_TYPE(c, int, &intfns);
+    if (fl1 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, fl1);
+    if (fl2 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, yes_we_have_no, fl2);
+    pubsub_connector_free(c);
+  }
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2"));
+    DISPATCH_REGISTER_TYPE(c, string, &stringfns);
+    if (fl3 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, yes_we_have_no, fl3);
+    if (fl4 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, bunch_of_coconuts, fl4);
+    pubsub_connector_free(c);
+  }
+}
+
+static void
+seed_pubsub_builder_basic(pubsub_builder_t *b)
+{
+  seed_dispatch_builder(b, 0, 0, 0, 0);
+}
+
+/* Regular builder with valid types and messages.
+ */
+static void
+test_pubsub_build_types_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  pubsub_items_t *items = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  dispatcher = pubsub_builder_finalize(b, &items);
+  b = NULL;
+  tt_assert(dispatcher);
+  tt_assert(items);
+  tt_int_op(smartlist_len(items->items), OP_EQ, 4);
+
+  // Make sure that the bindings got build correctly.
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, dispatcher);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+  tt_int_op(dispatcher->n_types, OP_GE, 2);
+  tt_assert(dispatcher->typefns);
+
+  tt_assert(dispatcher->typefns[get_msg_type_id("int")].fmt_fn == ex_int_fmt);
+  tt_assert(dispatcher->typefns[get_msg_type_id("string")].fmt_fn ==
+            ex_str_fmt);
+
+  // Now clear the bindings, like we would do before freeing the
+  // the dispatcher.
+  pubsub_items_clear_bindings(items);
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, NULL);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  pubsub_items_free(items);
+}
+
+/* We fail if the same type is defined in two places with different functions.
+ */
+static void
+test_pubsub_build_types_decls_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    // Extra declaration of int: we don't allow this.
+    DISPATCH_REGISTER_TYPE(c, int, &stringfns);
+    pubsub_connector_free(c);
+  }
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+  // expect_log_msg_containing("(int) declared twice"); // XXXX
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* If a message ID exists but nobody is publishing or subscribing to it,
+ * that's okay. */
+static void
+test_pubsub_build_unused_message(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  // This message isn't actually generated by anyone, but that will be fine:
+  // we just log it at info.
+  get_message_id("unused");
+  setup_capture_of_logs(LOG_INFO);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+  expect_log_msg_containing(
+     "Nobody is publishing or subscribing to message");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Publishing or subscribing to a message with no subscribers / publishers
+ * should fail and warn. */
+static void
+test_pubsub_build_missing_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, FLAG_SKIP, FLAG_SKIP);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing(
+       "Message \"bunch_of_coconuts\" has publishers, but no subscribers.");
+  expect_log_msg_containing(
+       "Message \"yes_we_have_no\" has subscribers, but no publishers.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Make sure that a stub publisher or subscriber prevents an error from
+ * happening even if there are no other publishers/subscribers for a message
+ */
+static void
+test_pubsub_build_stub_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, DISP_FLAG_STUB, DISP_FLAG_STUB);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // no subscribers
+  tt_ptr_op(NULL, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* Only one channel per msg id. */
+static void
+test_pubsub_build_channels_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("int"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent channels.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Only one type per msg id. */
+static void
+test_pubsub_build_types_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent message types.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* The same module can't publish and subscribe the same message */
+static void
+test_pubsub_build_pubsub_same(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    // already publishing this.
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is published "
+                            "and subscribed by the same subsystem \"sys1\".");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* More than one subsystem may publish or subscribe, and that's okay. */
+static void
+test_pubsub_build_pubsub_multi(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_add_pub_(c, &btmp, get_channel_id("main"),
+                    get_message_id("yes_we_have_no"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 2 subscribers.
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(2, OP_EQ, ent->n_enabled);
+  tt_int_op(2, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+static void
+some_other_coconut_hook(const msg_t *m)
+{
+  (void)m;
+}
+
+/* Subscribe hooks should be build correctly when there are a bunch of
+ * them. */
+static void
+test_pubsub_build_sub_many(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  char *sysname = NULL;
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  int i;
+  for (i = 1; i < 100; ++i) {
+    tor_asprintf(&sysname, "system%d",i);
+    c = pubsub_connector_for_subsystem(b, get_subsys_id(sysname));
+    if (i % 7) {
+      DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    } else {
+      pubsub_add_sub_(c, some_other_coconut_hook,
+                      get_channel_id("main"),
+                      get_message_id("bunch_of_coconuts"),
+                      get_msg_type_id("int"),
+                      0 /* flags */,
+                      "somewhere.c", 22);
+    }
+    pubsub_connector_free(c);
+    tor_free(sysname);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(100, OP_EQ, ent->n_enabled);
+  tt_int_op(100, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[76].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[77].fn, OP_EQ, some_other_coconut_hook);
+  tt_ptr_op(ent->rcv[78].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  tor_free(sysname);
+}
+
+/* It's fine to declare the excl flag. */
+static void
+test_pubsub_build_excl_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  // Try one excl/excl pair and one excl/non pair.
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, 0,
+                        DISP_FLAG_EXCL, DISP_FLAG_EXCL);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]->n_enabled);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* but if you declare the excl flag, you need to mean it. */
+static void
+test_pubsub_build_excl_bad(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, DISP_FLAG_EXCL,
+                        0, 0);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, 0);
+    DISPATCH_ADD_SUB_(c, main, yes_we_have_no, 0);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("has multiple publishers, but at least one is "
+                            "marked as exclusive.");
+  expect_log_msg_containing("has multiple subscribers, but at least one is "
+                            "marked as exclusive.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+#define T(name, flags)                                          \
+  { #name, test_pubsub_build_ ## name , (flags), NULL, NULL }
+
+struct testcase_t pubsub_build_tests[] = {
+  T(types_ok, TT_FORK),
+  T(types_decls_conflict, TT_FORK),
+  T(unused_message, TT_FORK),
+  T(missing_pubsub, TT_FORK),
+  T(stub_pubsub, TT_FORK),
+  T(channels_conflict, TT_FORK),
+  T(types_conflict, TT_FORK),
+  T(pubsub_same, TT_FORK),
+  T(pubsub_multi, TT_FORK),
+  T(sub_many, TT_FORK),
+  T(excl_ok, TT_FORK),
+  T(excl_bad, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_msg.c b/src/test/test_pubsub_msg.c
new file mode 100644
index 0000000000..73c7c9f540
--- /dev/null
+++ b/src/test/test_pubsub_msg.c
@@ -0,0 +1,305 @@
+/* Copyright (c) 2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+// We're using the lowest-level publish/subscribe logic here, to avoid the
+// pubsub_macros.h macros and just test the dispatch core.  We'll use a string
+// type for everything.
+
+#define DECLARE_MESSAGE(suffix)                         \
+  static pub_binding_t pub_binding_##suffix;            \
+  static int msg_received_##suffix = 0;                 \
+  static void recv_msg_##suffix(const msg_t *m) {       \
+    (void)m;                                            \
+    ++msg_received_##suffix;                            \
+  }                                                     \
+  EAT_SEMICOLON
+
+#define ADD_PUBLISH(binding_suffix, subsys, channel, msg, flags)        \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_pub_(con, &pub_binding_##binding_suffix,                 \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define ADD_SUBSCRIBE(hook_suffix, subsys, channel, msg, flags)         \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_sub_(con, recv_msg_##hook_suffix,                        \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define SEND(binding_suffix, val)                          \
+  STMT_BEGIN {                                             \
+    msg_aux_data_t data_;                                  \
+    data_.ptr = tor_strdup(val);                           \
+    pubsub_pub_(&pub_binding_##binding_suffix, data_);     \
+  } STMT_END
+
+DECLARE_MESSAGE(msg1);
+DECLARE_MESSAGE(msg2);
+DECLARE_MESSAGE(msg3);
+DECLARE_MESSAGE(msg4);
+DECLARE_MESSAGE(msg5);
+
+static smartlist_t *strings_received = NULL;
+static void
+recv_msg_copy_string(const msg_t *m)
+{
+  const char *s = m->aux_data__.ptr;
+  smartlist_add(strings_received, tor_strdup(s));
+}
+
+static void *
+setup_dispatcher(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  pubsub_connector_t *con;
+
+  {
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id("types"));
+    pubsub_connector_register_type_(con,
+                                    get_msg_type_id("string"),
+                                    &stringfns,
+                                    "nowhere.c", 99);
+    pubsub_connector_free(con);
+  }
+  // message1 has one publisher and one subscriber.
+  ADD_PUBLISH(msg1, sys1, main, message1, 0);
+  ADD_SUBSCRIBE(msg1, sys2, main, message1, 0);
+
+  // message2 has a publisher and a stub subscriber.
+  ADD_PUBLISH(msg2, sys1, main, message2, 0);
+  ADD_SUBSCRIBE(msg2, sys2, main, message2, DISP_FLAG_STUB);
+
+  // message3 has a publisher and three subscribers.
+  ADD_PUBLISH(msg3, sys1, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys2, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys3, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys4, main, message3, 0);
+
+  // message4 has one publisher and two subscribers, but it's on another
+  // channel.
+  ADD_PUBLISH(msg4, sys2, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys1, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys3, other, message4, 0);
+
+  // message5 has a huge number of recipients.
+  ADD_PUBLISH(msg5, sys3, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys4, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys5, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys6, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys7, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys8, main, message5, 0);
+  for (int i = 0; i < 1000-5; ++i) {
+    char *sys;
+    tor_asprintf(&sys, "xsys-%d", i);
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id(sys));
+    pubsub_add_sub_(con, recv_msg_copy_string,
+                    get_channel_id("main"),
+                    get_message_id("message5"),
+                    get_msg_type_id("string"), 0, "here", 100);
+    pubsub_connector_free(con);
+    tor_free(sys);
+  }
+
+  return pubsub_builder_finalize(builder, NULL);
+}
+
+static int
+cleanup_dispatcher(const struct testcase_t *testcase, void *dispatcher_)
+{
+  (void)testcase;
+  dispatch_t *dispatcher = dispatcher_;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+static const struct testcase_setup_t dispatcher_setup = {
+  setup_dispatcher, cleanup_dispatcher
+};
+
+static void
+test_pubsub_msg_minimal(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  SEND(msg1, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(1, OP_EQ, msg_received_msg1); // we got the message!
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_send_to_stub(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg2);
+  SEND(msg2, "hello silence");
+  tt_int_op(0, OP_EQ, msg_received_msg2); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(0, OP_EQ, msg_received_msg2); // doesn't arrive -- stub hook.
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_cancel_msgs(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  for (int i = 0; i < 100; ++i) {
+    SEND(msg1, "hello world");
+  }
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 10));
+  tt_int_op(10, OP_EQ, msg_received_msg1); // we got the message 10 times.
+
+  // At this point, the dispatcher will be freed with queued, undelivered
+  // messages.
+ done:
+  ;
+}
+
+struct alertfn_target {
+  dispatch_t *d;
+  channel_id_t ch;
+  int count;
+};
+static void
+alertfn_generic(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  struct alertfn_target *t = arg;
+  tt_ptr_op(d, OP_EQ, t->d);
+  tt_int_op(ch, OP_EQ, t->ch);
+  ++t->count;
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_alertfns(void *arg)
+{
+  dispatch_t *d = arg;
+  struct alertfn_target ch1_a = { d, get_channel_id("main"), 0 };
+  struct alertfn_target ch2_a = { d, get_channel_id("other"), 0 };
+
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("main"),
+                                  alertfn_generic, &ch1_a));
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("other"),
+                                  alertfn_generic, &ch2_a));
+
+  SEND(msg3, "hello");
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg3, "world");
+  tt_int_op(ch1_a.count, OP_EQ, 1); // only the first message sends an alert
+  tt_int_op(ch2_a.count, OP_EQ, 0); // no alert for 'other'
+
+  SEND(msg4, "worse things happen in C");
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+  // flush the first (main) channel...
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(6, OP_EQ, msg_received_msg3); // 3 subscribers, 2 instances.
+
+  // now that the main channel is flushed, sending another message on it
+  // starts another alert.
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg1, "plover");
+  tt_int_op(ch1_a.count, OP_EQ, 2);
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+/* try more than N_FAST_FNS hooks on msg5 */
+static void
+test_pubsub_msg_many_hooks(void *arg)
+{
+  dispatch_t *d = arg;
+  strings_received = smartlist_new();
+
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  SEND(msg5, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  tt_int_op(0, OP_EQ, smartlist_len(strings_received));
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 100000));
+  tt_int_op(5, OP_EQ, msg_received_msg5);
+  tt_int_op(995, OP_EQ, smartlist_len(strings_received));
+
+ done:
+  SMARTLIST_FOREACH(strings_received, char *, s, tor_free(s));
+  smartlist_free(strings_received);
+}
+
+#define T(name)                                                 \
+  { #name, test_pubsub_msg_ ## name , TT_FORK,                \
+      &dispatcher_setup, NULL }
+
+struct testcase_t pubsub_msg_tests[] = {
+  T(minimal),
+  T(send_to_stub),
+  T(cancel_msgs),
+  T(alertfns),
+  T(many_hooks),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py
index 30a587858f..c9b9200b2d 100644
--- a/src/test/test_rebind.py
+++ b/src/test/test_rebind.py
@@ -22,9 +22,10 @@ def skip(msg):
 
 def try_connecting_to_socksport():
     socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    if socks_socket.connect_ex(('127.0.0.1', socks_port)):
+    e = socks_socket.connect_ex(('127.0.0.1', socks_port))
+    if e:
         tor_process.terminate()
-        fail('Cannot connect to SOCKSPort')
+        fail('Cannot connect to SOCKSPort: error ' + os.strerror(e))
     socks_socket.close()
 
 def wait_for_log(s):
diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh
index ea2012957e..d6d9d86668 100755
--- a/src/test/test_rebind.sh
+++ b/src/test/test_rebind.sh
@@ -7,18 +7,21 @@ if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
   if test "$APPVEYOR" = 'True'; then
-    echo "This test is disabled on Windows CI, as it requires firewall examptions. Skipping." >&2
+    echo "This test is disabled on Windows CI, as it requires firewall exemptions. Skipping." >&2
     exit 77
   fi
 fi
 
-exitcode=0
-
 tmpdir=
-clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; }
+clean () {
+  if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then
+    rm -rf "$tmpdir"
+  fi
+}
+
 trap clean EXIT HUP INT TERM
 
-tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`"
+tmpdir="$(mktemp -d -t tor_rebind_test.XXXXXX)"
 if [ -z "$tmpdir" ]; then
   echo >&2 mktemp failed
   exit 2
diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c
index 0623583511..c65279fb25 100644
--- a/src/test/test_relaycell.c
+++ b/src/test/test_relaycell.c
@@ -17,6 +17,7 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/connection_edge.h"
+#include "core/or/sendme.h"
 #include "core/or/relay.h"
 #include "test/test.h"
 #include "test/log_test_helpers.h"
@@ -812,7 +813,11 @@ test_circbw_relay(void *arg)
   ASSERT_UNCOUNTED_BW();
 
   /* Sendme on circuit with non-full window: counted */
-  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "");
+  /* Recording a cell, the window is updated after decryption so off by one in
+   * order to record and then we process it with the proper window. */
+  circ->cpath->package_window = 901;
+  sendme_record_cell_digest_on_circ(TO_CIRCUIT(circ), circ->cpath);
   circ->cpath->package_window = 900;
   connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
                                      circ->cpath);
diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c
index fe6889e521..4bbf07c3ec 100644
--- a/src/test/test_relaycrypt.c
+++ b/src/test/test_relaycrypt.c
@@ -3,6 +3,8 @@
  * Copyright (c) 2007-2019, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define CRYPT_PATH_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/circuitbuild.h"
 #define CIRCUITLIST_PRIVATE
@@ -10,7 +12,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
-
+#include "core/or/crypt_path.h"
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -49,10 +51,10 @@ testing_circuitset_setup(const struct testcase_t *testcase)
   cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
   for (i=0; i<3; ++i) {
     crypt_path_t *hop = tor_malloc_zero(sizeof(*hop));
-    relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
-                      0, 0);
+    relay_crypto_init(&hop->pvt_crypto, KEY_MATERIAL[i],
+                      sizeof(KEY_MATERIAL[i]), 0, 0);
     hop->state = CPATH_STATE_OPEN;
-    onion_append_to_cpath(&cs->origin_circ->cpath, hop);
+    cpath_extend_linked_list(&cs->origin_circ->cpath, hop);
     tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev);
   }
 
diff --git a/src/test/test_rng.c b/src/test/test_rng.c
new file mode 100644
index 0000000000..dcf08fff1d
--- /dev/null
+++ b/src/test/test_rng.c
@@ -0,0 +1,59 @@
+/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Example usage:
+ *
+ * ./src/test/test-rng --emit | dieharder -g 200 -a
+ *
+ * Remember, dieharder can tell you that your RNG is completely broken, but if
+ * your RNG is not _completely_ broken, dieharder cannot tell you whether your
+ * RNG is actually secure.
+ */
+
+#include "orconfig.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+int
+main(int argc, char **argv)
+{
+  uint8_t buf[0x123];
+
+  if (argc != 2 || strcmp(argv[1], "--emit")) {
+    fprintf(stderr, "If you want me to fill stdout with a bunch of random "
+            "bytes, you need to say --emit.\n");
+    return 1;
+  }
+
+  if (crypto_seed_rng() < 0) {
+    fprintf(stderr, "Can't seed RNG.\n");
+    return 1;
+  }
+
+#if 0
+  while (1) {
+    crypto_rand(buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+#endif /* 0 */
+
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  while (1) {
+    crypto_fast_rng_getbytes(rng, buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+}
diff --git a/src/test/test_router.c b/src/test/test_router.c
index 601881a124..5477ab51e9 100644
--- a/src/test/test_router.c
+++ b/src/test/test_router.c
@@ -7,16 +7,25 @@
  * \brief Unittests for code in router.c
  **/
 
+#define CONFIG_PRIVATE
+#define ROUTER_PRIVATE
+
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerstatus_st.h"
 #include "feature/relay/router.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
+#include "lib/encoding/confline.h"
 
 /* Test suite stuff */
 #include "test/test.h"
@@ -91,6 +100,9 @@ test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
   router = (routerinfo_t*)router_get_my_routerinfo();
   tt_ptr_op(router, !=, NULL);
 
+  /* The real router_get_my_routerinfo() looks up onion_curve25519_pkey using
+   * get_current_curve25519_keypair(), but we don't initialise static data in
+   * this test. */
   router->onion_curve25519_pkey = &ntor_keypair.pubkey;
 
   /* Generate our server descriptor and ensure that the substring
@@ -231,11 +243,254 @@ test_router_check_descriptor_bandwidth_changed(void *arg)
   UNMOCK(we_are_hibernating);
 }
 
+static networkstatus_t *mock_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void)now;
+  return mock_ns;
+}
+
+static routerstatus_t *mock_rs = NULL;
+static const routerstatus_t *
+mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+{
+  (void)ns;
+  (void)digest;
+  return mock_rs;
+}
+
+static void
+test_router_mark_if_too_old(void *arg)
+{
+  (void)arg;
+  time_t now = approx_time();
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+  MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry);
+
+  routerstatus_t rs;
+  networkstatus_t ns;
+  memset(&rs, 0, sizeof(rs));
+  memset(&ns, 0, sizeof(ns));
+  mock_ns = &ns;
+  mock_ns->valid_after = now-3600;
+  mock_rs = &rs;
+  mock_rs->published_on = now - 10;
+
+  // no reason to mark this time.
+  desc_clean_since = now-10;
+  desc_dirty_reason = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+
+  // Doesn't appear in consensus?  Still don't mark it.
+  mock_ns = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+  mock_ns = &ns;
+
+  // No new descriptor in a long time?  Mark it.
+  desc_clean_since = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor");
+
+  // Version in consensus published a long time ago?  We won't mark it
+  // if it's been clean for only a short time.
+  desc_clean_since = now - 10;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now - 10);
+
+  // ... but if it's been clean a while, we mark.
+  desc_clean_since = now - 2 * 3600;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "version listed in consensus is quite old");
+
+  // same deal if we're marked stale.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 10;
+  mock_rs->is_staledesc = 1;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "listed as stale in consensus");
+
+  // same deal if we're absent from the consensus.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "not listed in consensus");
+
+ done:
+  UNMOCK(networkstatus_get_live_consensus);
+  UNMOCK(networkstatus_vote_find_entry);
+}
+
+static node_t fake_node;
+static const node_t *
+mock_node_get_by_nickname(const char *name, unsigned flags)
+{
+  (void)flags;
+  if (!strcasecmp(name, "crumpet"))
+    return &fake_node;
+  else
+    return NULL;
+}
+
+static void
+test_router_get_my_family(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  smartlist_t *sl = NULL;
+  char *join = NULL;
+  // Overwrite the result of router_get_my_identity_digest().  This
+  // happens to be okay, but only for testing.
+  set_server_identity_key_digest_testing(
+                                   (const uint8_t*)"holeinthebottomofthe");
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // No family listed -- so there's no list.
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+#define CLEAR() do {                                    \
+    if (sl) {                                           \
+      SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));  \
+      smartlist_free(sl);                               \
+    }                                                   \
+    tor_free(join);                                     \
+    mock_clean_saved_logs();                            \
+  } while (0)
+
+  // Add a single nice friendly hex member.  This should be enough
+  // to have our own ID added.
+  tt_ptr_op(options->MyFamily, OP_EQ, NULL);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 2);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Add a hex member with a ~.  The ~ part should get removed.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$0123456789abcdef0123456789abcdef01234567~Muffin");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Nickname lookup will fail, so a nickname will appear verbatim.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "BAGEL");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  expect_single_log_msg_containing(
+           "There is a router named \"BAGEL\" in my declared family, but "
+           "I have no descriptor for it.");
+  CLEAR();
+
+  // A bogus digest should fail entirely.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$painauchocolat");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.
+  expect_single_log_msg_containing(
+           "There is a router named \"$painauchocolat\" in my declared "
+           "family, but that isn't a legal digest or nickname. Skipping it.");
+  CLEAR();
+
+  // Let's introduce a node we can look up by nickname
+  memset(&fake_node, 0, sizeof(fake_node));
+  memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN);
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "CRUmpeT");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 5);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.  Some with "$painauchocolat".
+  expect_single_log_msg_containing(
+           "There is a router named \"CRUmpeT\" in my declared "
+           "family, but it wasn't listed by digest. Please consider saying "
+           "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's "
+           "what you meant.");
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+  // Try a singleton list containing only us: It should give us NULL.
+  config_free_lines(options->MyFamily);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$686F6C65696E746865626F74746F6D6F66746865");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+ done:
+  or_options_free(options);
+  teardown_capture_of_logs();
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+#undef CLEAR
+}
+
 #define ROUTER_TEST(name, flags)                          \
   { #name, test_router_ ## name, flags, NULL, NULL }
 
 struct testcase_t router_tests[] = {
   ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK),
   ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK),
+  ROUTER_TEST(mark_if_too_old, TT_FORK),
+  ROUTER_TEST(get_my_family, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c
index 727fa5660f..0c6b533698 100644
--- a/src/test/test_routerkeys.c
+++ b/src/test/test_routerkeys.c
@@ -399,7 +399,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -409,7 +409,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -455,11 +455,11 @@ test_routerkeys_ed_keys_init_all(void *arg)
   options->TestingLinkKeySlop = 2*3600;
 
 #ifdef _WIN32
-  mkdir(dir);
-  mkdir(keydir);
+  tt_int_op(0, OP_EQ, mkdir(dir));
+  tt_int_op(0, OP_EQ, mkdir(keydir));
 #else
-  mkdir(dir, 0700);
-  mkdir(keydir, 0700);
+  tt_int_op(0, OP_EQ, mkdir(dir, 0700));
+  tt_int_op(0, OP_EQ, mkdir(keydir, 0700));
 #endif /* defined(_WIN32) */
 
   options->DataDirectory = dir;
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 95c9176faa..6d596e87ea 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -46,7 +46,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "lib/encoding/confline.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "test/test.h"
 #include "test/test_dir_common.h"
@@ -265,7 +265,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   /* Init SR subsystem. */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -275,7 +277,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   construct_consensus(&consensus_text_md, now);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string(consensus_text_md,
+                                                strlen(consensus_text_md),
+                                                NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -301,7 +305,6 @@ test_router_pick_directory_server_impl(void *arg)
   tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until
                                                       + 24*60*60));
   /* These times are outside the test validity period */
-  tt_assert(networkstatus_consensus_is_bootstrapping(now));
   tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60));
   tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60));
 
@@ -475,7 +478,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -626,7 +631,7 @@ mock_clock_skew_warning(const connection_t *conn, long apparent_skew,
   (void)conn;
   mock_apparent_skew = apparent_skew;
   tt_int_op(trusted, OP_EQ, 1);
-  tt_int_op(domain, OP_EQ, LD_GENERAL);
+  tt_i64_op(domain, OP_EQ, LD_GENERAL);
   tt_str_op(received, OP_EQ, "microdesc flavor consensus");
   tt_str_op(source, OP_EQ, "CONSENSUS");
  done:
@@ -648,7 +653,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -662,7 +669,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
   MOCK(clock_skew_warning, mock_clock_skew_warning);
   /* Caller will call teardown_capture_of_logs() */
   setup_capture_of_logs(LOG_WARN);
-  retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+  retval = networkstatus_set_current_consensus(consensus, strlen(consensus),
+                                               "microdesc", 0,
                                                NULL);
 
  done:
diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c
index c45f0e1595..cc73e6c20a 100644
--- a/src/test/test_routerset.c
+++ b/src/test/test_routerset.c
@@ -1765,7 +1765,7 @@ NS(node_get_by_nickname)(const char *nickname, unsigned flags)
  * Structural test for routerset_get_all_nodes, when the nodelist has no nodes.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
+NS_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 static smartlist_t *NS(mock_smartlist);
 
@@ -1795,7 +1795,7 @@ NS(test_main)(void *arg)
     ;
 }
 
-smartlist_t *
+const smartlist_t *
 NS(nodelist_get_list)(void)
 {
   CALLED(nodelist_get_list)++;
@@ -1811,7 +1811,7 @@ NS(nodelist_get_list)(void)
  * the running_only flag is set, but the nodes are not running.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
+NS_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 static smartlist_t *NS(mock_smartlist);
 static node_t NS(mock_node);
@@ -1844,7 +1844,7 @@ NS(test_main)(void *arg)
     ;
 }
 
-smartlist_t *
+const smartlist_t *
 NS(nodelist_get_list)(void)
 {
   CALLED(nodelist_get_list)++;
diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh
index 00b3e88d37..804d2ada36 100755
--- a/src/test/test_rust.sh
+++ b/src/test/test_rust.sh
@@ -14,11 +14,12 @@ rustc_host=$(rustc -vV | grep host | sed 's/host: //')
 
 for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
     if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
+        # shellcheck disable=SC2086
 	cd "${abs_top_builddir:-../../..}/src/rust" && \
 	    CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \
-	    "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} \
+	    "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \
             --features "test_linking_hack" \
-            --target $rustc_host \
+            --target "$rustc_host" \
 	    ${EXTRA_CARGO_OPTIONS} \
 	    --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1
     fi
diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c
new file mode 100644
index 0000000000..eb402232bc
--- /dev/null
+++ b/src/test/test_sendme.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/* Unit tests for handling different kinds of relay cell */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define SENDME_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/circuitlist.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "lib/crypt_ops/crypto_digest.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_md_consensus = current_ns_consensus =
+    tor_malloc_zero(sizeof(networkstatus_t));
+  current_md_consensus->net_params = smartlist_new();
+  current_md_consensus->routerstatus_list = smartlist_new();
+}
+
+static void
+free_mock_consensus(void)
+{
+  SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r,
+                    tor_free(r));
+  smartlist_free(current_md_consensus->routerstatus_list);
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_v1_record_digest(void *arg)
+{
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  /* Create our dummy circuit. */
+  or_circ = or_circuit_new(1, NULL);
+  /* Points it to the OR circuit now. */
+  circ = TO_CIRCUIT(or_circ);
+
+  /* The package window has to be a multiple of CIRCWINDOW_INCREMENT minus 1
+   * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that
+   * shouldn't be noted. */
+  circ->package_window = CIRCWINDOW_INCREMENT;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(!circ->sendme_last_digests);
+
+  /* This should work now. Package window at CIRCWINDOW_INCREMENT + 1. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(circ->sendme_last_digests);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* Next cell in the package window shouldn't do anything. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* The next CIRCWINDOW_INCREMENT should add one more digest. */
+  circ->package_window = (CIRCWINDOW_INCREMENT * 2) + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 2);
+
+ done:
+  circuit_free_(circ);
+}
+
+static void
+test_v1_consensus_params(void *arg)
+{
+  (void) arg;
+
+  setup_mock_consensus();
+  tt_assert(current_md_consensus);
+
+  /* Both zeroes. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=0");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 0);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Both ones. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 1);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Different values from each other. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Validate is the cell version is coherent with our internal default value
+   * and the one in the consensus. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  /* Minimum acceptable value is 1. */
+  tt_int_op(cell_version_can_be_handled(1), OP_EQ, true);
+  /* Minimum acceptable value is 1 so a cell version of 0 is refused. */
+  tt_int_op(cell_version_can_be_handled(0), OP_EQ, false);
+
+ done:
+  free_mock_consensus();
+}
+
+static void
+test_v1_build_cell(void *arg)
+{
+  uint8_t payload[RELAY_PAYLOAD_SIZE], digest[DIGEST_LEN];
+  ssize_t ret;
+  crypto_digest_t *cell_digest = NULL;
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  or_circ = or_circuit_new(1, NULL);
+  circ = TO_CIRCUIT(or_circ);
+  circ->sendme_last_digests = smartlist_new();
+
+  cell_digest = crypto_digest_new();
+  tt_assert(cell_digest);
+  crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20);
+  crypto_digest_get_digest(cell_digest, (char *) digest, sizeof(digest));
+  smartlist_add(circ->sendme_last_digests, tor_memdup(digest, sizeof(digest)));
+
+  /* SENDME v1 payload is 3 bytes + 20 bytes digest. See spec. */
+  ret = build_cell_payload_v1(digest, payload);
+  tt_int_op(ret, OP_EQ, 23);
+
+  /* Validation. */
+
+  /* An empty payload means SENDME version 0 thus valid. */
+  tt_int_op(sendme_is_valid(circ, payload, 0), OP_EQ, true);
+  /* Current phoney digest should have been popped. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+  /* An unparseable cell means invalid. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, (const uint8_t *) "A", 1), OP_EQ, false);
+  expect_log_msg_containing("Unparseable SENDME cell received. "
+                            "Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* No cell digest recorded for this. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  expect_log_msg_containing("We received a SENDME but we have no cell digests "
+                            "to match. Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* Note the wrong digest in the circuit, cell should fail validation. */
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+  expect_log_msg_containing("SENDME v1 cell digest do not match.");
+  teardown_capture_of_logs();
+
+  /* Record the cell digest into the circuit, cell should validate. */
+  memcpy(or_circ->crypto.sendme_digest, digest, sizeof(digest));
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, true);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+ done:
+  crypto_digest_free(cell_digest);
+  circuit_free_(circ);
+}
+
+static void
+test_cell_payload_pad(void *arg)
+{
+  size_t pad_offset, payload_len, expected_offset;
+
+  (void) arg;
+
+  /* Offset should be 0, not enough room for padding. */
+  payload_len = RELAY_PAYLOAD_SIZE;
+  pad_offset = get_pad_cell_offset(payload_len);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Still no room because we keep 4 extra bytes. */
+  pad_offset = get_pad_cell_offset(payload_len - 4);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* We should have 1 byte of padding. Meaning, the offset should be the
+   * CELL_PAYLOAD_SIZE minus 1 byte. */
+  expected_offset = CELL_PAYLOAD_SIZE - 1;
+  pad_offset = get_pad_cell_offset(payload_len - 5);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Now some arbitrary small payload length. The cell size is header + 10 +
+   * extra 4 bytes we keep so the offset should be there. */
+  expected_offset = RELAY_HEADER_SIZE + 10 + 4;
+  pad_offset = get_pad_cell_offset(10);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Data length of 0. */
+  expected_offset = RELAY_HEADER_SIZE + 4;
+  pad_offset = get_pad_cell_offset(0);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+ done:
+  ;
+}
+
+static void
+test_cell_version_validation(void *arg)
+{
+  (void) arg;
+
+  /* We currently only support up to SENDME_MAX_SUPPORTED_VERSION so we are
+   * going to test the boundaries there. */
+
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION));
+
+  /* Version below our supported should pass. */
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION - 1));
+
+  /* Extra version from our supported should fail. */
+  tt_assert(!cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION + 1));
+
+  /* Simple check for version 0. */
+  tt_assert(cell_version_can_be_handled(0));
+
+  /* We MUST handle the default cell version that we emit or accept. */
+  tt_assert(cell_version_can_be_handled(SENDME_EMIT_MIN_VERSION_DEFAULT));
+  tt_assert(cell_version_can_be_handled(SENDME_ACCEPT_MIN_VERSION_DEFAULT));
+
+ done:
+  ;
+}
+
+/* check our decisions about how much stuff to put into relay cells. */
+static void
+test_package_payload_len(void *arg)
+{
+  (void)arg;
+  /* this is not a real circuit: it only has the fields needed for this
+   * test. */
+  circuit_t *c = tor_malloc_zero(sizeof(circuit_t));
+
+  /* check initial conditions. */
+  circuit_reset_sendme_randomness(c);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE, CIRCWINDOW_INCREMENT / 2);
+  tt_int_op(c->send_randomness_after_n_cells, OP_LT, CIRCWINDOW_INCREMENT);
+
+  /* We have a bunch of cells before we need to send randomness, so the first
+   * few can be packaged full. */
+  int initial = c->send_randomness_after_n_cells;
+  size_t n = connection_edge_get_inbuf_bytes_to_package(10000, 0, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  n = connection_edge_get_inbuf_bytes_to_package(95000, 1, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial isn't set, we won't package a partially full cell at
+   * all. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 0, c);
+  tt_int_op(n, OP_EQ, 0);
+  /* no change in our state, since nothing was sent. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial is set and the partial cell is not going to have
+   * _enough_ randomness, we package it, but we don't consider ourselves to
+   * have sent a sufficiently random cell. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-1);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 3);
+
+  /* Make sure we set have_set_sufficiently_random_cell as appropriate. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-64, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-64);
+  tt_assert(c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 4);
+
+  /* Now let's look at what happens when we get down to zero. Since we have
+   * sent a sufficiently random cell, we will not force this one to have a gap.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* What would happen if we hadn't sent a sufficiently random cell? */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  const size_t reduced_payload_size = RELAY_PAYLOAD_SIZE - 4 - 16;
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* Here is a fun case: if it's time to package a small cell, then
+   * package_partial==0 should mean we accept that many bytes.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(reduced_payload_size, 0, c);
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+
+ done:
+  tor_free(c);
+}
+
+struct testcase_t sendme_tests[] = {
+  { "v1_record_digest", test_v1_record_digest, TT_FORK,
+    NULL, NULL },
+  { "v1_consensus_params", test_v1_consensus_params, TT_FORK,
+    NULL, NULL },
+  { "v1_build_cell", test_v1_build_cell, TT_FORK,
+    NULL, NULL },
+  { "cell_payload_pad", test_cell_payload_pad, TT_FORK,
+    NULL, NULL },
+  { "cell_version_validation", test_cell_version_validation, TT_FORK,
+    NULL, NULL },
+  { "package_payload_len", test_package_payload_len, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index b4fe6eef64..9c8703fa6f 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -58,14 +58,17 @@ trusteddirserver_get_by_v3_auth_digest_m(const char *digest)
 }
 
 /* Setup a minimal dirauth environment by initializing the SR state and
- * making sure the options are set to be an authority directory. */
+ * making sure the options are set to be an authority directory.
+ * You must only call this function once per process. */
 static void
 init_authority_state(void)
 {
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   or_options_t *options = get_options_mutable();
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   tt_assert(mock_cert);
   options->AuthoritativeDir = 1;
   tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
@@ -310,6 +313,7 @@ test_get_start_time_of_current_run(void *arg)
 
     retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC",
                                 &mock_consensus.valid_after);
+    tt_int_op(retval, OP_EQ, 0);
 
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC",
                                 &current_time);
@@ -424,7 +428,9 @@ test_sr_commit(void *arg)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -443,12 +449,12 @@ test_sr_commit(void *arg)
     /* We should have a reveal value. */
     tt_assert(commit_has_reveal_value(our_commit));
     /* We should have a random value. */
-    tt_assert(!tor_mem_is_zero((char *) our_commit->random_number,
+    tt_assert(!fast_mem_is_zero((char *) our_commit->random_number,
                                sizeof(our_commit->random_number)));
     /* Commit and reveal timestamp should be the same. */
     tt_u64_op(our_commit->commit_ts, OP_EQ, our_commit->reveal_ts);
     /* We should have a hashed reveal. */
-    tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal,
+    tt_assert(!fast_mem_is_zero(our_commit->hashed_reveal,
                                sizeof(our_commit->hashed_reveal)));
     /* Do we have a valid encoded commit and reveal. Note the following only
      * tests if the generated values are correct. Their could be a bug in
@@ -835,7 +841,9 @@ test_sr_setup_commits(void)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -1073,73 +1081,99 @@ test_sr_get_majority_srv_from_votes(void *arg)
   smartlist_free(votes);
 }
 
+/* Testing sr_srv_dup(). */
 static void
-test_utils(void *arg)
+test_sr_svr_dup(void *arg)
 {
-  (void) arg;
+  (void)arg;
 
-  /* Testing srv_dup(). */
-  {
-    sr_srv_t *srv = NULL, *dup_srv = NULL;
-    const char *srv_value =
-      "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
-    srv = tor_malloc_zero(sizeof(*srv));
-    srv->num_reveals = 42;
-    memcpy(srv->value, srv_value, sizeof(srv->value));
-    dup_srv = srv_dup(srv);
-    tt_assert(dup_srv);
-    tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
-    tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
-    tor_free(srv);
-    tor_free(dup_srv);
-  }
+  sr_srv_t *srv = NULL, *dup_srv = NULL;
+  const char *srv_value =
+    "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
+  srv = tor_malloc_zero(sizeof(*srv));
+  srv->num_reveals = 42;
+  memcpy(srv->value, srv_value, sizeof(srv->value));
+  dup_srv = sr_srv_dup(srv);
+  tt_assert(dup_srv);
+  tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
+  tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
 
-  /* Testing commitments_are_the_same(). Currently, the check is to test the
-   * value of the encoded commit so let's make sure that actually works. */
-  {
-    /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
-     * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
-     * which comes down to SR_COMMIT_BASE64_LEN + 1. */
-    const char *payload =
-      "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
-      "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
-      "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
-      "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
-    sr_commit_t commit1, commit2;
-    memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
-    memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
-    /* Let's corrupt one of them. */
-    memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
-  }
+ done:
+  tor_free(srv);
+  tor_free(dup_srv);
+}
 
-  /* Testing commit_is_authoritative(). */
-  {
-    crypto_pk_t *k = crypto_pk_new();
-    char digest[DIGEST_LEN];
-    sr_commit_t commit;
+/* Testing commitments_are_the_same(). Currently, the check is to test the
+ * value of the encoded commit so let's make sure that actually works. */
+static void
+test_commitments_are_the_same(void *arg)
+{
+  (void)arg;
+
+  /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
+  * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
+  * which comes down to SR_COMMIT_BASE64_LEN + 1. */
+  const char *payload =
+  "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
+  "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
+  "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
+  "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
+  sr_commit_t commit1, commit2;
+  memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
+  memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
+  /* Let's corrupt one of them. */
+  memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
 
-    tt_assert(!crypto_pk_generate_key(k));
+ done:
+  return;
+}
 
-    tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
-    memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
-    /* Change the pubkey. */
-    memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
-    crypto_pk_free(k);
-  }
+/* Testing commit_is_authoritative(). */
+static void
+test_commit_is_authoritative(void *arg)
+{
+  (void)arg;
 
-  /* Testing get_phase_str(). */
-  {
-    tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
-    tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
-  }
+  crypto_pk_t *k = crypto_pk_new();
+  char digest[DIGEST_LEN];
+  sr_commit_t commit;
+
+  tt_assert(!crypto_pk_generate_key(k));
+
+  tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
+  memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
+  /* Change the pubkey. */
+  memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
+
+ done:
+  crypto_pk_free(k);
+}
+
+static void
+test_get_phase_str(void *arg)
+{
+  (void)arg;
+
+  tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
+  tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
+
+ done:
+  return;
+}
+
+/* Test utils that depend on authority state */
+static void
+test_utils_auth(void *arg)
+{
+  (void)arg;
+  init_authority_state();
 
   /* Testing phase transition */
   {
-    init_authority_state();
     set_sr_phase(SR_PHASE_COMMIT);
     tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1);
     tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0);
@@ -1150,8 +1184,193 @@ test_utils(void *arg)
     tt_int_op(is_phase_transition(42), OP_EQ, 1);
   }
 
+  /* Testing get, set, delete, clean SRVs */
+
+  {
+    /* Just set the previous SRV */
+    test_sr_setup_srv(0);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Delete the SRVs one at a time */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now set the SRVs to NULL instead */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now copy the values across */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the previous: the protocol goes the other way */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_previous_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is the same */
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  {
+    /* Now copy a value onto itself */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Take a copy of the old value */
+    sr_srv_t old_current_srv;
+    memcpy(&old_current_srv, sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the current: the protocol never replaces an SRV with
+     * the same value */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_current_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different between current and previous */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is the same as the old content */
+    tt_mem_op(&old_current_srv, OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  /* I don't think we can say "expect a BUG()" in our tests. */
+#if 0
+  {
+    /* Now copy a value onto itself without sr_srv_dup().
+     * This should fail with a BUG() warning. */
+    test_sr_setup_srv(1);
+    sr_state_set_current_srv(sr_state_get_current_srv());
+    sr_state_set_previous_srv(sr_state_get_previous_srv());
+  }
+#endif /* 0 */
+
  done:
-  return;
+  sr_state_free_all();
 }
 
 static void
@@ -1159,6 +1378,7 @@ test_state_transition(void *arg)
 {
   sr_state_t *state = NULL;
   time_t now = time(NULL);
+  sr_srv_t *cur = NULL;
 
   (void) arg;
 
@@ -1197,44 +1417,47 @@ test_state_transition(void *arg)
 
   /* Test SRV rotation in our state. */
   {
-    const sr_srv_t *cur, *prev;
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     tt_assert(cur);
-    /* After, current srv should be the previous and then set to NULL. */
+    /* After, the previous SRV should be the same as the old current SRV, and
+     * the current SRV should be set to NULL */
     state_rotate_srv();
-    prev = sr_state_get_previous_srv();
-    tt_assert(prev == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
     sr_state_clean_srvs();
+    tor_free(cur);
   }
 
   /* New protocol run. */
   {
-    const sr_srv_t *cur;
     /* Setup some new SRVs so we can confirm that a new protocol run
      * actually makes them rotate and compute new ones. */
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
-    tt_assert(cur);
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     set_sr_phase(SR_PHASE_REVEAL);
     MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
     new_protocol_run(now);
     UNMOCK(get_my_v3_authority_cert);
     /* Rotation happened. */
-    tt_assert(sr_state_get_previous_srv() == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     /* We are going into COMMIT phase so we had to rotate our SRVs. Usually
      * our current SRV would be NULL but a new protocol run should make us
      * compute a new SRV. */
     tt_assert(sr_state_get_current_srv());
     /* Also, make sure we did change the current. */
-    tt_assert(sr_state_get_current_srv() != cur);
+    tt_mem_op(sr_state_get_current_srv(), OP_NE, cur, sizeof(sr_srv_t));
     /* We should have our commitment alone. */
     tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
     tt_int_op(state->n_commit_rounds, OP_EQ, 0);
     /* 46 here since we were at 45 just before. */
     tt_u64_op(state->n_protocol_runs, OP_EQ, 46);
+    tor_free(cur);
   }
 
   /* Cleanup of SRVs. */
@@ -1245,6 +1468,7 @@ test_state_transition(void *arg)
   }
 
  done:
+  tor_free(cur);
   sr_state_free_all();
 }
 
@@ -1440,7 +1664,13 @@ struct testcase_t sr_tests[] = {
   { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL },
   { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes,
     TT_FORK, NULL, NULL },
-  { "utils", test_utils, TT_FORK, NULL, NULL },
+  { "sr_svr_dup", test_sr_svr_dup, TT_FORK, NULL, NULL },
+  { "commitments_are_the_same", test_commitments_are_the_same, TT_FORK, NULL,
+    NULL },
+  { "commit_is_authoritative", test_commit_is_authoritative, TT_FORK, NULL,
+    NULL },
+  { "get_phase_str", test_get_phase_str, TT_FORK, NULL, NULL },
+  { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL },
   { "state_transition", test_state_transition, TT_FORK, NULL, NULL },
   { "state_update", test_state_update, TT_FORK,
     NULL, NULL },
diff --git a/src/test/test_slow.c b/src/test/test_slow.c
index bda67b2d92..d4d5b755a5 100644
--- a/src/test/test_slow.c
+++ b/src/test/test_slow.c
@@ -20,7 +20,9 @@
 
 struct testgroup_t testgroups[] = {
   { "slow/crypto/", slow_crypto_tests },
-  { "slow/util/", slow_util_tests },
+  { "slow/process/", slow_process_tests },
+  { "slow/prob_distr/", slow_stochastic_prob_distr_tests },
+  { "slow/ptr/", slow_ptr_tests },
   END_OF_GROUPS
 };
 
diff --git a/src/test/test_socks.c b/src/test/test_socks.c
index 3686e1036b..6d3ebd2320 100644
--- a/src/test/test_socks.c
+++ b/src/test/test_socks.c
@@ -4,7 +4,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_socks.h"
diff --git a/src/test/test_status.c b/src/test/test_status.c
index 9c47469975..2fb2a7b24f 100644
--- a/src/test/test_status.c
+++ b/src/test/test_status.c
@@ -404,7 +404,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -412,7 +412,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -429,7 +429,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 3:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_circuit_handshake_stats"),
                 OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
@@ -442,13 +442,13 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 4:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_link_protocol_counts"),
                 OP_NE, NULL);
       break;
     case 5:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s");
       tt_str_op(va_arg(ap, char *), OP_EQ,
                 " 0 circuits killed with too many cells.");
@@ -574,7 +574,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
   ++NS(n_msgs);
 
   tt_int_op(severity, OP_EQ, LOG_NOTICE);
-  tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+  tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
   tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
   tt_ptr_op(suffix, OP_EQ, NULL);
   tt_str_op(format, OP_EQ,
@@ -709,7 +709,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -723,7 +723,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_accounting"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -889,7 +889,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -903,7 +903,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1038,7 +1038,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1052,7 +1052,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c
index baddf8d66e..19483713f7 100644
--- a/src/test/test_switch_id.c
+++ b/src/test/test_switch_id.c
@@ -87,7 +87,7 @@ main(int argc, char **argv)
 
   fprintf(stderr, "This test is not supported on your OS.\n");
   return 77;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *username;
   const char *testname;
   if (argc != 3) {
diff --git a/src/test/test_switch_id.sh b/src/test/test_switch_id.sh
index 79c44f2eb1..b13bf7602f 100755
--- a/src/test/test_switch_id.sh
+++ b/src/test/test_switch_id.sh
@@ -1,11 +1,11 @@
 #!/bin/sh
 
-if test "`id -u`" != '0'; then
+if test "$(id -u)" != '0'; then
     echo "This test only works when run as root. Skipping." >&2
     exit 77
 fi
 
-if test "`id -u nobody`" = ""; then
+if test "$(id -u nobody)" = ""; then
     echo "This test requires that your system have a 'nobody' user. Sorry." >&2
     exit 1
 fi
diff --git a/src/test/test_token_bucket.c b/src/test/test_token_bucket.c
new file mode 100644
index 0000000000..31670718d9
--- /dev/null
+++ b/src/test/test_token_bucket.c
@@ -0,0 +1,152 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_bwmgt.c
+ * \brief tests for bandwidth management / token bucket functions
+ */
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "lib/evloop/token_bucket.h"
+
+// an imaginary time, in timestamp units. Chosen so it will roll over.
+static const uint32_t START_TS = UINT32_MAX - 1000;
+static const uint32_t RATE = 10;
+static const uint32_t BURST = 50;
+
+static void
+test_token_bucket_ctr_init(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+  tt_uint_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_adjust(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Increase burst. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST * 2);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST * 2);
+
+  /* Decrease burst but still above bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST + 10);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST + 10);
+
+  /* Decrease burst below bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST - 1);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST - 1);
+
+  /* Change rate. */
+  token_bucket_ctr_adjust(&tb, RATE * 2, BURST);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE * 2);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_dec(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Simple decrement by one. */
+  tt_uint_op(0, OP_EQ, token_bucket_ctr_dec(&tb, 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+
+  /* Down to 0. Becomes empty. */
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST - 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, 0);
+
+  /* Reset and try to underflow. */
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST + 1));
+  tt_int_op(tb.counter.bucket, OP_EQ, -1);
+
+  /* Keep underflowing shouldn't flag the bucket as empty. */
+  tt_uint_op(false, OP_EQ, token_bucket_ctr_dec(&tb, BURST));
+  tt_int_op(tb.counter.bucket, OP_EQ, - (int32_t) (BURST + 1));
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_refill(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Reduce of half the bucket and let a single second go before refill. */
+  token_bucket_ctr_dec(&tb, BURST / 2);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST / 2);
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* No time change, nothing should move. */
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* Add 99 seconds, bucket should be back to a full BURST. */
+  token_bucket_ctr_refill(&tb, START_TS + 99);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 99);
+
+  /* Empty bucket at once. */
+  token_bucket_ctr_dec(&tb, BURST);
+  tt_int_op(tb.counter.bucket, OP_EQ, 0);
+  /* On second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 100);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 100);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE);
+  /* A second second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 101);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 101);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE * 2);
+
+ done:
+  ;
+}
+
+#define TOKEN_BUCKET(name)                                          \
+  { #name, test_token_bucket_ ## name , 0, NULL, NULL }
+
+struct testcase_t token_bucket_tests[] = {
+  TOKEN_BUCKET(ctr_init),
+  TOKEN_BUCKET(ctr_adjust),
+  TOKEN_BUCKET(ctr_dec),
+  TOKEN_BUCKET(ctr_refill),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c
index 853abc4f91..d59b8e001d 100644
--- a/src/test/test_tortls.c
+++ b/src/test/test_tortls.c
@@ -230,7 +230,7 @@ test_tortls_tor_tls_get_error(void *data)
   crypto_pk_free(key2);
   tor_tls_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_x509_cert_get_id_digests(void *ignored)
@@ -347,7 +347,7 @@ test_tortls_server_got_renegotiate(void *ignored)
  done:
   tor_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_evaluate_ecgroup_for_tls(void *ignored)
@@ -598,7 +598,7 @@ struct testcase_t tortls_tests[] = {
   LOCAL_TEST_CASE(get_forced_write_size, 0),
   LOCAL_TEST_CASE(used_v1_handshake, TT_FORK),
   LOCAL_TEST_CASE(server_got_renegotiate, 0),
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   LOCAL_TEST_CASE(evaluate_ecgroup_for_tls, 0),
   LOCAL_TEST_CASE(double_init, TT_FORK),
   LOCAL_TEST_CASE(address, TT_FORK),
diff --git a/src/test/test_tortls.h b/src/test/test_tortls.h
index 1a8b117d0f..4567b9f6a0 100644
--- a/src/test/test_tortls.h
+++ b/src/test/test_tortls.h
@@ -10,4 +10,4 @@ extern const char *notCompletelyValidCertString;
 extern const char *validCertString;
 extern const char *caCertString;
 
-#endif
+#endif /* !defined(TEST_TORTLS_H) */
diff --git a/src/test/test_tortls_openssl.c b/src/test/test_tortls_openssl.c
index f039980a25..93135d0b19 100644
--- a/src/test/test_tortls_openssl.c
+++ b/src/test/test_tortls_openssl.c
@@ -133,7 +133,7 @@ library_init(void)
 #else
   SSL_library_init();
   SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 }
 
 static void
@@ -477,7 +477,7 @@ fake_x509_free(X509 *cert)
     tor_free(cert);
   }
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
diff --git a/src/test/test_util.c b/src/test/test_util.c
index 3a0eb15157..aebefe64c5 100644
--- a/src/test/test_util.c
+++ b/src/test/test_util.c
@@ -6,22 +6,24 @@
 #include "orconfig.h"
 #define COMPAT_PRIVATE
 #define COMPAT_TIME_PRIVATE
-#define CONTROL_PRIVATE
 #define UTIL_PRIVATE
 #define UTIL_MALLOC_PRIVATE
 #define SOCKET_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define PROCESS_WIN32_PRIVATE
 #include "lib/testsupport/testsupport.h"
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "feature/control/control.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/transports.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "test/test.h"
 #include "lib/memarea/memarea.h"
 #include "lib/process/waitpid.h"
+#include "lib/process/process_win32.h"
 #include "test/log_test_helpers.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_zstd.h"
@@ -30,7 +32,6 @@
 #include "lib/fs/winlib.h"
 #include "lib/process/env.h"
 #include "lib/process/pidfile.h"
-#include "lib/process/subprocess.h"
 #include "lib/intmath/weakrng.h"
 #include "lib/thread/numcpus.h"
 #include "lib/math/fp.h"
@@ -39,6 +40,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/socketpair.h"
+#include "lib/malloc/map_anon.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
@@ -58,6 +60,12 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
 
 #ifdef _WIN32
 #include <tchar.h>
@@ -69,6 +77,28 @@
 #define INFINITY_DBL ((double)INFINITY)
 #define NAN_DBL ((double)NAN)
 
+/** Test the tor_isinf() wrapper */
+static void
+test_tor_isinf(void *arg)
+{
+  (void) arg;
+
+  tt_assert(tor_isinf(INFINITY_DBL));
+
+  tt_assert(!tor_isinf(NAN_DBL));
+  tt_assert(!tor_isinf(DBL_EPSILON));
+  tt_assert(!tor_isinf(DBL_MAX));
+  tt_assert(!tor_isinf(DBL_MIN));
+
+  tt_assert(!tor_isinf(0.0));
+  tt_assert(!tor_isinf(0.1));
+  tt_assert(!tor_isinf(3));
+  tt_assert(!tor_isinf(3.14));
+
+ done:
+  ;
+}
+
 /* XXXX this is a minimal wrapper to make the unit tests compile with the
  * changed tor_timegm interface. */
 static time_t
@@ -404,7 +434,6 @@ test_util_time(void *arg)
 
 /* Assume tv_usec is an unsigned integer until proven otherwise */
 #define TV_USEC_MAX UINT_MAX
-#define TOR_USEC_PER_SEC 1000000
 
   /* Overflows in the result type */
 
@@ -2058,14 +2087,14 @@ test_util_strmisc(void *arg)
   /* Test mem_is_zero */
   memset(buf,0,128);
   buf[128] = 'x';
-  tt_assert(tor_mem_is_zero(buf, 10));
-  tt_assert(tor_mem_is_zero(buf, 20));
-  tt_assert(tor_mem_is_zero(buf, 128));
-  tt_assert(!tor_mem_is_zero(buf, 129));
+  tt_assert(fast_mem_is_zero(buf, 10));
+  tt_assert(fast_mem_is_zero(buf, 20));
+  tt_assert(fast_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 129));
   buf[60] = (char)255;
-  tt_assert(!tor_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 128));
   buf[0] = (char)1;
-  tt_assert(!tor_mem_is_zero(buf, 10));
+  tt_assert(!fast_mem_is_zero(buf, 10));
 
   /* Test 'escaped' */
   tt_ptr_op(escaped(NULL), OP_EQ, NULL);
@@ -2179,15 +2208,6 @@ test_util_strmisc(void *arg)
   tt_int_op(strcmp_opt(NULL,  "foo"), OP_LT, 0);
   tt_int_op(strcmp_opt("foo", NULL),  OP_GT, 0);
 
-  /* Test strcmp_len */
-  tt_int_op(strcmp_len("foo", "bar", 3),   OP_GT, 0);
-  tt_int_op(strcmp_len("foo", "bar", 2),   OP_LT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */
-  tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0);  /* Really stop at len */
-  tt_int_op(strcmp_len("blah", "", 4),     OP_GT, 0);
-  tt_int_op(strcmp_len("blah", "", 0),    OP_EQ, 0);
-
  done:
   tor_free(cp_tmp);
 }
@@ -3769,7 +3789,7 @@ test_util_memarea(void *arg)
   tt_int_op(((uintptr_t)p3) % sizeof(void*),OP_EQ, 0);
   tt_assert(!memarea_owns_ptr(area, p3+8192));
   tt_assert(!memarea_owns_ptr(area, p3+30));
-  tt_assert(tor_mem_is_zero(p2, 52));
+  tt_assert(fast_mem_is_zero(p2, 52));
   /* Make sure we don't overalign. */
   p1 = memarea_alloc(area, 1);
   p2 = memarea_alloc(area, 1);
@@ -4050,6 +4070,13 @@ test_util_string_is_utf8(void *ptr)
   tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7));
   tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9));
 
+  /* Test the utf8_no_bom function */
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7));
+  tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect",
+                                            strlen("loveandrespect")));
+
   // Validate exactly 'len' bytes.
   tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2));
   tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6));
@@ -4320,204 +4347,6 @@ test_util_load_win_lib(void *ptr)
 }
 #endif /* defined(_WIN32) */
 
-#ifndef _WIN32
-static void
-clear_hex_errno(char *hex_errno)
-{
-  memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1);
-}
-
-static void
-test_util_exit_status(void *ptr)
-{
-  /* Leave an extra byte for a \0 so we can do string comparison */
-  char hex_errno[HEX_ERRNO_SIZE + 1];
-  int n;
-
-  (void)ptr;
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0, hex_errno);
-  tt_str_op("0/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-#if SIZEOF_INT == 4
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x80000000, hex_errno);
-  tt_str_op("FF/-80000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#elif SIZEOF_INT == 8
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno);
-  tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#endif /* SIZEOF_INT == 4 || ... */
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x7F, 0, hex_errno);
-  tt_str_op("7F/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x08, -0x242, hex_errno);
-  tt_str_op("8/-242\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
- done:
-  ;
-}
-#endif /* !defined(_WIN32) */
-
-#ifndef _WIN32
-static void
-test_util_string_from_pipe(void *ptr)
-{
-  int test_pipe[2] = {-1, -1};
-  int retval = 0;
-  enum stream_status status = IO_STREAM_TERM;
-  ssize_t retlen;
-  char buf[4] = { 0 };
-
-  (void)ptr;
-
-  errno = 0;
-
-  /* Set up a pipe to test on */
-  retval = pipe(test_pipe);
-  tt_int_op(retval, OP_EQ, 0);
-
-  /* Send in a string. */
-  retlen = write(test_pipe[1], "ABC", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "ABC");
-  errno = 0;
-
-  /* Send in a string that contains a nul. */
-  retlen = write(test_pipe[1], "AB\0", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a nul only. */
-  retlen = write(test_pipe[1], "\0", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string that contains a trailing newline. */
-  retlen = write(test_pipe[1], "AB\n", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a newline only. */
-  retlen = write(test_pipe[1], "\n", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string and check that we nul terminate return values. */
-  retlen = write(test_pipe[1], "AAA", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AAA");
-  tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf));
-  errno = 0;
-
-  retlen = write(test_pipe[1], "B", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  memset(buf, '\xff', sizeof(buf));
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "B");
-  tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf));
-  errno = 0;
-
-  /* Send in multiple lines. */
-  retlen = write(test_pipe[1], "A\nB", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "A\nB");
-  errno = 0;
-
-  /* Send in a line and close */
-  retlen = write(test_pipe[1], "AB", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retval = close(test_pipe[1]);
-  tt_int_op(retval, OP_EQ, 0);
-  test_pipe[1] = -1;
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Check for EOF */
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_CLOSED);
-  errno = 0;
-
- done:
-  if (test_pipe[0] != -1)
-    close(test_pipe[0]);
-  if (test_pipe[1] != -1)
-    close(test_pipe[1]);
-}
-
-#endif /* !defined(_WIN32) */
-
 /**
  * Test for format_hex_number_sigsafe()
  */
@@ -4612,57 +4441,6 @@ test_util_format_dec_number(void *ptr)
   return;
 }
 
-/**
- * Test that we can properly format a Windows command line
- */
-static void
-test_util_join_win_cmdline(void *ptr)
-{
-  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
-   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
-   * will try to only generate simple cases for the child process to parse;
-   * i.e. we never embed quoted strings in arguments. */
-
-  const char *argvs[][4] = {
-    {"a", "bb", "CCC", NULL}, // Normal
-    {NULL, NULL, NULL, NULL}, // Empty argument list
-    {"", NULL, NULL, NULL}, // Empty argument
-    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
-    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
-    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
-    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
-    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
-    { NULL } // Terminator
-  };
-
-  const char *cmdlines[] = {
-    "a bb CCC",
-    "",
-    "\"\"",
-    "\\\"a b\\\"b CCC\\\"",
-    "\"a\tbc\" \"dd  dd\" E",
-    "a\\\\\\b \"de fg\" H",
-    "a\\\\\\\"b \\c D\\",
-    "\"a\\\\b c\" d E",
-    NULL // Terminator
-  };
-
-  int i;
-  char *joined_argv = NULL;
-
-  (void)ptr;
-
-  for (i=0; cmdlines[i]!=NULL; i++) {
-    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
-    joined_argv = tor_join_win_cmdline(argvs[i]);
-    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
-    tor_free(joined_argv);
-  }
-
- done:
-  tor_free(joined_argv);
-}
-
 #define MAX_SPLIT_LINE_COUNT 4
 struct split_lines_test_t {
   const char *orig_line; // Line to be split (may contain \0's)
@@ -4670,67 +4448,6 @@ struct split_lines_test_t {
   const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines
 };
 
-/**
- * Test that we properly split a buffer into lines
- */
-static void
-test_util_split_lines(void *ptr)
-{
-  /* Test cases. orig_line of last test case must be NULL.
-   * The last element of split_line[i] must be NULL. */
-  struct split_lines_test_t tests[] = {
-    {"", 0, {NULL}},
-    {"foo", 3, {"foo", NULL}},
-    {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}},
-    {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}},
-    {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}},
-    {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}},
-    {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}},
-    {NULL, 0, { NULL }}
-  };
-
-  int i, j;
-  char *orig_line=NULL;
-  smartlist_t *sl=NULL;
-
-  (void)ptr;
-
-  for (i=0; tests[i].orig_line; i++) {
-    sl = smartlist_new();
-    /* Allocate space for string and trailing NULL */
-    orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1);
-    tor_split_lines(sl, orig_line, tests[i].orig_length);
-
-    j = 0;
-    log_info(LD_GENERAL, "Splitting test %d of length %d",
-             i, tests[i].orig_length);
-    SMARTLIST_FOREACH_BEGIN(sl, const char *, line) {
-      /* Check we have not got too many lines */
-      tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j);
-      /* Check that there actually should be a line here */
-      tt_ptr_op(tests[i].split_line[j], OP_NE, NULL);
-      log_info(LD_GENERAL, "Line %d of test %d, should be <%s>",
-               j, i, tests[i].split_line[j]);
-      /* Check that the line is as expected */
-      tt_str_op(line,OP_EQ, tests[i].split_line[j]);
-      j++;
-    } SMARTLIST_FOREACH_END(line);
-    /* Check that we didn't miss some lines */
-    tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]);
-    tor_free(orig_line);
-    smartlist_free(sl);
-    sl = NULL;
-  }
-
- done:
-  tor_free(orig_line);
-  smartlist_free(sl);
-}
-
 static void
 test_util_di_ops(void *arg)
 {
@@ -5682,6 +5399,13 @@ test_util_socketpair(void *arg)
     tt_skip();
   }
 #endif /* defined(__FreeBSD__) */
+#ifdef ENETUNREACH
+  if (ersatz && socketpair_result == -ENETUNREACH) {
+    /* We can also fail with -ENETUNREACH if we have no network stack at
+     * all. */
+    tt_skip();
+  }
+#endif /* defined(ENETUNREACH) */
   tt_int_op(0, OP_EQ, socketpair_result);
 
   tt_assert(SOCKET_OK(fds[0]));
@@ -5834,6 +5558,18 @@ test_util_ipv4_validation(void *arg)
 }
 
 static void
+test_util_ipv6_validation(void *arg)
+{
+  (void)arg;
+
+  tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e"));
+  tt_assert(!string_is_valid_ipv6_address("11:22::33:44:"));
+
+  done:
+  return;
+}
+
+static void
 test_util_writepid(void *arg)
 {
   (void) arg;
@@ -6387,9 +6123,9 @@ test_util_log_mallinfo(void *arg)
   } else {
     tt_u64_op(mem1, OP_LT, mem2);
   }
-#else
+#else /* !defined(HAVE_MALLINFO) */
   tt_skip();
-#endif
+#endif /* defined(HAVE_MALLINFO) */
  done:
   teardown_capture_of_logs();
   tor_free(log1);
@@ -6397,6 +6133,130 @@ test_util_log_mallinfo(void *arg)
   tor_free(mem);
 }
 
+static void
+test_util_map_anon(void *arg)
+{
+  (void)arg;
+  char *ptr = NULL;
+  size_t sz = 16384;
+  unsigned inherit=0;
+
+  /* Basic checks. */
+  ptr = tor_mmap_anonymous(sz, 0, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 3;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz-2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 3);
+
+  /* Try again, with a private (non-swappable) mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+  /* Now let's test a drop-on-fork mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+}
+
+static void
+test_util_map_anon_nofork(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  /* The operating system doesn't support forking. */
+  tt_skip();
+ done:
+  ;
+#else /* !defined(_WIN32) */
+  /* We have the right OS support.  We're going to try marking the buffer as
+   * either zero-on-fork or as drop-on-fork, whichever is supported.  Then we
+   * will fork and send a byte back to the parent process.  This will either
+   * crash, or send zero. */
+
+  char *ptr = NULL;
+  const char TEST_VALUE = 0xd0;
+  size_t sz = 16384;
+  int pipefd[2] = {-1, -1};
+  unsigned inherit=0;
+
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  memset(ptr, (uint8_t)TEST_VALUE, sz);
+
+  tt_int_op(0, OP_EQ, pipe(pipefd));
+  pid_t child = fork();
+  if (child == 0) {
+    /* We're in the child. */
+    close(pipefd[0]);
+    ssize_t r = write(pipefd[1], &ptr[sz-1], 1); /* This may crash. */
+    close(pipefd[1]);
+    if (r < 0)
+      exit(1);
+    exit(0);
+  }
+  tt_int_op(child, OP_GT, 0);
+  /* In the parent. */
+  close(pipefd[1]);
+  pipefd[1] = -1;
+  char buf[1];
+  ssize_t r = read(pipefd[0], buf, 1);
+
+  if (inherit == INHERIT_RES_ZERO) {
+    // We should be seeing clear-on-fork behavior.
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, 0); // that byte should be zero.
+  } else if (inherit == INHERIT_RES_DROP) {
+    // We should be seeing noinherit behavior.
+    tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed.
+  } else {
+    // noinherit isn't implemented.
+    tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, TEST_VALUE); // that byte should be TEST_VALUE.
+  }
+
+  int ws;
+  waitpid(child, &ws, 0);
+
+#ifndef NOINHERIT_CAN_FAIL
+  /* Only if NOINHERIT_CAN_FAIL should it be possible for us to get
+   * INHERIT_KEEP behavior in this case. */
+  tt_int_op(inherit, OP_NE, INHERIT_RES_KEEP);
+#else
+  if (inherit == INHERIT_RES_KEEP) {
+    /* Call this test "skipped", not "passed", since noinherit wasn't
+     * implemented. */
+    tt_skip();
+  }
+#endif /* !defined(NOINHERIT_CAN_FAIL) */
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+  if (pipefd[0] >= 0) {
+    close(pipefd[0]);
+  }
+  if (pipefd[1] >= 0) {
+    close(pipefd[1]);
+  }
+#endif /* defined(_WIN32) */
+}
+
 #define UTIL_LEGACY(name)                                               \
   { #name, test_util_ ## name , 0, NULL, NULL }
 
@@ -6490,12 +6350,8 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(nowrap_math, 0),
   UTIL_TEST(num_cpus, 0),
   UTIL_TEST_WIN_ONLY(load_win_lib, 0),
-  UTIL_TEST_NO_WIN(exit_status, 0),
-  UTIL_TEST_NO_WIN(string_from_pipe, 0),
   UTIL_TEST(format_hex_number, 0),
   UTIL_TEST(format_dec_number, 0),
-  UTIL_TEST(join_win_cmdline, 0),
-  UTIL_TEST(split_lines, 0),
   UTIL_TEST(n_bits_set, 0),
   UTIL_TEST(eat_whitespace, 0),
   UTIL_TEST(sl_new_from_text_lines, 0),
@@ -6512,6 +6368,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(mathlog, 0),
   UTIL_TEST(fraction, 0),
   UTIL_TEST(weak_random, 0),
+  { "tor_isinf", test_tor_isinf, TT_FORK, NULL, NULL },
   { "socket_ipv4", test_util_socket, TT_FORK, &passthrough_setup,
     (void*)"4" },
   { "socket_ipv6", test_util_socket, TT_FORK,
@@ -6524,6 +6381,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(hostname_validation, 0),
   UTIL_TEST(dest_validation_edgecase, 0),
   UTIL_TEST(ipv4_validation, 0),
+  UTIL_TEST(ipv6_validation, 0),
   UTIL_TEST(writepid, 0),
   UTIL_TEST(get_avail_disk_space, 0),
   UTIL_TEST(touch_file, 0),
@@ -6536,5 +6394,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(htonll, 0),
   UTIL_TEST(get_unquoted_path, 0),
   UTIL_TEST(log_mallinfo, 0),
+  UTIL_TEST(map_anon, 0),
+  UTIL_TEST(map_anon_nofork, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c
index d344d0e95c..2859da66b2 100644
--- a/src/test/test_util_format.c
+++ b/src/test/test_util_format.c
@@ -346,7 +346,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswqmjs";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 10);
     tt_str_op(expected, OP_EQ, dst);
   }
 
@@ -357,7 +357,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswq";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 8);
     tt_mem_op(expected, OP_EQ, dst, strlen(expected));
   }
 
@@ -367,7 +367,7 @@ test_util_format_base32_decode(void *arg)
     ret = base32_decode(dst, real_dstlen, "#abcde", 6);
     tt_int_op(ret, OP_EQ, -1);
     /* Make sure the destination buffer has been zeroed even on error. */
-    tt_int_op(tor_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
+    tt_int_op(fast_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
   }
 
  done:
@@ -392,10 +392,13 @@ test_util_format_encoded_size(void *arg)
 
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0);
     tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
+
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i,
                   BASE64_ENCODE_MULTILINE);
     tt_int_op(strlen(outbuf), OP_EQ,
               base64_encode_size(i, BASE64_ENCODE_MULTILINE));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
   }
 
  done:
@@ -417,4 +420,3 @@ struct testcase_t util_format_tests[] = {
   { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c
deleted file mode 100644
index 29e30eaa11..0000000000
--- a/src/test/test_util_slow.c
+++ /dev/null
@@ -1,396 +0,0 @@
-/* Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#define UTIL_PRIVATE
-#define SUBPROCESS_PRIVATE
-#include "lib/crypt_ops/crypto_cipher.h"
-#include "lib/log/log.h"
-#include "lib/process/subprocess.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/printf.h"
-#include "lib/time/compat_time.h"
-#include "test/test.h"
-
-#include <errno.h>
-#include <string.h>
-
-#ifndef BUILDDIR
-#define BUILDDIR "."
-#endif
-
-#ifdef _WIN32
-#define notify_pending_waitpid_callbacks() STMT_NIL
-#define TEST_CHILD "test-child.exe"
-#define EOL "\r\n"
-#else
-#define TEST_CHILD (BUILDDIR "/src/test/test-child")
-#define EOL "\n"
-#endif /* defined(_WIN32) */
-
-#ifdef _WIN32
-/* I've assumed Windows doesn't have the gap between fork and exec
- * that causes the race condition on unix-like platforms */
-#define MATCH_PROCESS_STATUS(s1,s2)         ((s1) == (s2))
-
-#else /* !(defined(_WIN32)) */
-/* work around a race condition of the timing of SIGCHLD handler updates
- * to the process_handle's fields, and checks of those fields
- *
- * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to
- * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */
-#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING  (PROCESS_STATUS_RUNNING+1)
-#define IS_RUNNING_OR_NOTRUNNING(s)           \
-  ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING)
-/* well, this is ugly */
-#define MATCH_PROCESS_STATUS(s1,s2)           \
-  (  (s1) == (s2)                                         \
-     ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s2))                  \
-     ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s1)))
-
-#endif /* defined(_WIN32) */
-
-/** Helper function for testing tor_spawn_background */
-static void
-run_util_spawn_background(const char *argv[], const char *expected_out,
-                          const char *expected_err, int expected_exit,
-                          int expected_status)
-{
-  int retval, exit_code;
-  ssize_t pos;
-  process_handle_t *process_handle=NULL;
-  char stdout_buf[100], stderr_buf[100];
-  int status;
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  notify_pending_waitpid_callbacks();
-
-  /* the race condition doesn't affect status,
-   * because status isn't updated by the SIGCHLD handler,
-   * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, status));
-  if (status == PROCESS_STATUS_ERROR) {
-    tt_ptr_op(process_handle, OP_EQ, NULL);
-    return;
-  }
-
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* When a spawned process forks, fails, then exits very quickly,
-   * (this typically occurs when exec fails)
-   * there is a race condition between the SIGCHLD handler
-   * updating the process_handle's fields, and this test
-   * checking the process status in those fields.
-   * The SIGCHLD update can occur before or after the code below executes.
-   * This causes intermittent failures in spawn_background_fail(),
-   * typically when the machine is under load.
-   * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */
-
-  /* the race condition affects the change in
-   * process_handle->status from RUNNING to NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status));
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  /* the race condition affects the change in
-   * process_handle->waitpid_cb to NULL,
-   * so we skip the check if expected_status is ambiguous,
-   * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(process_handle->waitpid_cb != NULL
-              || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING);
-#endif /* !defined(_WIN32) */
-
-#ifdef _WIN32
-  tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE);
-#else
-  tt_assert(process_handle->stdout_pipe >= 0);
-  tt_assert(process_handle->stderr_pipe >= 0);
-  tt_assert(process_handle->stdin_pipe >= 0);
-#endif /* defined(_WIN32) */
-
-  /* Check stdout */
-  pos = tor_read_all_from_process_stdout(process_handle, stdout_buf,
-                                         sizeof(stdout_buf) - 1);
-  tt_assert(pos >= 0);
-  stdout_buf[pos] = '\0';
-  tt_int_op(strlen(expected_out),OP_EQ, pos);
-  tt_str_op(expected_out,OP_EQ, stdout_buf);
-
-  notify_pending_waitpid_callbacks();
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-  // TODO: Make test-child exit with something other than 0
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
-  notify_pending_waitpid_callbacks();
-
- done:
-  if (process_handle)
-    tor_process_handle_destroy(process_handle, 1);
-}
-
-/** Check that we can launch a process and read the output */
-static void
-test_util_spawn_background_ok(void *ptr)
-{
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL;
-  const char *expected_err = "ERR"EOL;
-
-  (void)ptr;
-
-  run_util_spawn_background(argv, expected_out, expected_err, 0,
-      PROCESS_STATUS_RUNNING);
-}
-
-/** Check that failing to find the executable works as expected */
-static void
-test_util_spawn_background_fail(void *ptr)
-{
-  const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL};
-  const char *expected_err = "";
-  char expected_out[1024];
-  char code[32];
-#ifdef _WIN32
-  const int expected_status = PROCESS_STATUS_ERROR;
-#else
-  /* TODO: Once we can signal failure to exec, set this to be
-   * PROCESS_STATUS_RUNNING_OR_ERROR */
-  const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING;
-#endif /* defined(_WIN32) */
-
-  memset(expected_out, 0xf0, sizeof(expected_out));
-  memset(code, 0xf0, sizeof(code));
-
-  (void)ptr;
-
-  tor_snprintf(code, sizeof(code), "%x/%x",
-    9 /* CHILD_STATE_FAILEXEC */ , ENOENT);
-  tor_snprintf(expected_out, sizeof(expected_out),
-    "ERR: Failed to spawn background process - code %s\n", code);
-
-  run_util_spawn_background(argv, expected_out, expected_err, 255,
-      expected_status);
-}
-
-/** Test that reading from a handle returns a partial read rather than
- * blocking */
-static void
-test_util_spawn_background_partial_read_impl(int exit_early)
-{
-  const int expected_exit = 0;
-  const int expected_status = PROCESS_STATUS_RUNNING;
-
-  int retval, exit_code;
-  ssize_t pos = -1;
-  process_handle_t *process_handle=NULL;
-  int status;
-  char stdout_buf[100], stderr_buf[100];
-
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL,
-                                 "DONE" EOL,
-                                 NULL };
-  const char *expected_err = "ERR" EOL;
-
-#ifndef _WIN32
-  int eof = 0;
-#endif
-  int expected_out_ctr;
-
-  if (exit_early) {
-    argv[1] = "--hang";
-    expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL;
-  }
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-  tt_int_op(expected_status,OP_EQ, status);
-  tt_assert(process_handle);
-  tt_int_op(expected_status,OP_EQ, process_handle->status);
-
-  /* Check stdout */
-  for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) {
-#ifdef _WIN32
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL);
-#else
-    /* Check that we didn't read the end of file last time */
-    tt_assert(!eof);
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL, &eof);
-#endif /* defined(_WIN32) */
-    log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos);
-
-    /* We would have blocked, keep on trying */
-    if (0 == pos)
-      continue;
-
-    tt_assert(pos > 0);
-    stdout_buf[pos] = '\0';
-    tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf);
-    tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos);
-    expected_out_ctr++;
-  }
-
-  if (exit_early) {
-    tor_process_handle_destroy(process_handle, 1);
-    process_handle = NULL;
-    goto done;
-  }
-
-  /* The process should have exited without writing more */
-#ifdef _WIN32
-  pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                            sizeof(stdout_buf) - 1,
-                            process_handle);
-  tt_int_op(0,OP_EQ, pos);
-#else /* !(defined(_WIN32)) */
-  if (!eof) {
-    /* We should have got all the data, but maybe not the EOF flag */
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1,
-                              process_handle, &eof);
-    tt_int_op(0,OP_EQ, pos);
-    tt_assert(eof);
-  }
-  /* Otherwise, we got the EOF on the last read */
-#endif /* defined(_WIN32) */
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-
-  // TODO: Make test-child exit with something other than 0
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-static void
-test_util_spawn_background_partial_read(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(0);
-}
-
-static void
-test_util_spawn_background_exit_early(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(1);
-}
-
-static void
-test_util_spawn_background_waitpid_notify(void *arg)
-{
-  int retval, exit_code;
-  process_handle_t *process_handle=NULL;
-  int status;
-  int ms_timer;
-
-  const char *argv[] = {TEST_CHILD, "--fast", NULL};
-
-  (void) arg;
-
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* We're not going to look at the stdout/stderr output this time. Instead,
-   * we're testing whether notify_pending_waitpid_calbacks() can report the
-   * process exit (on unix) and/or whether tor_get_exit_code() can notice it
-   * (on windows) */
-
-#ifndef _WIN32
-  ms_timer = 30*1000;
-  tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL);
-  while (process_handle->waitpid_cb && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-    notify_pending_waitpid_callbacks();
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif /* !defined(_WIN32) */
-
-  ms_timer = 30*1000;
-  while (((retval = tor_get_exit_code(process_handle, 0, &exit_code))
-                == PROCESS_EXIT_RUNNING) && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-
-  tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-#undef TEST_CHILD
-#undef EOL
-
-#undef MATCH_PROCESS_STATUS
-
-#ifndef _WIN32
-#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING
-#undef IS_RUNNING_OR_NOTRUNNING
-#endif
-
-#define UTIL_TEST(name, flags)                          \
-  { #name, test_util_ ## name, flags, NULL, NULL }
-
-struct testcase_t slow_util_tests[] = {
-  UTIL_TEST(spawn_background_ok, 0),
-  UTIL_TEST(spawn_background_fail, 0),
-  UTIL_TEST(spawn_background_partial_read, 0),
-  UTIL_TEST(spawn_background_exit_early, 0),
-  UTIL_TEST(spawn_background_waitpid_notify, 0),
-  END_OF_TESTCASES
-};
diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c
new file mode 100644
index 0000000000..c8111ea5df
--- /dev/null
+++ b/src/test/test_voting_flags.c
@@ -0,0 +1,191 @@
+/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define VOTEFLAGS_PRIVATE
+
+#include "core/or/or.h"
+
+#include "feature/dirauth/voteflags.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "app/config/config.h"
+
+#include "test/test.h"
+
+typedef struct {
+  time_t now;
+  routerinfo_t ri;
+  node_t node;
+
+  routerstatus_t expected;
+} flag_vote_test_cfg_t;
+
+static void
+setup_cfg(flag_vote_test_cfg_t *c)
+{
+  memset(c, 0, sizeof(*c));
+
+  c->now = approx_time();
+
+  c->ri.nickname = (char *) "testing100";
+  strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname));
+
+  memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN);
+  memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN);
+
+  c->ri.cache_info.published_on = c->now - 100;
+  c->expected.published_on = c->now - 100;
+
+  c->ri.addr = 0x7f010105;
+  c->expected.addr = 0x7f010105;
+  c->ri.or_port = 9090;
+  c->expected.or_port = 9090;
+
+  tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6);
+  tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6);
+
+  // By default we have no loaded information about stability or speed,
+  // so we'll default to voting "yeah sure." on these two.
+  c->expected.is_fast = 1;
+  c->expected.is_stable = 1;
+}
+
+static bool
+check_result(flag_vote_test_cfg_t *c)
+{
+  bool result = false;
+  routerstatus_t rs;
+  memset(&rs, 0, sizeof(rs));
+  dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0);
+
+  tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on);
+  tt_str_op(rs.nickname, OP_EQ, c->expected.nickname);
+
+  // identity_digest and descriptor_digest are not set here.
+
+  tt_uint_op(rs.addr, OP_EQ, c->expected.addr);
+  tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port);
+  tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port);
+
+  tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr));
+  tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport);
+
+#define FLAG(flagname) \
+  tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname)
+
+  FLAG(is_authority);
+  FLAG(is_exit);
+  FLAG(is_stable);
+  FLAG(is_fast);
+  FLAG(is_flagged_running);
+  FLAG(is_named);
+  FLAG(is_unnamed);
+  FLAG(is_valid);
+  FLAG(is_possible_guard);
+  FLAG(is_bad_exit);
+  FLAG(is_hs_dir);
+  FLAG(is_v2_dir);
+  FLAG(is_staledesc);
+  FLAG(has_bandwidth);
+  FLAG(has_exitsummary);
+  FLAG(bw_is_unmeasured);
+
+  result = true;
+
+ done:
+  return result;
+}
+
+static void
+test_voting_flags_minimal(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  (void) check_result(cfg);
+}
+
+static void
+test_voting_flags_ipv6(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+
+  tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->ri.ipv6_orport = 9091;
+  // no change in expected results, since we aren't set up with ipv6
+  // connectivity.
+  if (!check_result(cfg))
+    goto done;
+
+  get_options_mutable()->AuthDirHasIPv6Connectivity = 1;
+  // no change in expected results, since last_reachable6 won't be set.
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->node.last_reachable6 = cfg->now - 10;
+  // now that lastreachable6 is set, we expect to see the result.
+  tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->expected.ipv6_orport = 9091;
+  if (!check_result(cfg))
+    goto done;
+ done:
+  ;
+}
+
+static void
+test_voting_flags_staledesc(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  time_t now = cfg->now;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  // no change in expectations for is_staledesc
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.is_staledesc = 1;
+  if (!check_result(cfg))
+    goto done;
+
+ done:
+  ;
+}
+
+static void *
+setup_voting_flags_test(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  setup_cfg(cfg);
+  return cfg;
+}
+
+static int
+teardown_voting_flags_test(const struct testcase_t *testcase, void *arg)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = arg;
+  tor_free(cfg);
+  return 1;
+}
+
+static const struct testcase_setup_t voting_flags_setup = {
+  .setup_fn = setup_voting_flags_test,
+  .cleanup_fn = teardown_voting_flags_test,
+};
+
+#define T(name,flags)                                   \
+  { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL }
+
+struct testcase_t voting_flags_tests[] = {
+  T(minimal, 0),
+  T(ipv6, TT_FORK),
+  // TODO: Add more of these tests.
+  T(staledesc, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c
index c58634da5c..ba478a45a4 100644
--- a/src/test/test_workqueue.c
+++ b/src/test/test_workqueue.c
@@ -63,7 +63,7 @@ mark_handled(int serial)
   tor_assert(! bitarray_is_set(handled, serial));
   bitarray_set(handled, serial);
   tor_mutex_release(&bitmap_mutex);
-#else /* !(defined(TRACK_RESPONSES)) */
+#else /* !defined(TRACK_RESPONSES) */
   (void)serial;
 #endif /* defined(TRACK_RESPONSES) */
 }
diff --git a/src/test/test_workqueue_cancel.sh b/src/test/test_workqueue_cancel.sh
index f7c663171e..e50b884f26 100755
--- a/src/test/test_workqueue_cancel.sh
+++ b/src/test/test_workqueue_cancel.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue -C 1
+"${builddir:-.}/src/test/test_workqueue" -C 1
 
diff --git a/src/test/test_workqueue_efd.sh b/src/test/test_workqueue_efd.sh
index 4d89396819..592841fc91 100755
--- a/src/test/test_workqueue_efd.sh
+++ b/src/test/test_workqueue_efd.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_efd2.sh b/src/test/test_workqueue_efd2.sh
index 7cfff45ff3..4cf1b76cbe 100755
--- a/src/test/test_workqueue_efd2.sh
+++ b/src/test/test_workqueue_efd2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_pipe.sh b/src/test/test_workqueue_pipe.sh
index afcef87853..fc3ef34c6c 100755
--- a/src/test/test_workqueue_pipe.sh
+++ b/src/test/test_workqueue_pipe.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-socketpair
diff --git a/src/test/test_workqueue_pipe2.sh b/src/test/test_workqueue_pipe2.sh
index a20a1427e0..7f19ea880d 100755
--- a/src/test/test_workqueue_pipe2.sh
+++ b/src/test/test_workqueue_pipe2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_socketpair.sh b/src/test/test_workqueue_socketpair.sh
index 76af79746d..1ee1776447 100755
--- a/src/test/test_workqueue_socketpair.sh
+++ b/src/test/test_workqueue_socketpair.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-pipe
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index 2c9c4538b9..06a6f312ad 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -12,6 +12,7 @@
 #include "orconfig.h"
 #include "core/or/or.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -25,6 +26,8 @@
 #include "lib/compress/compress.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
+#include "app/main/subsysmgr.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -86,7 +89,7 @@ setup_directory(void)
                  (int)getpid(), rnd32);
     r = mkdir(temp_dir);
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s",
                (int) getpid(), rnd32);
   r = mkdir(temp_dir, 0700);
@@ -230,17 +233,17 @@ void
 tinytest_prefork(void)
 {
   free_pregenerated_keys();
-  crypto_prefork();
+  subsystems_prefork();
 }
 void
 tinytest_postfork(void)
 {
-  crypto_postfork();
+  subsystems_postfork();
   init_pregenerated_keys();
 }
 
 static void
-log_callback_failure(int severity, uint32_t domain, const char *msg)
+log_callback_failure(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)msg;
   if (severity == LOG_ERR || (domain & LD_BUG)) {
@@ -259,24 +262,15 @@ main(int c, const char **v)
   int loglevel = LOG_ERR;
   int accel_crypto = 0;
 
-  /* We must initialise logs before we call tor_assert() */
-  init_logging(1);
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
 
-  update_approx_time(time(NULL));
   options = options_new();
-  tor_threads_init();
-  tor_compress_init();
-
-  network_init();
-
-  monotime_init();
 
   struct tor_libevent_cfg cfg;
   memset(&cfg, 0, sizeof(cfg));
   tor_libevent_initialize(&cfg);
 
   control_initialize_event_queue();
-  configure_backtrace_handler(get_version());
 
   for (i_out = i = 1; i < c; ++i) {
     if (!strcmp(v[i], "--warn")) {
@@ -301,7 +295,7 @@ main(int c, const char **v)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
   {
@@ -309,9 +303,10 @@ main(int c, const char **v)
     log_severity_list_t s;
     memset(&s, 0, sizeof(s));
     set_log_severity_config(LOG_ERR, LOG_ERR, &s);
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_callback_log(&s, log_callback_failure);
   }
+  flush_log_messages_from_startup();
   init_protocol_warning_severity_level();
 
   options->command = CMD_RUN_UNITTESTS;
@@ -367,8 +362,6 @@ main(int c, const char **v)
 
   free_pregenerated_keys();
 
-  crypto_global_cleanup();
-
   if (have_failed)
     return 1;
   else
diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c
index 0f22d4e01b..727449e6eb 100644
--- a/src/test/testing_rsakeys.c
+++ b/src/test/testing_rsakeys.c
@@ -448,7 +448,8 @@ static int next_key_idx_2048;
 static crypto_pk_t *
 pk_generate_internal(int bits)
 {
-  tor_assert(bits == 2048 || bits == 1024);
+  tor_assertf(bits == 2048 || bits == 1024,
+             "Wrong key size: %d", bits);
 
 #ifdef USE_PREGENERATED_RSA_KEYS
   int *idxp;
@@ -467,7 +468,7 @@ pk_generate_internal(int bits)
   *idxp += crypto_rand_int_range(1,3);
   *idxp %= n_pregen;
   return crypto_pk_dup_key(pregen_array[*idxp]);
-#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */
+#else /* !defined(USE_PREGENERATED_RSA_KEYS) */
   crypto_pk_t *result;
   int res;
   result = crypto_pk_new();
diff --git a/src/test/zero_length_keys.sh b/src/test/zero_length_keys.sh
index 5635bdfd89..1702d11245 100755
--- a/src/test/zero_length_keys.sh
+++ b/src/test/zero_length_keys.sh
@@ -19,7 +19,7 @@
 #   3: a command failed - the test could not be completed
 #
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   echo "Usage: ${0} PATH_TO_TOR [-z|-d|-e]"
   exit 1
 elif [ $# -eq 1 ]; then
@@ -31,7 +31,7 @@ else #[$# -gt 1 ]; then
   shift
 fi
 
-DATA_DIR=`mktemp -d -t tor_zero_length_keys.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_zero_length_keys.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -40,7 +40,7 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 touch "$DATA_DIR"/empty_torrc
 touch "$DATA_DIR"/empty_defaults_torrc
diff --git a/src/tools/include.am b/src/tools/include.am
index f7aa7e0d1e..72dfe6017c 100644
--- a/src/tools/include.am
+++ b/src/tools/include.am
@@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve
 endif
 
 src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c
-src_tools_tor_resolve_LDFLAGS =
+src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_LIBS) \
+        $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\
 	$(rust_ldadd) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@
 
@@ -15,8 +17,11 @@ if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_cov_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_TESTING_LIBS) \
+	$(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@
 endif
 
diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c
index 25113420df..ea96f41dbf 100644
--- a/src/tools/tor-gencert.c
+++ b/src/tools/tor-gencert.c
@@ -31,7 +31,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/err.h>
 
 ENABLE_GCC_WARNING(redundant-decls)
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <errno.h>
 
diff --git a/src/tools/tor-print-ed-signing-cert.c b/src/tools/tor-print-ed-signing-cert.c
index 1f1a01ab5c..43a1d7bcbd 100644
--- a/src/tools/tor-print-ed-signing-cert.c
+++ b/src/tools/tor-print-ed-signing-cert.c
@@ -10,11 +10,13 @@
 #include "lib/cc/torint.h"  /* TOR_PRIdSZ */
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/malloc/malloc.h"
+#include "lib/encoding/time_fmt.h"
 
 int
 main(int argc, char **argv)
 {
   ed25519_cert_t *cert = NULL;
+  char rfc1123_buf[RFC1123_TIME_LEN+1] = "";
 
   if (argc != 2) {
     fprintf(stderr, "Usage:\n");
@@ -59,6 +61,11 @@ main(int argc, char **argv)
 
   printf("Expires at: %s", ctime(&expires_at));
 
+  format_rfc1123_time(rfc1123_buf, expires_at);
+  printf("RFC 1123 timestamp: %s\n", rfc1123_buf);
+
+  printf("UNIX timestamp: %ld\n", (long int)expires_at);
+
   ed25519_cert_free(cert);
 
   return 0;
diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c
index 6a84abe557..5d97696c18 100644
--- a/src/tools/tor-resolve.c
+++ b/src/tools/tor-resolve.c
@@ -15,6 +15,7 @@
 #include "lib/string/util_string.h"
 
 #include "lib/net/socks5_status.h"
+#include "trunnel/socks5.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -49,68 +50,204 @@
 
 static void usage(void) ATTR_NORETURN;
 
+/**
+ * Set <b>out</b> to a pointer to newly allocated buffer containing
+ * SOCKS4a RESOLVE request with <b>username</b> and <b>hostname</b>.
+ * Return number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks4a_resolve_request(uint8_t **out,
+                              const char *username,
+                              const char *hostname)
+{
+  tor_assert(out);
+  tor_assert(username);
+  tor_assert(hostname);
+
+  const char *errmsg = NULL;
+  uint8_t *output = NULL;
+  socks4_client_request_t *rq = socks4_client_request_new();
+
+  socks4_client_request_set_version(rq, 4);
+  socks4_client_request_set_command(rq, CMD_RESOLVE);
+  socks4_client_request_set_port(rq, 0);
+  socks4_client_request_set_addr(rq, 0x00000001u);
+  socks4_client_request_set_username(rq, username);
+  socks4_client_request_set_socks4a_addr_hostname(rq, hostname);
+
+  errmsg = socks4_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks4_client_request_encoded_len(rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encoded_len failed";
+    goto cleanup;
+  }
+
+  output = tor_malloc(encoded_len);
+  memset(output, 0, encoded_len);
+
+  encoded_len = socks4_client_request_encode(output, encoded_len, rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encode failed";
+    goto cleanup;
+  }
+
+  *out = output;
+
+ cleanup:
+  socks4_client_request_free(rq);
+  if (errmsg) {
+    log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg);
+    *out = NULL;
+    tor_free(output);
+  }
+  return errmsg ? -1 : encoded_len;
+}
+
+#define SOCKS5_ATYPE_HOSTNAME 0x03
+#define SOCKS5_ATYPE_IPV4     0x01
+#define SOCKS5_ATYPE_IPV6     0x04
+
+/**
+ * Set <b>out</b> to pointer to newly allocated buffer containing
+ * SOCKS5 RESOLVE/RESOLVE_PTR request with given <b>hostname<b>.
+ * Generate a reverse request if <b>reverse</b> is true.
+ * Return the number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks5_resolve_request(uint8_t **out,
+                             const char *hostname,
+                             int reverse)
+{
+  const char *errmsg = NULL;
+  uint8_t *outbuf = NULL;
+  int is_ip_address;
+  tor_addr_t addr;
+  size_t addrlen;
+  int ipv6;
+  is_ip_address = tor_addr_parse(&addr, hostname) != -1;
+  if (!is_ip_address && reverse) {
+    log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
+    return -1;
+  }
+  ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
+  addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
+  if (addrlen > UINT8_MAX) {
+    log_err(LD_GENERAL, "Hostname is too long!");
+    return -1;
+  }
+
+  socks5_client_request_t *rq = socks5_client_request_new();
+
+  socks5_client_request_set_version(rq, 5);
+  /* RESOLVE_PTR or RESOLVE */
+  socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR :
+                                                  CMD_RESOLVE);
+  socks5_client_request_set_reserved(rq, 0);
+
+  uint8_t atype = SOCKS5_ATYPE_HOSTNAME;
+  if (reverse)
+    atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4;
+
+  socks5_client_request_set_atype(rq, atype);
+
+  switch (atype) {
+    case SOCKS5_ATYPE_IPV4: {
+      socks5_client_request_set_dest_addr_ipv4(rq,
+                                        tor_addr_to_ipv4h(&addr));
+    } break;
+    case SOCKS5_ATYPE_IPV6: {
+      uint8_t *ipv6_array =
+        socks5_client_request_getarray_dest_addr_ipv6(rq);
+
+      tor_assert(ipv6_array);
+
+      memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16);
+    } break;
+
+    case SOCKS5_ATYPE_HOSTNAME: {
+      domainname_t *dn = domainname_new();
+      domainname_set_len(dn, addrlen - 1);
+      domainname_setlen_name(dn, addrlen - 1);
+      char *dn_buf = domainname_getarray_name(dn);
+      memcpy(dn_buf, hostname, addrlen - 1);
+
+      errmsg = domainname_check(dn);
+
+      if (errmsg) {
+        domainname_free(dn);
+        goto cleanup;
+      } else {
+        socks5_client_request_set_dest_addr_domainname(rq, dn);
+      }
+    } break;
+    default:
+      tor_assert_unreached();
+      break;
+  }
+
+  socks5_client_request_set_dest_port(rq, 0);
+
+  errmsg = socks5_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks5_client_request_encoded_len(rq);
+  if (encoded_len < 0) {
+    errmsg = "Cannot predict encoded length";
+    goto cleanup;
+  }
+
+  outbuf = tor_malloc(encoded_len);
+  memset(outbuf, 0, encoded_len);
+
+  encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq);
+  if (encoded_len < 0) {
+    errmsg = "encoding failed";
+    goto cleanup;
+  }
+
+  *out = outbuf;
+
+ cleanup:
+  socks5_client_request_free(rq);
+  if (errmsg) {
+    tor_free(outbuf);
+    log_err(LD_NET, "build_socks5_resolve_request failed with error: %s",
+            errmsg);
+  }
+
+  return errmsg ? -1 : encoded_len;
+}
+
 /** Set *<b>out</b> to a newly allocated SOCKS4a resolve request with
  * <b>username</b> and <b>hostname</b> as provided.  Return the number
  * of bytes in the request. */
 static ssize_t
-build_socks_resolve_request(char **out,
+build_socks_resolve_request(uint8_t **out,
                             const char *username,
                             const char *hostname,
                             int reverse,
                             int version)
 {
-  size_t len = 0;
   tor_assert(out);
   tor_assert(username);
   tor_assert(hostname);
 
+  tor_assert(version == 4 || version == 5);
+
   if (version == 4) {
-    len = 8 + strlen(username) + 1 + strlen(hostname) + 1;
-    *out = tor_malloc(len);
-    (*out)[0] = 4;      /* SOCKS version 4 */
-    (*out)[1] = '\xF0'; /* Command: resolve. */
-    set_uint16((*out)+2, htons(0)); /* port: 0. */
-    set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */
-    memcpy((*out)+8, username, strlen(username)+1);
-    memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1);
+    return build_socks4a_resolve_request(out, username, hostname);
   } else if (version == 5) {
-    int is_ip_address;
-    tor_addr_t addr;
-    size_t addrlen;
-    int ipv6;
-    is_ip_address = tor_addr_parse(&addr, hostname) != -1;
-    if (!is_ip_address && reverse) {
-      log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
-      return -1;
-    }
-    ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
-    addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
-    if (addrlen > UINT8_MAX) {
-      log_err(LD_GENERAL, "Hostname is too long!");
-      return -1;
-    }
-    len = 6 + addrlen;
-    *out = tor_malloc(len);
-    (*out)[0] = 5; /* SOCKS version 5 */
-    (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */
-    (*out)[2] = 0; /* reserved. */
-    if (reverse) {
-      (*out)[3] = ipv6 ? 4 : 1;
-      if (ipv6)
-        memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16);
-      else
-        set_uint32((*out)+4, tor_addr_to_ipv4n(&addr));
-    } else {
-      (*out)[3] = 3;
-      (*out)[4] = (char)(uint8_t)(addrlen - 1);
-      memcpy((*out)+5, hostname, addrlen - 1);
-    }
-    set_uint16((*out)+4+addrlen, 0); /* port */
-  } else {
-    tor_assert(0);
+    return build_socks5_resolve_request(out, hostname, reverse);
   }
 
-  return len;
+  tor_assert_unreached();
+  return -1;
 }
 
 static void
@@ -134,34 +271,50 @@ parse_socks4a_resolve_response(const char *hostname,
                                const char *response, size_t len,
                                tor_addr_t *addr_out)
 {
+  int result = 0;
   uint8_t status;
   tor_assert(response);
   tor_assert(addr_out);
 
-  if (len < RESPONSE_LEN_4) {
+  socks4_server_reply_t *reply;
+
+  ssize_t parsed = socks4_server_reply_parse(&reply,
+                                             (const uint8_t *)response,
+                                             len);
+
+  if (parsed == -1) {
+    log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response");
+    result = -1; goto cleanup;
+  }
+
+  if (parsed == -2) {
     log_warn(LD_PROTOCOL,"Truncated socks response.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  if (((uint8_t)response[0])!=0) { /* version: 0 */
+
+  if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */
     log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  status = (uint8_t)response[1];
-  if (get_uint16(response+2)!=0) { /* port: 0 */
+  if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */
     log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
+  status = socks4_server_reply_get_status(reply);
   if (status != 90) {
     log_warn(LD_NET,"Got status response '%d': socks request failed.", status);
     if (!strcasecmpend(hostname, ".onion")) {
       onion_warning(hostname);
-      return -1;
+      result = -1; goto cleanup;
     }
-    return -1;
+    result = -1; goto cleanup;
   }
 
-  tor_addr_from_ipv4n(addr_out, get_uint32(response+4));
-  return 0;
+  tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply));
+
+ cleanup:
+  socks4_server_reply_free(reply);
+  return result;
 }
 
 /* It would be nice to let someone know what SOCKS5 issue a user may have */
@@ -205,7 +358,7 @@ do_resolve(const char *hostname,
   int s = -1;
   struct sockaddr_storage ss;
   socklen_t socklen;
-  char *req = NULL;
+  uint8_t *req = NULL;
   ssize_t len = 0;
 
   tor_assert(hostname);
@@ -230,23 +383,58 @@ do_resolve(const char *hostname,
   }
 
   if (version == 5) {
-    char method_buf[2];
-    if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) {
+    socks5_client_version_t *v = socks5_client_version_new();
+
+    socks5_client_version_set_version(v, 5);
+    socks5_client_version_set_n_methods(v, 1);
+    socks5_client_version_setlen_methods(v, 1);
+    socks5_client_version_set_methods(v, 0, 0x00);
+
+    tor_assert(!socks5_client_version_check(v));
+    ssize_t encoded_len = socks5_client_version_encoded_len(v);
+    tor_assert(encoded_len > 0);
+
+    uint8_t *buf = tor_malloc(encoded_len);
+    encoded_len = socks5_client_version_encode(buf, encoded_len, v);
+    tor_assert(encoded_len > 0);
+
+    socks5_client_version_free(v);
+
+    if (write_all_to_socket(s, (const char *)buf,
+                            encoded_len) != encoded_len) {
       log_err(LD_NET, "Error sending SOCKS5 method list.");
+      tor_free(buf);
+
       goto err;
     }
-    if (read_all_from_socket(s, method_buf, 2) != 2) {
+
+    tor_free(buf);
+
+    uint8_t method_buf[2];
+
+    if (read_all_from_socket(s, (char *)method_buf, 2) != 2) {
       log_err(LD_NET, "Error reading SOCKS5 methods.");
       goto err;
     }
-    if (method_buf[0] != '\x05') {
-      log_err(LD_NET, "Unrecognized socks version: %u",
-              (unsigned)method_buf[0]);
+
+    socks5_server_method_t *m;
+    ssize_t parsed = socks5_server_method_parse(&m, method_buf,
+                                                sizeof(method_buf));
+
+    if (parsed < 2) {
+      log_err(LD_NET, "Failed to parse SOCKS5 method selection "
+                      "message");
+      socks5_server_method_free(m);
       goto err;
     }
-    if (method_buf[1] != '\x00') {
+
+    uint8_t method = socks5_server_method_get_method(m);
+
+    socks5_server_method_free(m);
+
+    if (method != 0x00) {
       log_err(LD_NET, "Unrecognized socks authentication method: %u",
-              (unsigned)method_buf[1]);
+              (unsigned)method);
       goto err;
     }
   }
@@ -257,7 +445,7 @@ do_resolve(const char *hostname,
     tor_assert(!req);
     goto err;
   }
-  if (write_all_to_socket(s, req, len) != len) {
+  if (write_all_to_socket(s, (const char *)req, len) != len) {
     log_sock_error("sending SOCKS request", s);
     tor_free(req);
     goto err;
@@ -276,55 +464,59 @@ do_resolve(const char *hostname,
       goto err;
     }
   } else {
-    char reply_buf[16];
-    if (read_all_from_socket(s, reply_buf, 4) != 4) {
-      log_err(LD_NET, "Error reading SOCKS5 response.");
+    uint8_t reply_buf[512];
+
+    len = read_all_from_socket(s, (char *)reply_buf,
+                               sizeof(reply_buf));
+
+    socks5_server_reply_t *reply;
+
+    ssize_t parsed = socks5_server_reply_parse(&reply,
+                                               reply_buf,
+                                               len);
+    if (parsed == -1) {
+      log_err(LD_NET, "Failed parsing SOCKS5 response");
       goto err;
     }
-    if (reply_buf[0] != 5) {
-      log_err(LD_NET, "Bad SOCKS5 reply version.");
+
+    if (parsed == -2) {
+      log_err(LD_NET, "Truncated SOCKS5 response");
       goto err;
     }
+
     /* Give a user some useful feedback about SOCKS5 errors */
-    if (reply_buf[1] != 0) {
+    uint8_t reply_field = socks5_server_reply_get_reply(reply);
+    if (reply_field != 0) {
       log_warn(LD_NET,"Got SOCKS5 status response '%u': %s",
-               (unsigned)reply_buf[1],
-               socks5_reason_to_string(reply_buf[1]));
-      if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) {
+               (unsigned)reply_field,
+               socks5_reason_to_string(reply_field));
+      if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) {
         onion_warning(hostname);
       }
+
+      socks5_server_reply_free(reply);
       goto err;
     }
-    if (reply_buf[3] == 1) {
+
+    uint8_t atype = socks5_server_reply_get_atype(reply);
+
+    if (atype == SOCKS5_ATYPE_IPV4) {
       /* IPv4 address */
-      if (read_all_from_socket(s, reply_buf, 4) != 4) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf));
-    } else if (reply_buf[3] == 4) {
+      tor_addr_from_ipv4h(result_addr,
+        socks5_server_reply_get_bind_addr_ipv4(reply));
+    } else if (atype == SOCKS5_ATYPE_IPV6) {
       /* IPv6 address */
-      if (read_all_from_socket(s, reply_buf, 16) != 16) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv6_bytes(result_addr, reply_buf);
-    } else if (reply_buf[3] == 3) {
+      tor_addr_from_ipv6_bytes(result_addr,
+        (const char *)socks5_server_reply_getarray_bind_addr_ipv6(reply));
+    } else if (atype == SOCKS5_ATYPE_HOSTNAME) {
       /* Domain name */
-      size_t result_len;
-      if (read_all_from_socket(s, reply_buf, 1) != 1) {
-        log_err(LD_NET, "Error reading address_length in socks5 response.");
-        goto err;
-      }
-      result_len = *(uint8_t*)(reply_buf);
-      *result_hostname = tor_malloc(result_len+1);
-      if (read_all_from_socket(s, *result_hostname, result_len)
-          != (int) result_len) {
-        log_err(LD_NET, "Error reading hostname in socks5 response.");
-        goto err;
-      }
-      (*result_hostname)[result_len] = '\0';
+      domainname_t *dn =
+        socks5_server_reply_get_bind_addr_domainname(reply);
+
+      *result_hostname = tor_strdup(domainname_getstr_name(dn));
     }
+
+    socks5_server_reply_free(reply);
   }
 
   tor_close_socket(s);
diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c
index 59e6b38384..d96496e90c 100644
--- a/src/trunnel/channelpadding_negotiation.c
+++ b/src/trunnel/channelpadding_negotiation.c
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h
index fcfc232fea..3f96174f68 100644
--- a/src/trunnel/channelpadding_negotiation.h
+++ b/src/trunnel/channelpadding_negotiation.h
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c
new file mode 100644
index 0000000000..547818f2ec
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.c
@@ -0,0 +1,549 @@
+/* circpad_negotiation.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "circpad_negotiation.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int circpadnegotiation_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || circpadnegotiation_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+circpad_negotiate_t *
+circpad_negotiate_new(void)
+{
+  circpad_negotiate_t *val = trunnel_calloc(1, sizeof(circpad_negotiate_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiate_clear(circpad_negotiate_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiate_free(circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiate_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiate_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiate_get_version(const circpad_negotiate_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_command(const circpad_negotiate_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp)
+{
+  return inp->echo_request;
+}
+int
+circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->echo_request = val;
+  return 0;
+}
+const char *
+circpad_negotiate_check(const circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiate_encoded_len(const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiate_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+
+  /* Length of u8 echo_request IN [0, 1] */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiate_clear_errors(circpad_negotiate_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiate_encode(uint8_t *output, const size_t avail, const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiate_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiate_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 echo_request IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->echo_request));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiate_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiate_parse_into(circpad_negotiate_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 echo_request IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->echo_request = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    goto fail;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiate_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiate_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiate_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+circpad_negotiated_t *
+circpad_negotiated_new(void)
+{
+  circpad_negotiated_t *val = trunnel_calloc(1, sizeof(circpad_negotiated_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  val->response = CIRCPAD_RESPONSE_ERR;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiated_clear(circpad_negotiated_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiated_free(circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiated_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiated_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiated_get_version(const circpad_negotiated_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_command(const circpad_negotiated_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_response(const circpad_negotiated_t *inp)
+{
+  return inp->response;
+}
+int
+circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_RESPONSE_ERR || val == CIRCPAD_RESPONSE_OK))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->response = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+const char *
+circpad_negotiated_check(const circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiated_encoded_len(const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiated_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiated_clear_errors(circpad_negotiated_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiated_encode(uint8_t *output, const size_t avail, const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiated_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiated_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->response));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiated_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiated_parse_into(circpad_negotiated_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  CHECK_REMAINING(1, truncated);
+  obj->response = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiated_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiated_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiated_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h
new file mode 100644
index 0000000000..ba9155019e
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.h
@@ -0,0 +1,195 @@
+/* circpad_negotiation.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CIRCPAD_NEGOTIATION_H
+#define TRUNNEL_CIRCPAD_NEGOTIATION_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CIRCPAD_COMMAND_STOP 1
+#define CIRCPAD_COMMAND_START 2
+#define CIRCPAD_RESPONSE_OK 1
+#define CIRCPAD_RESPONSE_ERR 2
+#define CIRCPAD_MACHINE_CIRC_SETUP 1
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATE)
+struct circpad_negotiate_st {
+  uint8_t version;
+  uint8_t command;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  /** If true, send a relay_drop reply.. */
+  uint8_t echo_request;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiate_st circpad_negotiate_t;
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATED)
+struct circpad_negotiated_st {
+  uint8_t version;
+  uint8_t command;
+  uint8_t response;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiated_st circpad_negotiated_t;
+/** Return a newly allocated circpad_negotiate with all elements set
+ * to zero.
+ */
+circpad_negotiate_t *circpad_negotiate_new(void);
+/** Release all storage held by the circpad_negotiate in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void circpad_negotiate_free(circpad_negotiate_t *victim);
+/** Try to parse a circpad_negotiate from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * circpad_negotiate_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiate in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiate_encoded_len(const circpad_negotiate_t *obj);
+/** Try to encode the circpad_negotiate from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiate_encode(uint8_t *output, size_t avail, const circpad_negotiate_t *input);
+/** Check whether the internal state of the circpad_negotiate in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *circpad_negotiate_check(const circpad_negotiate_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiate_clear_errors(circpad_negotiate_t *obj);
+/** Return the value of the version field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_version(const circpad_negotiate_t *inp);
+/** Set the value of the version field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_command(const circpad_negotiate_t *inp);
+/** Set the value of the command field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp);
+/** Set the value of the machine_type field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the echo_request field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp);
+/** Set the value of the echo_request field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val);
+/** Return a newly allocated circpad_negotiated with all elements set
+ * to zero.
+ */
+circpad_negotiated_t *circpad_negotiated_new(void);
+/** Release all storage held by the circpad_negotiated in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void circpad_negotiated_free(circpad_negotiated_t *victim);
+/** Try to parse a circpad_negotiated from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated circpad_negotiated_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiated in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiated_encoded_len(const circpad_negotiated_t *obj);
+/** Try to encode the circpad_negotiated from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiated_encode(uint8_t *output, size_t avail, const circpad_negotiated_t *input);
+/** Check whether the internal state of the circpad_negotiated in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *circpad_negotiated_check(const circpad_negotiated_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiated_clear_errors(circpad_negotiated_t *obj);
+/** Return the value of the version field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_version(const circpad_negotiated_t *inp);
+/** Set the value of the version field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_command(const circpad_negotiated_t *inp);
+/** Set the value of the command field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the response field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_response(const circpad_negotiated_t *inp);
+/** Set the value of the response field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp'
+ */
+uint8_t circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp);
+/** Set the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val);
+
+
+#endif
diff --git a/src/trunnel/circpad_negotiation.trunnel b/src/trunnel/circpad_negotiation.trunnel
new file mode 100644
index 0000000000..abbc929cc5
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.trunnel
@@ -0,0 +1,44 @@
+/* These are the padding negotiation commands */
+const CIRCPAD_COMMAND_STOP = 1;
+const CIRCPAD_COMMAND_START = 2;
+
+/* Responses to commands */
+const CIRCPAD_RESPONSE_OK = 1;
+const CIRCPAD_RESPONSE_ERR = 2;
+
+/* Built-in machine types */
+
+/* 1) Machine that obscures circuit setup */
+const CIRCPAD_MACHINE_CIRC_SETUP = 1;
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiate {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+
+  /** If true, send a relay_drop reply.. */
+  // FIXME-MP-AP: Maybe we just say to transition to the first state
+  // here instead.. Also what about delay before responding?
+  u8 echo_request IN [0,1];
+};
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiated {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+  u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+};
diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c
index 1276c7a505..86b79ef9b6 100644
--- a/src/trunnel/ed25519_cert.c
+++ b/src/trunnel/ed25519_cert.c
@@ -1,4 +1,4 @@
-/* ed25519_cert.c -- generated by Trunnel v1.5.2.
+/* ed25519_cert.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h
index e086c6fced..bd91ce1055 100644
--- a/src/trunnel/ed25519_cert.h
+++ b/src/trunnel/ed25519_cert.h
@@ -1,4 +1,4 @@
-/* ed25519_cert.h -- generated by Trunnel v1.5.2.
+/* ed25519_cert.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel
index 8d6483d558..e424ce5464 100644
--- a/src/trunnel/ed25519_cert.trunnel
+++ b/src/trunnel/ed25519_cert.trunnel
@@ -28,12 +28,6 @@ const LS_IPV6 = 0x01;
 const LS_LEGACY_ID = 0x02;
 const LS_ED25519_ID = 0x03;
 
-// XXX hs_link_specifier_dup() violates the opaqueness of link_specifier_t by
-//  taking its sizeof(). If we ever want to turn on TRUNNEL_OPAQUE, or
-//  if we ever make link_specifier contain other types, we will
-//  need to refactor that function to do the copy by encoding and decoding the
-//  object.
-
 // amended from tor.trunnel
 struct link_specifier {
   u8 ls_type;
diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c
index af223560c1..1f50961d69 100644
--- a/src/trunnel/hs/cell_common.c
+++ b/src/trunnel/hs/cell_common.c
@@ -1,4 +1,4 @@
-/* cell_common.c -- generated by Trunnel v1.5.2.
+/* cell_common.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -28,10 +28,10 @@ int cellcommon_deadcode_dummy__ = 0;
     }                                                            \
   } while (0)
 
-trn_cell_extension_fields_t *
-trn_cell_extension_fields_new(void)
+trn_cell_extension_field_t *
+trn_cell_extension_field_new(void)
 {
-  trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_field_t));
   if (NULL == val)
     return NULL;
   return val;
@@ -40,7 +40,7 @@ trn_cell_extension_fields_new(void)
 /** Release all storage held inside 'obj', but do not free 'obj'.
  */
 static void
-trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear(trn_cell_extension_field_t *obj)
 {
   (void) obj;
   TRUNNEL_DYNARRAY_WIPE(&obj->field);
@@ -48,62 +48,62 @@ trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
 }
 
 void
-trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_free(trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return;
-  trn_cell_extension_fields_clear(obj);
-  trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_field_t));
   trunnel_free_(obj);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp)
 {
   return inp->field_type;
 }
 int
-trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_type = val;
   return 0;
 }
 uint8_t
-trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp)
 {
   return inp->field_len;
 }
 int
-trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_len = val;
   return 0;
 }
 size_t
-trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp)
 {
   return TRUNNEL_DYNARRAY_LEN(&inp->field);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->field, idx);
 }
 
 uint8_t
-trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx)
 {
-  return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx);
+  return trn_cell_extension_field_get_field((trn_cell_extension_field_t*)inp, idx);
 }
 int
-trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt)
+trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt)
+trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->field.n_ == UINT8_MAX)
@@ -117,17 +117,17 @@ trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t el
 }
 
 uint8_t *
-trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp)
 {
   return inp->field.elts_;
 }
 const uint8_t  *
-trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp)
 {
-  return (const uint8_t  *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp);
+  return (const uint8_t  *)trn_cell_extension_field_getarray_field((trn_cell_extension_field_t*)inp);
 }
 int
-trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen)
+trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen)
 {
   uint8_t *newptr;
 #if UINT8_MAX < SIZE_MAX
@@ -147,7 +147,7 @@ trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t
   return -1;
 }
 const char *
-trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_check(const trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return "Object was NULL";
@@ -159,11 +159,11 @@ trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
 }
 
 ssize_t
-trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
 
-  if (NULL != trn_cell_extension_fields_check(obj))
+  if (NULL != trn_cell_extension_field_check(obj))
      return -1;
 
 
@@ -178,24 +178,24 @@ trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
   return result;
 }
 int
-trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj)
 {
   int r = obj->trunnel_error_code_;
   obj->trunnel_error_code_ = 0;
   return r;
 }
 ssize_t
-trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encode(uint8_t *output, const size_t avail, const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
   size_t written = 0;
   uint8_t *ptr = output;
   const char *msg;
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
-  const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj);
+  const ssize_t encoded_len = trn_cell_extension_field_encoded_len(obj);
 #endif
 
-  if (NULL != (msg = trn_cell_extension_fields_check(obj)))
+  if (NULL != (msg = trn_cell_extension_field_check(obj)))
     goto check_failed;
 
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
@@ -252,11 +252,11 @@ trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_
   return result;
 }
 
-/** As trn_cell_extension_fields_parse(), but do not allocate the
+/** As trn_cell_extension_field_parse(), but do not allocate the
  * output object.
  */
 static ssize_t
-trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse_into(trn_cell_extension_field_t *obj, const uint8_t *input, const size_t len_in)
 {
   const uint8_t *ptr = input;
   size_t remaining = len_in;
@@ -290,15 +290,15 @@ trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uin
 }
 
 ssize_t
-trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in)
 {
   ssize_t result;
-  *output = trn_cell_extension_fields_new();
+  *output = trn_cell_extension_field_new();
   if (NULL == *output)
     return -1;
-  result = trn_cell_extension_fields_parse_into(*output, input, len_in);
+  result = trn_cell_extension_field_parse_into(*output, input, len_in);
   if (result < 0) {
-    trn_cell_extension_fields_free(*output);
+    trn_cell_extension_field_free(*output);
     *output = NULL;
   }
   return result;
@@ -322,7 +322,7 @@ trn_cell_extension_clear(trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      trn_cell_extension_field_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   TRUNNEL_DYNARRAY_WIPE(&obj->fields);
@@ -356,66 +356,66 @@ trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp)
   return TRUNNEL_DYNARRAY_LEN(&inp->fields);
 }
 
-struct trn_cell_extension_fields_st *
+struct trn_cell_extension_field_st *
 trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
 }
 
- const struct trn_cell_extension_fields_st *
+ const struct trn_cell_extension_field_st *
 trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx)
 {
   return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx);
 }
 int
-trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
-  trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+  trn_cell_extension_field_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
   if (oldval && oldval != elt)
-    trn_cell_extension_fields_free(oldval);
+    trn_cell_extension_field_free(oldval);
   return trn_cell_extension_set0_fields(inp, idx, elt);
 }
 int
-trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->fields.n_ == UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
-  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {});
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_field_st *, &inp->fields, elt, {});
   return 0;
  trunnel_alloc_failed:
   TRUNNEL_SET_ERROR_CODE(inp);
   return -1;
 }
 
-struct trn_cell_extension_fields_st * *
+struct trn_cell_extension_field_st * *
 trn_cell_extension_getarray_fields(trn_cell_extension_t *inp)
 {
   return inp->fields.elts_;
 }
-const struct trn_cell_extension_fields_st *  const  *
+const struct trn_cell_extension_field_st *  const  *
 trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp)
 {
-  return (const struct trn_cell_extension_fields_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
+  return (const struct trn_cell_extension_field_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
 }
 int
 trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen)
 {
-  struct trn_cell_extension_fields_st * *newptr;
+  struct trn_cell_extension_field_st * *newptr;
 #if UINT8_MAX < SIZE_MAX
   if (newlen > UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
   newptr = trunnel_dynarray_setlen(&inp->fields.allocated_,
                  &inp->fields.n_, inp->fields.elts_, newlen,
-                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free,
+                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_field_free,
                  &inp->trunnel_error_code_);
   if (newlen != 0 && newptr == NULL)
     goto trunnel_alloc_failed;
@@ -437,7 +437,7 @@ trn_cell_extension_check(const trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
+      if (NULL != (msg = trn_cell_extension_field_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
         return msg;
     }
   }
@@ -458,12 +458,12 @@ trn_cell_extension_encoded_len(const trn_cell_extension_t *obj)
   /* Length of u8 num */
   result += 1;
 
-  /* Length of struct trn_cell_extension_fields fields[num] */
+  /* Length of struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result += trn_cell_extension_field_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   return result;
@@ -500,13 +500,13 @@ trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_ex
   trunnel_set_uint8(ptr, (obj->num));
   written += 1; ptr += 1;
 
-  /* Encode struct trn_cell_extension_fields fields[num] */
+  /* Encode struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
       trunnel_assert(written <= avail);
-      result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result = trn_cell_extension_field_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
       if (result < 0)
         goto fail; /* XXXXXXX !*/
       written += result; ptr += result;
@@ -553,18 +553,18 @@ trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, c
   obj->num = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
 
-  /* Parse struct trn_cell_extension_fields fields[num] */
-  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {});
+  /* Parse struct trn_cell_extension_field fields[num] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_field_t *, &obj->fields, obj->num, {});
   {
-    trn_cell_extension_fields_t * elt;
+    trn_cell_extension_field_t * elt;
     unsigned idx;
     for (idx = 0; idx < obj->num; ++idx) {
-      result = trn_cell_extension_fields_parse(&elt, ptr, remaining);
+      result = trn_cell_extension_field_parse(&elt, ptr, remaining);
       if (result < 0)
         goto relay_fail;
       trunnel_assert((size_t)result <= remaining);
       remaining -= result; ptr += result;
-      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);});
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_field_t *, &obj->fields, elt, {trn_cell_extension_field_free(elt);});
     }
   }
   trunnel_assert(ptr + remaining == input + len_in);
diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h
index e08eedfdb3..beb65e015f 100644
--- a/src/trunnel/hs/cell_common.h
+++ b/src/trunnel/hs/cell_common.h
@@ -1,4 +1,4 @@
-/* cell_common.h -- generated by Trunnel v1.5.2.
+/* cell_common.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -8,112 +8,112 @@
 #include <stdint.h>
 #include "trunnel.h"
 
-#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS)
-struct trn_cell_extension_fields_st {
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELD)
+struct trn_cell_extension_field_st {
   uint8_t field_type;
   uint8_t field_len;
   TRUNNEL_DYNARRAY_HEAD(, uint8_t) field;
   uint8_t trunnel_error_code_;
 };
 #endif
-typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t;
+typedef struct trn_cell_extension_field_st trn_cell_extension_field_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION)
 struct trn_cell_extension_st {
   uint8_t num;
-  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_field_st *) fields;
   uint8_t trunnel_error_code_;
 };
 #endif
 typedef struct trn_cell_extension_st trn_cell_extension_t;
-/** Return a newly allocated trn_cell_extension_fields with all
+/** Return a newly allocated trn_cell_extension_field with all
  * elements set to zero.
  */
-trn_cell_extension_fields_t *trn_cell_extension_fields_new(void);
-/** Release all storage held by the trn_cell_extension_fields in
+trn_cell_extension_field_t *trn_cell_extension_field_new(void);
+/** Release all storage held by the trn_cell_extension_field in
  * 'victim'. (Do nothing if 'victim' is NULL.)
  */
-void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim);
-/** Try to parse a trn_cell_extension_fields from the buffer in
+void trn_cell_extension_field_free(trn_cell_extension_field_t *victim);
+/** Try to parse a trn_cell_extension_field from the buffer in
  * 'input', using up to 'len_in' bytes from the input buffer. On
  * success, return the number of bytes consumed and set *output to the
- * newly allocated trn_cell_extension_fields_t. On failure, return -2
+ * newly allocated trn_cell_extension_field_t. On failure, return -2
  * if the input appears truncated, and -1 if the input is otherwise
  * invalid.
  */
-ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in);
 /** Return the number of bytes we expect to need to encode the
- * trn_cell_extension_fields in 'obj'. On failure, return a negative
+ * trn_cell_extension_field in 'obj'. On failure, return a negative
  * value. Note that this value may be an overestimate, and can even be
  * an underestimate for certain unencodeable objects.
  */
-ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj);
-/** Try to encode the trn_cell_extension_fields from 'input' into the
+ssize_t trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj);
+/** Try to encode the trn_cell_extension_field from 'input' into the
  * buffer at 'output', using up to 'avail' bytes of the output buffer.
  * On success, return the number of bytes used. On failure, return -2
  * if the buffer was not long enough, and -1 if the input was invalid.
  */
-ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input);
-/** Check whether the internal state of the trn_cell_extension_fields
+ssize_t trn_cell_extension_field_encode(uint8_t *output, size_t avail, const trn_cell_extension_field_t *input);
+/** Check whether the internal state of the trn_cell_extension_field
  * in 'obj' is consistent. Return NULL if it is, and a short message
  * if it is not.
  */
-const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj);
+const char *trn_cell_extension_field_check(const trn_cell_extension_field_t *obj);
 /** Clear any errors that were set on the object 'obj' by its setter
  * functions. Return true iff errors were cleared.
  */
-int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj);
+int trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj);
 /** Return the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the length of the dynamic array holding the field field of
- * the trn_cell_extension_fields_t in 'inp'.
+ * the trn_cell_extension_field_t in 'inp'.
  */
-size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp);
+size_t trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp'.
+ * field of the trn_cell_extension_field_t in 'inp'.
  */
-uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx);
+uint8_t trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp', so that it will
+ * field of the trn_cell_extension_field_t in 'inp', so that it will
  * hold the value 'elt'.
  */
-int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt);
+int trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt);
 /** Append a new element 'elt' to the dynamic array field field of the
- * trn_cell_extension_fields_t in 'inp'.
+ * trn_cell_extension_field_t in 'inp'.
  */
-int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt);
+int trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field field of
  * 'inp'.
  */
-uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t * trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-const uint8_t  * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp);
+const uint8_t  * trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp);
 /** Change the length of the variable-length array field field of
  * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen);
+int trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_extension with all elements set
  * to zero.
  */
@@ -166,32 +166,32 @@ size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp'.
  */
-struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
+struct trn_cell_extension_field_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
- const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
+ const struct trn_cell_extension_field_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp', so that it will hold
  * the value 'elt'. Free the previous value, if any.
  */
-int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** As trn_cell_extension_set_fields, but does not free the previous
  * value.
  */
-int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** Append a new element 'elt' to the dynamic array field fields of
  * the trn_cell_extension_t in 'inp'.
  */
-int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt);
 /** Return a pointer to the variable-length array field fields of
  * 'inp'.
  */
-struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
+struct trn_cell_extension_field_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
-const struct trn_cell_extension_fields_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
+const struct trn_cell_extension_field_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
 /** Change the length of the variable-length array field fields of
  * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
  * elements. Return 0 on success; return -1 and set the error code on
diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel
index 1aa6999de7..7e99cbfa66 100644
--- a/src/trunnel/hs/cell_common.trunnel
+++ b/src/trunnel/hs/cell_common.trunnel
@@ -1,6 +1,6 @@
 /* This file contains common data structure that cells use. */
 
-struct trn_cell_extension_fields {
+struct trn_cell_extension_field {
   u8 field_type;
   u8 field_len;
   u8 field[field_len];
@@ -8,5 +8,5 @@ struct trn_cell_extension_fields {
 
 struct trn_cell_extension {
   u8 num;
-  struct trn_cell_extension_fields fields[num];
+  struct trn_cell_extension_field fields[num];
 };
diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c
index ae3b7b1bc8..f31404c55f 100644
--- a/src/trunnel/hs/cell_establish_intro.c
+++ b/src/trunnel/hs/cell_establish_intro.c
@@ -1,4 +1,4 @@
-/* cell_establish_intro.c -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -36,6 +36,185 @@ ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
 ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
 const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
 int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+trn_cell_extension_dos_param_t *
+trn_cell_extension_dos_param_new(void)
+{
+  trn_cell_extension_dos_param_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_param_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_param_clear(trn_cell_extension_dos_param_t *obj)
+{
+  (void) obj;
+}
+
+void
+trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_param_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_param_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->type;
+}
+int
+trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val)
+{
+  inp->type = val;
+  return 0;
+}
+uint64_t
+trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->value;
+}
+int
+trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val)
+{
+  inp->value = val;
+  return 0;
+}
+const char *
+trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_param_check(obj))
+     return -1;
+
+
+  /* Length of u8 type */
+  result += 1;
+
+  /* Length of u64 value */
+  result += 8;
+  return result;
+}
+int
+trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_param_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_param_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_param_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->type));
+  written += 1; ptr += 1;
+
+  /* Encode u64 value */
+  trunnel_assert(written <= avail);
+  if (avail - written < 8)
+    goto truncated;
+  trunnel_set_uint64(ptr, trunnel_htonll(obj->value));
+  written += 8; ptr += 8;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_param_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_extension_dos_param_parse_into(trn_cell_extension_dos_param_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 type */
+  CHECK_REMAINING(1, truncated);
+  obj->type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u64 value */
+  CHECK_REMAINING(8, truncated);
+  obj->value = trunnel_ntohll(trunnel_get_uint64(ptr));
+  remaining -= 8; ptr += 8;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_param_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_param_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_param_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_establish_intro_t *
 trn_cell_establish_intro_new(void)
 {
@@ -561,6 +740,296 @@ trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_
   }
   return result;
 }
+trn_cell_extension_dos_t *
+trn_cell_extension_dos_new(void)
+{
+  trn_cell_extension_dos_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_clear(trn_cell_extension_dos_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trn_cell_extension_dos_param_free(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->params);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->params);
+}
+
+void
+trn_cell_extension_dos_free(trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp)
+{
+  return inp->n_params;
+}
+int
+trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val)
+{
+  inp->n_params = val;
+  return 0;
+}
+size_t
+trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->params);
+}
+
+struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+}
+
+ const struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return trn_cell_extension_dos_get_params((trn_cell_extension_dos_t*)inp, idx);
+}
+int
+trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  trn_cell_extension_dos_param_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+  if (oldval && oldval != elt)
+    trn_cell_extension_dos_param_free(oldval);
+  return trn_cell_extension_dos_set0_params(inp, idx, elt);
+}
+int
+trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->params, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->params.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_dos_param_st *, &inp->params, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct trn_cell_extension_dos_param_st * *
+trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp)
+{
+  return inp->params.elts_;
+}
+const struct trn_cell_extension_dos_param_st *  const  *
+trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp)
+{
+  return (const struct trn_cell_extension_dos_param_st *  const  *)trn_cell_extension_dos_getarray_params((trn_cell_extension_dos_t*)inp);
+}
+int
+trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen)
+{
+  struct trn_cell_extension_dos_param_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->params.allocated_,
+                 &inp->params.n_, inp->params.elts_, newlen,
+                 sizeof(inp->params.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_dos_param_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->params.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      if (NULL != (msg = trn_cell_extension_dos_param_check(TRUNNEL_DYNARRAY_GET(&obj->params, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->params) != obj->n_params)
+    return "Length mismatch for params";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_params */
+  result += 1;
+
+  /* Length of struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      result += trn_cell_extension_dos_param_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  return result;
+}
+int
+trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_params */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_params));
+  written += 1; ptr += 1;
+
+  /* Encode struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trunnel_assert(written <= avail);
+      result = trn_cell_extension_dos_param_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_extension_dos_parse_into(trn_cell_extension_dos_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_params */
+  CHECK_REMAINING(1, truncated);
+  obj->n_params = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct trn_cell_extension_dos_param params[n_params] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_dos_param_t *, &obj->params, obj->n_params, {});
+  {
+    trn_cell_extension_dos_param_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_params; ++idx) {
+      result = trn_cell_extension_dos_param_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_dos_param_t *, &obj->params, elt, {trn_cell_extension_dos_param_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_intro_established_t *
 trn_cell_intro_established_new(void)
 {
diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h
index ccaef5488c..1924d9cab6 100644
--- a/src/trunnel/hs/cell_establish_intro.h
+++ b/src/trunnel/hs/cell_establish_intro.h
@@ -1,4 +1,4 @@
-/* cell_establish_intro.h -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -10,6 +10,17 @@
 
 struct trn_cell_extension_st;
 #define TRUNNEL_SHA3_256_LEN 32
+#define TRUNNEL_CELL_EXTENSION_TYPE_DOS 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC 2
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS_PARAM)
+struct trn_cell_extension_dos_param_st {
+  uint8_t type;
+  uint64_t value;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_param_st trn_cell_extension_dos_param_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO)
 struct trn_cell_establish_intro_st {
   const uint8_t *start_cell;
@@ -26,6 +37,14 @@ struct trn_cell_establish_intro_st {
 };
 #endif
 typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS)
+struct trn_cell_extension_dos_st {
+  uint8_t n_params;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_dos_param_st *) params;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_st trn_cell_extension_dos_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED)
 struct trn_cell_intro_established_st {
   struct trn_cell_extension_st *extensions;
@@ -33,6 +52,62 @@ struct trn_cell_intro_established_st {
 };
 #endif
 typedef struct trn_cell_intro_established_st trn_cell_intro_established_t;
+/** Return a newly allocated trn_cell_extension_dos_param with all
+ * elements set to zero.
+ */
+trn_cell_extension_dos_param_t *trn_cell_extension_dos_param_new(void);
+/** Release all storage held by the trn_cell_extension_dos_param in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *victim);
+/** Try to parse a trn_cell_extension_dos_param from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_extension_dos_param_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos_param in 'obj'. On failure, return a
+ * negative value. Note that this value may be an overestimate, and
+ * can even be an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj);
+/** Try to encode the trn_cell_extension_dos_param from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t trn_cell_extension_dos_param_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_param_t *input);
+/** Check whether the internal state of the
+ * trn_cell_extension_dos_param in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj);
+/** Return the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val);
+/** Return the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint64_t trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val);
 /** Return a newly allocated trn_cell_establish_intro with all
  * elements set to zero.
  */
@@ -216,6 +291,90 @@ const uint8_t  * trn_cell_establish_intro_getconstarray_sig(const trn_cell_estab
  * -1 and set the error code on 'inp' on failure.
  */
 int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_extension_dos with all elements
+ * set to zero.
+ */
+trn_cell_extension_dos_t *trn_cell_extension_dos_new(void);
+/** Release all storage held by the trn_cell_extension_dos in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_free(trn_cell_extension_dos_t *victim);
+/** Try to parse a trn_cell_extension_dos from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_extension_dos_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj);
+/** Try to encode the trn_cell_extension_dos from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_dos_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_t *input);
+/** Check whether the internal state of the trn_cell_extension_dos in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj);
+/** Return the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp);
+/** Set the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the params field of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+size_t trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+ const struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp', so that it will
+ * hold the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** As trn_cell_extension_dos_set_params, but does not free the
+ * previous value.
+ */
+int trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** Append a new element 'elt' to the dynamic array field params of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+int trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt);
+/** Return a pointer to the variable-length array field params of
+ * 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * * trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_dos_param_st *  const  * trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp);
+/** Change the length of the variable-length array field params of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_intro_established with all
  * elements set to zero.
  */
diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel
index 011ee62a15..e30938f6c2 100644
--- a/src/trunnel/hs/cell_establish_intro.trunnel
+++ b/src/trunnel/hs/cell_establish_intro.trunnel
@@ -39,3 +39,26 @@ struct trn_cell_intro_established {
   /* Extension(s). Reserved fields. */
   struct trn_cell_extension extensions;
 };
+
+/*
+ * ESTABLISH_INTRO cell extensions.
+ */
+
+const TRUNNEL_CELL_EXTENSION_TYPE_DOS = 0x01;
+
+/* DoS Parameter types. */
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC = 0x01;
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC = 0x02;
+
+/*
+ * DoS Parameters Extension. See proposal 305 for more details.
+ */
+struct trn_cell_extension_dos_param {
+  u8 type;
+  u64 value;
+};
+
+struct trn_cell_extension_dos {
+  u8 n_params;
+  struct trn_cell_extension_dos_param params[n_params];
+};
diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c
index 53b3d299f2..016c9fa8d6 100644
--- a/src/trunnel/hs/cell_introduce1.c
+++ b/src/trunnel/hs/cell_introduce1.c
@@ -1,4 +1,4 @@
-/* cell_introduce1.c -- generated by Trunnel v1.5.2.
+/* cell_introduce1.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h
index 986a531ca7..8dabff3cb5 100644
--- a/src/trunnel/hs/cell_introduce1.h
+++ b/src/trunnel/hs/cell_introduce1.h
@@ -1,4 +1,4 @@
-/* cell_introduce1.h -- generated by Trunnel v1.5.2.
+/* cell_introduce1.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c
index 53cb609138..1204e93cfc 100644
--- a/src/trunnel/hs/cell_rendezvous.c
+++ b/src/trunnel/hs/cell_rendezvous.c
@@ -1,4 +1,4 @@
-/* cell_rendezvous.c -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h
index 39e14da25b..5a8c2ff52a 100644
--- a/src/trunnel/hs/cell_rendezvous.h
+++ b/src/trunnel/hs/cell_rendezvous.h
@@ -1,4 +1,4 @@
-/* cell_rendezvous.h -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/include.am b/src/trunnel/include.am
index 03c1753e96..6c3a5ff06b 100644
--- a/src/trunnel/include.am
+++ b/src/trunnel/include.am
@@ -11,7 +11,9 @@ TRUNNELINPUTS = \
 	src/trunnel/link_handshake.trunnel \
 	src/trunnel/pwbox.trunnel \
 	src/trunnel/channelpadding_negotiation.trunnel \
-	src/trunner/socks5.trunnel
+	src/trunnel/sendme_cell.trunnel \
+	src/trunnel/socks5.trunnel \
+	src/trunnel/circpad_negotiation.trunnel
 
 TRUNNELSOURCES = \
 	src/ext/trunnel/trunnel.c \
@@ -23,7 +25,10 @@ TRUNNELSOURCES = \
 	src/trunnel/hs/cell_introduce1.c \
 	src/trunnel/hs/cell_rendezvous.c \
 	src/trunnel/channelpadding_negotiation.c \
-	src/trunnel/socks5.c
+	src/trunnel/sendme_cell.c                    \
+	src/trunnel/socks5.c \
+	src/trunnel/netinfo.c \
+	src/trunnel/circpad_negotiation.c
 
 TRUNNELHEADERS = \
 	src/ext/trunnel/trunnel.h		\
@@ -37,7 +42,10 @@ TRUNNELHEADERS = \
 	src/trunnel/hs/cell_introduce1.h \
 	src/trunnel/hs/cell_rendezvous.h \
 	src/trunnel/channelpadding_negotiation.h \
-	src/trunnel/socks5.h
+	src/trunnel/sendme_cell.h                    \
+	src/trunnel/socks5.h                    \
+	src/trunnel/netinfo.h \
+	src/trunnel/circpad_negotiation.h
 
 src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES)
 src_trunnel_libor_trunnel_a_CPPFLAGS = \
diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c
index 03ead31c62..76db4b0e29 100644
--- a/src/trunnel/link_handshake.c
+++ b/src/trunnel/link_handshake.c
@@ -1,4 +1,4 @@
-/* link_handshake.c -- generated by Trunnel v1.5.2.
+/* link_handshake.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h
index 6a23483adc..0c7ac36b1b 100644
--- a/src/trunnel/link_handshake.h
+++ b/src/trunnel/link_handshake.h
@@ -1,4 +1,4 @@
-/* link_handshake.h -- generated by Trunnel v1.5.2.
+/* link_handshake.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c
new file mode 100644
index 0000000000..d7d0cddc89
--- /dev/null
+++ b/src/trunnel/netinfo.c
@@ -0,0 +1,723 @@
+/* netinfo.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "netinfo.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int netinfo_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+netinfo_addr_t *
+netinfo_addr_new(void)
+{
+  netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_addr_clear(netinfo_addr_t *obj)
+{
+  (void) obj;
+}
+
+void
+netinfo_addr_free(netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_addr_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_addr_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+netinfo_addr_get_addr_type(const netinfo_addr_t *inp)
+{
+  return inp->addr_type;
+}
+int
+netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->addr_type = val;
+  return 0;
+}
+uint8_t
+netinfo_addr_get_len(const netinfo_addr_t *inp)
+{
+  return inp->len;
+}
+int
+netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->len = val;
+  return 0;
+}
+uint32_t
+netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp)
+{
+  return inp->addr_ipv4;
+}
+int
+netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val)
+{
+  inp->addr_ipv4 = val;
+  return 0;
+}
+size_t
+netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->addr_ipv6[idx];
+}
+
+uint8_t
+netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx)
+{
+  return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx);
+}
+int
+netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->addr_ipv6[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp)
+{
+  return inp->addr_ipv6;
+}
+const uint8_t  *
+netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp)
+{
+  return (const uint8_t  *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp);
+}
+const char *
+netinfo_addr_check(const netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+netinfo_addr_encoded_len(const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_addr_check(obj))
+     return -1;
+
+
+  /* Length of u8 addr_type */
+  result += 1;
+
+  /* Length of u8 len */
+  result += 1;
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+
+      /* Length of u32 addr_ipv4 */
+      result += 4;
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+
+      /* Length of u8 addr_ipv6[16] */
+      result += 16;
+      break;
+
+    default:
+      break;
+  }
+  return result;
+}
+int
+netinfo_addr_clear_errors(netinfo_addr_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_addr_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_len = NULL;
+
+  if (NULL != (msg = netinfo_addr_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 addr_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->addr_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 len */
+  backptr_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->len));
+  written += 1; ptr += 1;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union addr[addr_type] */
+    trunnel_assert(written <= avail);
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Encode u32 addr_ipv4 */
+        trunnel_assert(written <= avail);
+        if (avail - written < 4)
+          goto truncated;
+        trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4));
+        written += 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Encode u8 addr_ipv6[16] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 16)
+          goto truncated;
+        memcpy(ptr, obj->addr_ipv6, 16);
+        written += 16; ptr += 16;
+        break;
+
+      default:
+        break;
+    }
+    /* Write the length field back to len */
+    trunnel_assert(written >= written_before_union);
+#if UINT8_MAX < SIZE_MAX
+    if (written - written_before_union > UINT8_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint8(backptr_len, (written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_addr_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 addr_type */
+  CHECK_REMAINING(1, truncated);
+  obj->addr_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 len */
+  CHECK_REMAINING(1, truncated);
+  obj->len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->len, truncated);
+    remaining_after = remaining - obj->len;
+    remaining = obj->len;
+
+    /* Parse union addr[addr_type] */
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Parse u32 addr_ipv4 */
+        CHECK_REMAINING(4, fail);
+        obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr));
+        remaining -= 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Parse u8 addr_ipv6[16] */
+        CHECK_REMAINING(16, fail);
+        memcpy(obj->addr_ipv6, ptr, 16);
+        remaining -= 16; ptr += 16;
+        break;
+
+      default:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_addr_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_addr_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_addr_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+netinfo_cell_t *
+netinfo_cell_new(void)
+{
+  netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_cell_clear(netinfo_cell_t *obj)
+{
+  (void) obj;
+  netinfo_addr_free(obj->other_addr);
+  obj->other_addr = NULL;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs);
+}
+
+void
+netinfo_cell_free(netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_cell_t));
+  trunnel_free_(obj);
+}
+
+uint32_t
+netinfo_cell_get_timestamp(const netinfo_cell_t *inp)
+{
+  return inp->timestamp;
+}
+int
+netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val)
+{
+  inp->timestamp = val;
+  return 0;
+}
+struct netinfo_addr_st *
+netinfo_cell_get_other_addr(netinfo_cell_t *inp)
+{
+  return inp->other_addr;
+}
+const struct netinfo_addr_st *
+netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp)
+{
+  return netinfo_cell_get_other_addr((netinfo_cell_t*) inp);
+}
+int
+netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  if (inp->other_addr && inp->other_addr != val)
+    netinfo_addr_free(inp->other_addr);
+  return netinfo_cell_set0_other_addr(inp, val);
+}
+int
+netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  inp->other_addr = val;
+  return 0;
+}
+uint8_t
+netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp)
+{
+  return inp->n_my_addrs;
+}
+int
+netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val)
+{
+  inp->n_my_addrs = val;
+  return 0;
+}
+size_t
+netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs);
+}
+
+struct netinfo_addr_st *
+netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+}
+
+ const struct netinfo_addr_st *
+netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx)
+{
+  return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx);
+}
+int
+netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+  if (oldval && oldval != elt)
+    netinfo_addr_free(oldval);
+  return netinfo_cell_set0_my_addrs(inp, idx, elt);
+}
+int
+netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt);
+  return 0;
+}
+int
+netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->my_addrs.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct netinfo_addr_st * *
+netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp)
+{
+  return inp->my_addrs.elts_;
+}
+const struct netinfo_addr_st *  const  *
+netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp)
+{
+  return (const struct netinfo_addr_st *  const  *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp);
+}
+int
+netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen)
+{
+  struct netinfo_addr_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_,
+                 &inp->my_addrs.n_, inp->my_addrs.elts_, newlen,
+                 sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->my_addrs.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+netinfo_cell_check(const netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = netinfo_addr_check(obj->other_addr)))
+      return msg;
+  }
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs)
+    return "Length mismatch for my_addrs";
+  return NULL;
+}
+
+ssize_t
+netinfo_cell_encoded_len(const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_cell_check(obj))
+     return -1;
+
+
+  /* Length of u32 timestamp */
+  result += 4;
+
+  /* Length of struct netinfo_addr other_addr */
+  result += netinfo_addr_encoded_len(obj->other_addr);
+
+  /* Length of u8 n_my_addrs */
+  result += 1;
+
+  /* Length of struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  return result;
+}
+int
+netinfo_cell_clear_errors(netinfo_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_cell_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = netinfo_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u32 timestamp */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp));
+  written += 4; ptr += 4;
+
+  /* Encode struct netinfo_addr other_addr */
+  trunnel_assert(written <= avail);
+  result = netinfo_addr_encode(ptr, avail - written, obj->other_addr);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 n_my_addrs */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_my_addrs));
+  written += 1; ptr += 1;
+
+  /* Encode struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      trunnel_assert(written <= avail);
+      result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u32 timestamp */
+  CHECK_REMAINING(4, truncated);
+  obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse struct netinfo_addr other_addr */
+  result = netinfo_addr_parse(&obj->other_addr, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 n_my_addrs */
+  CHECK_REMAINING(1, truncated);
+  obj->n_my_addrs = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct netinfo_addr my_addrs[n_my_addrs] */
+  TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {});
+  {
+    netinfo_addr_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_my_addrs; ++idx) {
+      result = netinfo_addr_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h
new file mode 100644
index 0000000000..37c2ae3c2d
--- /dev/null
+++ b/src/trunnel/netinfo.h
@@ -0,0 +1,226 @@
+/* netinfo.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_NETINFO_H
+#define TRUNNEL_NETINFO_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define NETINFO_ADDR_TYPE_IPV4 4
+#define NETINFO_ADDR_TYPE_IPV6 6
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR)
+struct netinfo_addr_st {
+  uint8_t addr_type;
+  uint8_t len;
+  uint32_t addr_ipv4;
+  uint8_t addr_ipv6[16];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_addr_st netinfo_addr_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL)
+struct netinfo_cell_st {
+  uint32_t timestamp;
+  struct netinfo_addr_st *other_addr;
+  uint8_t n_my_addrs;
+  TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_cell_st netinfo_cell_t;
+/** Return a newly allocated netinfo_addr with all elements set to
+ * zero.
+ */
+netinfo_addr_t *netinfo_addr_new(void);
+/** Release all storage held by the netinfo_addr in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_addr_free(netinfo_addr_t *victim);
+/** Try to parse a netinfo_addr from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_addr_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_addr in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj);
+/** Try to encode the netinfo_addr from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input);
+/** Check whether the internal state of the netinfo_addr in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_addr_check(const netinfo_addr_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_addr_clear_errors(netinfo_addr_t *obj);
+/** Return the value of the addr_type field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp);
+/** Set the value of the addr_type field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the len field of the netinfo_addr_t in 'inp'
+ */
+uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp);
+/** Set the value of the len field of the netinfo_addr_t in 'inp' to
+ * 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp);
+/** Set the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val);
+/** Return the (constant) length of the array holding the addr_ipv6
+ * field of the netinfo_addr_t in 'inp'.
+ */
+size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp'.
+ */
+uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the
+ * value 'elt'.
+ */
+int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'.
+ */
+uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+const uint8_t  * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp);
+/** Return a newly allocated netinfo_cell with all elements set to
+ * zero.
+ */
+netinfo_cell_t *netinfo_cell_new(void);
+/** Release all storage held by the netinfo_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_cell_free(netinfo_cell_t *victim);
+/** Try to parse a netinfo_cell from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj);
+/** Try to encode the netinfo_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input);
+/** Check whether the internal state of the netinfo_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_cell_check(const netinfo_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_cell_clear_errors(netinfo_cell_t *obj);
+/** Return the value of the timestamp field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp);
+/** Set the value of the timestamp field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val);
+/** Return the value of the other_addr field of the netinfo_cell_t in
+ * 'inp'
+ */
+struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp);
+/** As netinfo_cell_get_other_addr, but take and return a const
+ * pointer
+ */
+const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp);
+/** Set the value of the other_addr field of the netinfo_cell_t in
+ * 'inp' to 'val'. Free the old value if any. Steals the referenceto
+ * 'val'.Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** As netinfo_cell_set_other_addr, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** Return the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp);
+/** Set the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the my_addrs field
+ * of the netinfo_cell_t in 'inp'.
+ */
+size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp'.
+ */
+struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+ const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the
+ * value 'elt'. Free the previous value, if any.
+ */
+int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** As netinfo_cell_set_my_addrs, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** Append a new element 'elt' to the dynamic array field my_addrs of
+ * the netinfo_cell_t in 'inp'.
+ */
+int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt);
+/** Return a pointer to the variable-length array field my_addrs of
+ * 'inp'.
+ */
+struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+const struct netinfo_addr_st *  const  * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp);
+/** Change the length of the variable-length array field my_addrs of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel
new file mode 100644
index 0000000000..2c4b7a7591
--- /dev/null
+++ b/src/trunnel/netinfo.trunnel
@@ -0,0 +1,24 @@
+// Warning: make sure these values are consistent with RESOLVED_TYPE_*
+// constants in Tor code and numbers in Section 6.4 of tor-spec.txt.
+
+const NETINFO_ADDR_TYPE_IPV4 = 4;
+const NETINFO_ADDR_TYPE_IPV6 = 6;
+
+struct netinfo_addr {
+  u8 addr_type;
+  u8 len;
+  union addr[addr_type] with length len {
+    NETINFO_ADDR_TYPE_IPV4: u32 ipv4;
+    NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16];
+    default: ignore;
+  };
+
+}
+
+struct netinfo_cell {
+  u32 timestamp;
+  struct netinfo_addr other_addr;
+  u8 n_my_addrs;
+  struct netinfo_addr my_addrs[n_my_addrs];
+}
+
diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c
index c356515d36..c159a5e687 100644
--- a/src/trunnel/pwbox.c
+++ b/src/trunnel/pwbox.c
@@ -1,4 +1,4 @@
-/* pwbox.c -- generated by Trunnel v1.5.2.
+/* pwbox.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h
index a9a421408a..36d595f4ef 100644
--- a/src/trunnel/pwbox.h
+++ b/src/trunnel/pwbox.h
@@ -1,4 +1,4 @@
-/* pwbox.h -- generated by Trunnel v1.5.2.
+/* pwbox.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/sendme_cell.c b/src/trunnel/sendme_cell.c
new file mode 100644
index 0000000000..b9f8fe967f
--- /dev/null
+++ b/src/trunnel/sendme_cell.c
@@ -0,0 +1,347 @@
+/* sendme_cell.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "sendme_cell.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int sendmecell_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || sendmecell_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+sendme_cell_t *
+sendme_cell_new(void)
+{
+  sendme_cell_t *val = trunnel_calloc(1, sizeof(sendme_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+sendme_cell_clear(sendme_cell_t *obj)
+{
+  (void) obj;
+}
+
+void
+sendme_cell_free(sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  sendme_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(sendme_cell_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+sendme_cell_get_version(const sendme_cell_t *inp)
+{
+  return inp->version;
+}
+int
+sendme_cell_set_version(sendme_cell_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint16_t
+sendme_cell_get_data_len(const sendme_cell_t *inp)
+{
+  return inp->data_len;
+}
+int
+sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val)
+{
+  inp->data_len = val;
+  return 0;
+}
+size_t
+sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp)
+{
+  (void)inp;  return TRUNNEL_SENDME_V1_DIGEST_LEN;
+}
+
+uint8_t
+sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  return inp->data_v1_digest[idx];
+}
+
+uint8_t
+sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx)
+{
+  return sendme_cell_get_data_v1_digest((sendme_cell_t*)inp, idx);
+}
+int
+sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  inp->data_v1_digest[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp)
+{
+  return inp->data_v1_digest;
+}
+const uint8_t  *
+sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp)
+{
+  return (const uint8_t  *)sendme_cell_getarray_data_v1_digest((sendme_cell_t*)inp);
+}
+const char *
+sendme_cell_check(const sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0 || obj->version == 1))
+    return "Integer out of bounds";
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+      break;
+
+    default:
+        return "Bad tag for union";
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+sendme_cell_encoded_len(const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != sendme_cell_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0, 1] */
+  result += 1;
+
+  /* Length of u16 data_len */
+  result += 2;
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+
+      /* Length of u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+      result += TRUNNEL_SENDME_V1_DIGEST_LEN;
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+  return result;
+}
+int
+sendme_cell_clear_errors(sendme_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+sendme_cell_encode(uint8_t *output, const size_t avail, const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = sendme_cell_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_data_len = NULL;
+
+  if (NULL != (msg = sendme_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u16 data_len */
+  backptr_data_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->data_len));
+  written += 2; ptr += 2;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union data[version] */
+    trunnel_assert(written <= avail);
+    switch (obj->version) {
+
+      case 0:
+        break;
+
+      case 1:
+
+        /* Encode u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        trunnel_assert(written <= avail);
+        if (avail - written < TRUNNEL_SENDME_V1_DIGEST_LEN)
+          goto truncated;
+        memcpy(ptr, obj->data_v1_digest, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        written += TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        trunnel_assert(0);
+        break;
+    }
+    /* Write the length field back to data_len */
+    trunnel_assert(written >= written_before_union);
+#if UINT16_MAX < SIZE_MAX
+    if (written - written_before_union > UINT16_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint16(backptr_data_len, trunnel_htons(written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As sendme_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+sendme_cell_parse_into(sendme_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0 || obj->version == 1))
+    goto fail;
+
+  /* Parse u16 data_len */
+  CHECK_REMAINING(2, truncated);
+  obj->data_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->data_len, truncated);
+    remaining_after = remaining - obj->data_len;
+    remaining = obj->data_len;
+
+    /* Parse union data[version] */
+    switch (obj->version) {
+
+      case 0:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+
+      case 1:
+
+        /* Parse u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        CHECK_REMAINING(TRUNNEL_SENDME_V1_DIGEST_LEN, fail);
+        memcpy(obj->data_v1_digest, ptr, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        remaining -= TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        goto fail;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = sendme_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = sendme_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    sendme_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/sendme_cell.h b/src/trunnel/sendme_cell.h
new file mode 100644
index 0000000000..45efb9f10d
--- /dev/null
+++ b/src/trunnel/sendme_cell.h
@@ -0,0 +1,101 @@
+/* sendme_cell.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_SENDME_CELL_H
+#define TRUNNEL_SENDME_CELL_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define TRUNNEL_SENDME_V1_DIGEST_LEN 20
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_CELL)
+struct sendme_cell_st {
+  uint8_t version;
+  uint16_t data_len;
+  uint8_t data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct sendme_cell_st sendme_cell_t;
+/** Return a newly allocated sendme_cell with all elements set to
+ * zero.
+ */
+sendme_cell_t *sendme_cell_new(void);
+/** Release all storage held by the sendme_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void sendme_cell_free(sendme_cell_t *victim);
+/** Try to parse a sendme_cell from the buffer in 'input', using up to
+ * 'len_in' bytes from the input buffer. On success, return the number
+ * of bytes consumed and set *output to the newly allocated
+ * sendme_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * sendme_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t sendme_cell_encoded_len(const sendme_cell_t *obj);
+/** Try to encode the sendme_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t sendme_cell_encode(uint8_t *output, size_t avail, const sendme_cell_t *input);
+/** Check whether the internal state of the sendme_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *sendme_cell_check(const sendme_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int sendme_cell_clear_errors(sendme_cell_t *obj);
+/** Return the value of the version field of the sendme_cell_t in
+ * 'inp'
+ */
+uint8_t sendme_cell_get_version(const sendme_cell_t *inp);
+/** Set the value of the version field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_version(sendme_cell_t *inp, uint8_t val);
+/** Return the value of the data_len field of the sendme_cell_t in
+ * 'inp'
+ */
+uint16_t sendme_cell_get_data_len(const sendme_cell_t *inp);
+/** Set the value of the data_len field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the
+ * data_v1_digest field of the sendme_cell_t in 'inp'.
+ */
+size_t sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp'.
+ */
+uint8_t sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+uint8_t sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SENDME_V1_DIGEST_LEN-element array
+ * field data_v1_digest of 'inp'.
+ */
+uint8_t * sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+const uint8_t  * sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp);
+
+
+#endif
diff --git a/src/trunnel/sendme_cell.trunnel b/src/trunnel/sendme_cell.trunnel
new file mode 100644
index 0000000000..300963e679
--- /dev/null
+++ b/src/trunnel/sendme_cell.trunnel
@@ -0,0 +1,19 @@
+/* This file contains the SENDME cell definition. */
+
+/* v1 digest length in bytes. */
+const TRUNNEL_SENDME_V1_DIGEST_LEN = 20;
+
+/* SENDME cell declaration. */
+struct sendme_cell {
+  /* Version field. */
+  u8 version IN [0x00, 0x01];
+
+  /* Length of data contained in this cell. */
+  u16 data_len;
+
+  /* The data content depends on the version. */
+  union data[version] with length data_len {
+    0x00: ignore;
+    0x01: u8 v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  };
+}
diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c
index 9e5f6fcfed..f32862e353 100644
--- a/src/trunnel/socks5.c
+++ b/src/trunnel/socks5.c
@@ -1,4 +1,4 @@
-/* socks5.c -- generated by Trunnel v1.5.2.
+/* socks5.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -1694,7 +1694,6 @@ socks4_server_reply_new(void)
   socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t));
   if (NULL == val)
     return NULL;
-  val->version = 4;
   return val;
 }
 
@@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp)
 int
 socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val)
 {
-  if (! ((val == 4))) {
+  if (! ((val == 0 || val == 4))) {
      TRUNNEL_SET_ERROR_CODE(inp);
      return -1;
   }
@@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj)
     return "Object was NULL";
   if (obj->trunnel_error_code_)
     return "A set function failed on this object";
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     return "Integer out of bounds";
   return NULL;
 }
@@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj)
      return -1;
 
 
-  /* Length of u8 version IN [4] */
+  /* Length of u8 version IN [0, 4] */
   result += 1;
 
   /* Length of u8 status */
@@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser
   trunnel_assert(encoded_len >= 0);
 #endif
 
-  /* Encode u8 version IN [4] */
+  /* Encode u8 version IN [0, 4] */
   trunnel_assert(written <= avail);
   if (avail - written < 1)
     goto truncated;
@@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input,
   ssize_t result = 0;
   (void)result;
 
-  /* Parse u8 version IN [4] */
+  /* Parse u8 version IN [0, 4] */
   CHECK_REMAINING(1, truncated);
   obj->version = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     goto fail;
 
   /* Parse u8 status */
diff --git a/src/trunnel/socks5.h b/src/trunnel/socks5.h
index d3bea152e7..23ac64faba 100644
--- a/src/trunnel/socks5.h
+++ b/src/trunnel/socks5.h
@@ -1,4 +1,4 @@
-/* socks5.h -- generated by Trunnel v1.5.2.
+/* socks5.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel
index b86ec03b9d..17fa2ed996 100644
--- a/src/trunnel/socks5.trunnel
+++ b/src/trunnel/socks5.trunnel
@@ -86,7 +86,7 @@ struct socks4_client_request {
 }
 
 struct socks4_server_reply {
-   u8 version IN [4];
+   u8 version IN [0,4];
    u8 status;
    u16 port;
    u32 addr;
diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h
index c4118fce4c..80da371560 100644
--- a/src/trunnel/trunnel-local.h
+++ b/src/trunnel/trunnel-local.h
@@ -14,5 +14,6 @@
 #define trunnel_reallocarray tor_reallocarray
 #define trunnel_assert tor_assert
 #define trunnel_memwipe(mem, len) memwipe((mem), 0, (len))
+#define trunnel_abort tor_abort_
 
 #endif
diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h
index ec7a5b2429..38464f1c59 100644
--- a/src/win32/orconfig.h
+++ b/src/win32/orconfig.h
@@ -218,7 +218,7 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.3.5.11-dev"
+#define VERSION "0.4.2.8-dev"